From c51e8a9698a728e664448ae3f0a441a6643249b0 Mon Sep 17 00:00:00 2001 From: sunnycase Date: Wed, 20 Sep 2023 19:26:59 +0800 Subject: [PATCH 001/132] Add ncnn backend --- .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 19 ++ .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnUnary.cs | 56 ++++ .../Nncase.Modules.Ncnn.csproj | 20 ++ .../Passes/Rules/Ncnn/LowerUnary.cs | 47 +++ .../Nncase.Modules.Ncnn/packages.lock.json | 282 ++++++++++++++++++ nncase.sln | 7 + src/Nncase.Core/IR/TypePattern.cs | 4 +- 7 files changed, 433 insertions(+), 2 deletions(-) create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnUnary.cs create mode 100644 modules/Nncase.Modules.Ncnn/Nncase.Modules.Ncnn.csproj create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnary.cs create mode 100644 modules/Nncase.Modules.Ncnn/packages.lock.json diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs new file mode 100644 index 0000000000..f2e2e7c6c4 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -0,0 +1,19 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR.Ncnn; + +namespace Nncase.IR.F; + +public sealed class Ncnn +{ + public static Call NcnnUnary(UnaryOperationType unaryOp, Expr expr) + { + return new Call(new NcnnUnary(unaryOp), expr); + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnUnary.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnUnary.cs new file mode 100644 index 0000000000..ec2cb4065e --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnUnary.cs @@ -0,0 +1,56 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +public enum UnaryOperationType +{ + ABS = 0, + NEG = 1, + FLOOR = 2, + CEIL = 3, + SQUARE = 4, + SQRT = 5, + RSQRT = 6, + EXP = 7, + LOG = 8, + SIN = 9, + COS = 10, + TAN = 11, + ASIN = 12, + ACOS = 13, + ATAN = 14, + RECIPROCAL = 15, + TANH = 16, + LOG10 = 17, + ROUND = 18, + TRUNC = 19, +} + +/// +/// Unary expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnUnary : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnUnary), 0, "input"); + + public UnaryOperationType OpType { get; } + + /// + public override string DisplayProperty() + { + return $"UnaryOp.{OpType}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Nncase.Modules.Ncnn.csproj b/modules/Nncase.Modules.Ncnn/Nncase.Modules.Ncnn.csproj new file mode 100644 index 0000000000..3d14c19536 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Nncase.Modules.Ncnn.csproj @@ -0,0 +1,20 @@ + + + + Nncase + enable + true + true + True + + + + + + + + + + + + diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnary.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnary.cs new file mode 100644 index 0000000000..e756588078 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnary.cs @@ -0,0 +1,47 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Math; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerUnary : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsUnary( + target_name: "unary", + _ => true, + IsWildcard("input") with { TypePattern = IsFloat() & HasRank(x => x <= 3) }); + + private static UnaryOperationType? MapUnaryOp(UnaryOp unaryOp) => + unaryOp switch + { + UnaryOp.Abs => UnaryOperationType.ABS, + UnaryOp.Acos => UnaryOperationType.ACOS, + _ => null, + }; + + private Expr? GetReplace(Unary unary, Expr input) + { + if (MapUnaryOp(unary.UnaryOp) is UnaryOperationType op) + { + return NcnnUnary(op, input); + } + + return null; + } +} diff --git a/modules/Nncase.Modules.Ncnn/packages.lock.json b/modules/Nncase.Modules.Ncnn/packages.lock.json new file mode 100644 index 0000000000..b69bdc40f8 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/packages.lock.json @@ -0,0 +1,282 @@ +{ + "version": 2, + "dependencies": { + "net7.0": { + "StyleCop.Analyzers": { + "type": "Direct", + "requested": "[1.2.0-beta.507, )", + "resolved": "1.2.0-beta.507", + "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "dependencies": { + "StyleCop.Analyzers.Unstable": "1.2.0.507" + } + }, + "Google.OrTools.runtime.linux-arm64": { + "type": "Transitive", + "resolved": "9.4.1874", + "contentHash": "Z46ndZcZa2Lt5b76xU9kxVYbPLg/LfuMufhUVsu3Qo3L7Bibf7WXd9j7RRldjnuv8RIHWTqb0b+2FwwMxs0c5A==" + }, + "Google.OrTools.runtime.linux-x64": { + "type": "Transitive", + "resolved": "9.4.1874", + "contentHash": "zGeDb8FuvP9HXjrsU7krVXtSDFpR+DUGNEsH51k94jL9tzf2vWYI8+WUBRHZ/cGe50dpLr+vIjfcNo3gFyOpkQ==" + }, + "Google.OrTools.runtime.osx-arm64": { + "type": "Transitive", + "resolved": "9.4.1874", + "contentHash": "Wo0ZfDaH6DhiQw0jZm4HWJm/oPGPpWNwOLUz+EYaoH3MLtocSxItHGQj/Ta3HyhXnYNOv+TliAH8L+8RCXu/2w==" + }, + "Google.OrTools.runtime.osx-x64": { + "type": "Transitive", + "resolved": "9.4.1874", + "contentHash": "IAfGgKR1og6vU87axK1d37Ak/4jy8B4NMoElovG/KZc/2UY+cJEAQDA709UMegtI4lBhuxTWFNUiHQYmRIB9yQ==" + }, + "Google.OrTools.runtime.win-x64": { + "type": "Transitive", + "resolved": "9.4.1874", + "contentHash": "fUs5qDnZA6itygolcX6nPuachQkY9CVvQbakIzIiRAWKcaj8umQAbFdGwbkyzp3qp34BKW5mtPVsmMyfQBBjOQ==" + }, + "libortki": { + "type": "Transitive", + "resolved": "0.0.2", + "contentHash": "svfuG5mxGY/QC/5DVheHOCELmdSP90RtxQ73j23KarPXZ9ZXW+7v1l5J77hGDyQbEh1BGrnGgKBlyn76RauGHg==", + "dependencies": { + "libortki-linux": "0.0.2", + "libortki-osx": "0.0.2", + "libortki-osx-arm64": "0.0.2", + "libortki-win": "0.0.2" + } + }, + "libortki-linux": { + "type": "Transitive", + "resolved": "0.0.2", + "contentHash": "b04LWD4lgGy60tys3hPFhnUpgWDM6dN5r1PI7GOcPj8VupXCaI70LKNQ5/5twbDE6rkowOGanVTw0S2wBGBqBQ==" + }, + "libortki-osx": { + "type": "Transitive", + "resolved": "0.0.2", + "contentHash": "O6Q9GLULkDkZEPAZJVKLPH0ROXGVOE7BxuddgOcHNK2oiTEM7wIRnzp2OIlYgLpaOLyxJMisbGOhtWgdzt2Wng==" + }, + "libortki-osx-arm64": { + "type": "Transitive", + "resolved": "0.0.2", + "contentHash": "4Qn2dirJmRicnUG945oWpq7HVGwgqCKKxYPMISv/MRvmpZBbXrZ1cVvRaF8WwTu4XXgfKTa1sLv+i8zLifUMeQ==" + }, + "libortki-win": { + "type": "Transitive", + "resolved": "0.0.2", + "contentHash": "HAoROgAKn8XBun11X43HZuspKlo5JGy8/OYw5IUPo7FVh5TCaPrLjGmyGYYZ2dqLlv31yv/b6s254PIRGn95cA==" + }, + "Microsoft.Extensions.Configuration.Abstractions": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "qWzV9o+ZRWq+pGm+1dF+R7qTgTYoXvbyowRoBxQJGfqTpqDun2eteerjRQhq5PQ/14S+lqto3Ft4gYaRyl4rdQ==", + "dependencies": { + "Microsoft.Extensions.Primitives": "6.0.0" + } + }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "xlzi2IYREJH3/m6+lUrQlujzX8wDitm4QGnUu6kUXTQAWPuZY8i+ticFJbzfqaetLA6KR/rO6Ew/HuYD+bxifg==" + }, + "Microsoft.Extensions.FileProviders.Abstractions": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "0pd4/fho0gC12rQswaGQxbU34jOS1TPS8lZPpkFCH68ppQjHNHYle9iRuHeev1LhrJ94YPvzcRd8UmIuFk23Qw==", + "dependencies": { + "Microsoft.Extensions.Primitives": "6.0.0" + } + }, + "Microsoft.Extensions.Primitives": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "9+PnzmQFfEFNR9J2aDTfJGGupShHjOuGw4VUv+JB044biSHrnmCIMD+mJHmb2H7YryrfBEXDurxQ47gJZdCKNQ==", + "dependencies": { + "System.Runtime.CompilerServices.Unsafe": "6.0.0" + } + }, + "NetFabric.Hyperlinq.Abstractions": { + "type": "Transitive", + "resolved": "1.3.0", + "contentHash": "WXnEcGwmXfa8gW9N2MlcaPNUzM3NLMwnAhacbtH554F8YcoXbIkTB+uGa1Aa+9gyb/9JZgYVHnmADgJUKP52nA==" + }, + "StyleCop.Analyzers.Unstable": { + "type": "Transitive", + "resolved": "1.2.0.507", + "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + }, + "System.Buffers": { + "type": "Transitive", + "resolved": "4.5.1", + "contentHash": "Rw7ijyl1qqRS0YQD/WycNst8hUUMgrMH4FCn1nNm27M4VxchZ1js3fVjQaANHO5f3sN4isvP4a+Met9Y4YomAg==" + }, + "System.Runtime.CompilerServices.Unsafe": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg==" + }, + "nncase.codegen": { + "type": "Project", + "dependencies": { + "Extension.Mathematics": "[1.2.12, )", + "Nncase.Core": "[1.0.0, )", + "Nncase.IO": "[1.0.0, )" + } + }, + "nncase.core": { + "type": "Project", + "dependencies": { + "DryIoc.dll": "[5.3.1, )", + "GiGraph.Dot": "[2.0.0, )", + "Microsoft.Extensions.Hosting.Abstractions": "[6.0.0, )", + "Microsoft.Extensions.Logging.Abstractions": "[6.0.0, )", + "Microsoft.Extensions.Options": "[6.0.0, )", + "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", + "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.Reactive": "[5.0.0, )" + } + }, + "nncase.egraph": { + "type": "Project", + "dependencies": { + "GiGraph.Dot": "[2.0.0, )", + "Google.OrTools": "[9.4.1874, )", + "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "Nncase.Core": "[1.0.0, )", + "Nncase.Evaluator": "[1.0.0, )", + "Singulink.Collections.Weak": "[1.0.2, )" + } + }, + "nncase.evaluator": { + "type": "Project", + "dependencies": { + "Nncase.Core": "[1.0.0, )", + "OrtKISharp": "[0.0.2, )" + } + }, + "nncase.graph": { + "type": "Project", + "dependencies": { + "Nncase.Core": "[1.0.0, )", + "Nncase.Evaluator": "[1.0.0, )" + } + }, + "nncase.io": { + "type": "Project" + }, + "nncase.passes": { + "type": "Project", + "dependencies": { + "Nncase.Core": "[1.0.0, )", + "Nncase.EGraph": "[1.0.0, )", + "Nncase.Evaluator": "[1.0.0, )", + "Nncase.Graph": "[1.0.0, )" + } + }, + "DryIoc.dll": { + "type": "CentralTransitive", + "requested": "[5.3.1, )", + "resolved": "5.3.1", + "contentHash": "E3zclUh2CIBks1t2uBD1k18pyGFJ1YSKCrbCDbB7qCdl2RAB+k68AyDpjeplhF1ot2XPV82AgyCWBXMf0ggL1g==" + }, + "Extension.Mathematics": { + "type": "CentralTransitive", + "requested": "[1.2.12, )", + "resolved": "1.2.12", + "contentHash": "D4mn5Cab4ztPLJ0V8uMErDrO/Y61098nwrvyIOLZymVAYOQcwP1vomVWKbTagf1aPU3cX5Q7adZtQEQwOy6XEg==" + }, + "GiGraph.Dot": { + "type": "CentralTransitive", + "requested": "[2.0.0, )", + "resolved": "2.0.0", + "contentHash": "ThvS2mQVveSkTMUm04tMbRYzu1XFPV8xBHISrUMp02APjhv9IRbLu3v3upTPCywORx2Ds/c6AqEUL1WU6kPfuQ==" + }, + "Google.OrTools": { + "type": "CentralTransitive", + "requested": "[9.4.1874, )", + "resolved": "9.4.1874", + "contentHash": "jqRoI+pYlym+fhoU25u+13oti5h+772bllQ9zDitTVMclDXVTiG6pxzvmYO74wnADBMdpb2SQlgiNQxoNk5dlA==", + "dependencies": { + "Google.OrTools.runtime.linux-arm64": "9.4.1874", + "Google.OrTools.runtime.linux-x64": "9.4.1874", + "Google.OrTools.runtime.osx-arm64": "9.4.1874", + "Google.OrTools.runtime.osx-x64": "9.4.1874", + "Google.OrTools.runtime.win-x64": "9.4.1874", + "Google.Protobuf": "3.19.4" + } + }, + "Google.Protobuf": { + "type": "CentralTransitive", + "requested": "[3.19.4, )", + "resolved": "3.19.4", + "contentHash": "fd07/ykL4O4FhqrZIELm5lmiyOHfdPg9+o+hWr6tcfRdS7tHXnImg/2wtogLzlW2eEmr0J7j6ZrZvaWOLiJbxQ==" + }, + "Microsoft.Extensions.Hosting.Abstractions": { + "type": "CentralTransitive", + "requested": "[6.0.0, )", + "resolved": "6.0.0", + "contentHash": "GcT5l2CYXL6Sa27KCSh0TixsRfADUgth+ojQSD5EkzisZxmGFh7CwzkcYuGwvmXLjr27uWRNrJ2vuuEjMhU05Q==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "6.0.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", + "Microsoft.Extensions.FileProviders.Abstractions": "6.0.0" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "CentralTransitive", + "requested": "[6.0.0, )", + "resolved": "6.0.0", + "contentHash": "/HggWBbTwy8TgebGSX5DBZ24ndhzi93sHUBDvP1IxbZD7FDokYzdAr6+vbWGjw2XAfR2EJ1sfKUotpjHnFWPxA==" + }, + "Microsoft.Extensions.Options": { + "type": "CentralTransitive", + "requested": "[6.0.0, )", + "resolved": "6.0.0", + "contentHash": "dzXN0+V1AyjOe2xcJ86Qbo233KHuLEY0njf/P2Kw8SfJU+d45HNS2ctJdnEnrWbM9Ye2eFgaC5Mj9otRMU6IsQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", + "Microsoft.Extensions.Primitives": "6.0.0" + } + }, + "Microsoft.Toolkit.HighPerformance": { + "type": "CentralTransitive", + "requested": "[7.1.1, )", + "resolved": "7.1.1", + "contentHash": "TRnvDpZPXO30hTOtjfLw6Y9BtTKtTpzk9lefeh4RMCaUihWrVKQR454nYH4/mMJAh+LXqfAPyk0kfkJs0Amopw==" + }, + "NetFabric.Hyperlinq": { + "type": "CentralTransitive", + "requested": "[3.0.0-beta48, )", + "resolved": "3.0.0-beta48", + "contentHash": "oYUhXvxNS8bBJWqNkvx5g8y0P/0LtyqS2pN0w4OWjVDNWEpLbdbvPy9w/9z1n2PrqIjX3jxUsEnoCmxxGnI3gw==", + "dependencies": { + "NetFabric.Hyperlinq.Abstractions": "1.3.0", + "System.Buffers": "4.5.1", + "System.Runtime.CompilerServices.Unsafe": "5.0.0" + } + }, + "OrtKISharp": { + "type": "CentralTransitive", + "requested": "[0.0.2, )", + "resolved": "0.0.2", + "contentHash": "q8j0yR5836Zhv9WB9BFkQt1UaEFyibq8bqJcTiULlILF6/sz8z7Wy2N8sgYdDKsdW25zncIz7j6IDbKM5ynePg==", + "dependencies": { + "libortki": "0.0.2" + } + }, + "Singulink.Collections.Weak": { + "type": "CentralTransitive", + "requested": "[1.0.2, )", + "resolved": "1.0.2", + "contentHash": "giLAHrjJe0Bh7yhNexR6pmcv02+Fi+lEPxQVdB8zvkuJCmy6rnqu8CZLIpxrUfLcWDuTCSiK0IfGmMhig3UDhA==" + }, + "System.Reactive": { + "type": "CentralTransitive", + "requested": "[5.0.0, )", + "resolved": "5.0.0", + "contentHash": "erBZjkQHWL9jpasCE/0qKAryzVBJFxGHVBAvgRN1bzM0q2s1S4oYREEEL0Vb+1kA/6BKb5FjUZMp5VXmy+gzkQ==" + } + } + } +} \ No newline at end of file diff --git a/nncase.sln b/nncase.sln index afde606dc0..6d00ea995c 100644 --- a/nncase.sln +++ b/nncase.sln @@ -79,6 +79,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nncase.Tests.TestFixture", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nncase.Passes", "src\Nncase.Passes\Nncase.Passes.csproj", "{E6462E82-B48F-4AFA-AE34-725EF0A9CB42}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nncase.Modules.Ncnn", "modules\Nncase.Modules.Ncnn\Nncase.Modules.Ncnn.csproj", "{FA535CB0-C8B9-4CA5-B3E3-8B7912E5F061}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -173,6 +175,10 @@ Global {E6462E82-B48F-4AFA-AE34-725EF0A9CB42}.Debug|Any CPU.Build.0 = Debug|Any CPU {E6462E82-B48F-4AFA-AE34-725EF0A9CB42}.Release|Any CPU.ActiveCfg = Release|Any CPU {E6462E82-B48F-4AFA-AE34-725EF0A9CB42}.Release|Any CPU.Build.0 = Release|Any CPU + {FA535CB0-C8B9-4CA5-B3E3-8B7912E5F061}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FA535CB0-C8B9-4CA5-B3E3-8B7912E5F061}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FA535CB0-C8B9-4CA5-B3E3-8B7912E5F061}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FA535CB0-C8B9-4CA5-B3E3-8B7912E5F061}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -203,6 +209,7 @@ Global {E365B1B1-4D13-4839-9763-A7A7C5F32FD4} = {BCA74168-F015-4B5B-B4CD-C83AE06B9822} {98A03405-CA53-4EC4-9B18-94D1C8DF9453} = {E5A4516C-4080-4346-991D-57A7AA76ADA6} {E6462E82-B48F-4AFA-AE34-725EF0A9CB42} = {BCA74168-F015-4B5B-B4CD-C83AE06B9822} + {FA535CB0-C8B9-4CA5-B3E3-8B7912E5F061} = {9859F5E8-5504-4AFE-B955-9497A0A0CD66} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9492E141-292E-4D60-9C6E-3738AB234DB2} diff --git a/src/Nncase.Core/IR/TypePattern.cs b/src/Nncase.Core/IR/TypePattern.cs index 6eea450f5c..a0d3c269ba 100644 --- a/src/Nncase.Core/IR/TypePattern.cs +++ b/src/Nncase.Core/IR/TypePattern.cs @@ -133,8 +133,8 @@ public static TypePattern HasShape(Shape target_shape) => HasShape( /// /// is custom rank. /// - public static TypePattern HasRank(Func cond, string reason) => HasShape( - inshape => cond(inshape.Rank), reason); + public static TypePattern HasRank(Func cond, [CallerArgumentExpression(nameof(cond))] string? reason = null) => HasShape( + inshape => cond(inshape.Rank), reason!); /// /// is target rank. From 121cb8dd131c1236f086f7de5254b289c15086f1 Mon Sep 17 00:00:00 2001 From: sunnycase Date: Tue, 17 Oct 2023 18:02:12 +0800 Subject: [PATCH 002/132] Pass test softmax --- CMakeLists.txt | 1 + cmake/dependencies.cmake | 2 + conanfile.py | 1 + .../CodeGen/Ncnn/NcnnEmitter.cs | 60 +++++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 97 +++++++++ .../CodeGen/Ncnn/NcnnLinkableFunction.cs | 43 ++++ .../CodeGen/Ncnn/NcnnLinkableModule.cs | 26 +++ .../CodeGen/Ncnn/NcnnLinkedModule.cs | 31 +++ .../CodeGen/Ncnn/NcnnModuleBuilder.cs | 32 +++ .../Evaluator/Ncnn/NcnnModule.cs | 18 ++ .../Evaluator/Ncnn/NcnnSoftmax.cs | 69 ++++++ .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 9 +- .../IR/Ncnn/NcnnSoftmax.cs | 32 +++ .../NcnnApplicationPart.cs | 30 +++ .../Passes/Rules/Ncnn/LowerSoftmax.cs | 34 +++ .../Passes/Rules/Ncnn/LowerUnary.cs | 3 +- .../Nncase.Modules.StackVM.csproj | 1 + .../Targets/CPUTarget.cs | 12 + .../Nncase.Modules.StackVM/packages.lock.json | 7 + modules/ncnn/CMakeLists.txt | 59 +++++ .../nncase/runtime/ncnn/compiler_defs.h | 28 +++ .../nncase/runtime/ncnn/runtime_module.h | 28 +++ modules/ncnn/src/runtime/CMakeLists.txt | 17 ++ modules/ncnn/src/runtime/runtime_function.cpp | 205 ++++++++++++++++++ modules/ncnn/src/runtime/runtime_function.h | 45 ++++ modules/ncnn/src/runtime/runtime_module.cpp | 65 ++++++ modules/ncnn/src/runtime/runtime_module.h | 42 ++++ modules/ncnn/src/runtime/tensor_utils.h | 55 +++++ .../include/nncase/runtime/stream_reader.h | 26 +++ src/Native/src/runtime/CMakeLists.txt | 3 +- src/Nncase.Cli/packages.lock.json | 8 + src/Nncase.CodeGen/CodeGen/LinkedSection.cs | 17 ++ .../Hosting/CompilerHostBuilderExtensions.cs | 3 +- src/Nncase.Compiler/Hosting/PluginLoader.cs | 1 + src/Nncase.Compiler/packages.lock.json | 8 + src/Nncase.Core/IR/ExprCollector.cs | 15 +- src/Nncase.Core/IR/Function.cs | 12 +- src/Nncase.Core/Runtime/Ncnn/NcnnLayer.cs | 62 ++++++ .../Runtime}/Ncnn/NcnnModel.cs | 58 ++--- .../Runtime}/Ncnn/NcnnModelBin.cs | 4 +- .../Runtime}/Ncnn/ParamDict.cs | 63 +++++- src/Nncase.Importer/Ncnn/Concat.cs | 1 + src/Nncase.Importer/Ncnn/Convolution.cs | 1 + src/Nncase.Importer/Ncnn/InnerProduct.cs | 1 + src/Nncase.Importer/Ncnn/NcnnImporter.cs | 1 + src/Nncase.Importer/Ncnn/Pooling.cs | 1 + src/Nncase.Importer/Ncnn/ShuffleChannel.cs | 1 + src/Nncase.Importer/Ncnn/Softmax.cs | 1 + src/Nncase.Importer/Ncnn/Split.cs | 1 + .../Transforms/FusionToFunctionPass.cs | 45 ++++ .../packages.lock.json | 8 + src/Nncase.Tests/packages.lock.json | 8 + 52 files changed, 1342 insertions(+), 59 deletions(-) create mode 100644 modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs create mode 100644 modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs create mode 100644 modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkableFunction.cs create mode 100644 modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkableModule.cs create mode 100644 modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkedModule.cs create mode 100644 modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnModuleBuilder.cs create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSoftmax.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSoftmax.cs create mode 100644 modules/Nncase.Modules.Ncnn/NcnnApplicationPart.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftmax.cs create mode 100644 modules/ncnn/CMakeLists.txt create mode 100644 modules/ncnn/include/nncase/runtime/ncnn/compiler_defs.h create mode 100644 modules/ncnn/include/nncase/runtime/ncnn/runtime_module.h create mode 100644 modules/ncnn/src/runtime/CMakeLists.txt create mode 100644 modules/ncnn/src/runtime/runtime_function.cpp create mode 100644 modules/ncnn/src/runtime/runtime_function.h create mode 100644 modules/ncnn/src/runtime/runtime_module.cpp create mode 100644 modules/ncnn/src/runtime/runtime_module.h create mode 100644 modules/ncnn/src/runtime/tensor_utils.h create mode 100644 src/Nncase.Core/Runtime/Ncnn/NcnnLayer.cs rename src/{Nncase.Importer => Nncase.Core/Runtime}/Ncnn/NcnnModel.cs (71%) rename src/{Nncase.Importer => Nncase.Core/Runtime}/Ncnn/NcnnModelBin.cs (96%) rename src/{Nncase.Importer => Nncase.Core/Runtime}/Ncnn/ParamDict.cs (69%) create mode 100644 src/Nncase.Passes/Transforms/FusionToFunctionPass.cs diff --git a/CMakeLists.txt b/CMakeLists.txt index 05ab14e495..2b2959b62e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -263,5 +263,6 @@ if(BUILD_TESTING) endif() # Modules +add_subdirectory(modules/ncnn) #add_subdirectory(modules/k210) #add_subdirectory(modules/vulkan) diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 85e8e1213e..143e21b52a 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -1,4 +1,6 @@ find_package(gsl-lite REQUIRED) +find_package(ncnn REQUIRED) + if (ENABLE_OPENMP) find_package(OpenMP COMPONENTS CXX REQUIRED) endif () diff --git a/conanfile.py b/conanfile.py index 8a3a0c72b7..46e0a74fcf 100644 --- a/conanfile.py +++ b/conanfile.py @@ -48,6 +48,7 @@ def imports(self): def requirements(self): self.requires('gsl-lite/0.37.0') self.requires('hkg/0.0.1') + self.requires('ncnn/20230816') if self.options.tests: self.requires('gtest/1.10.0') self.requires('ortki/0.0.2') diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs new file mode 100644 index 0000000000..91f708479f --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -0,0 +1,60 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.Runtime.Ncnn; + +namespace Nncase.CodeGen.Ncnn; + +internal class NcnnEmitter +{ + private readonly NcnnModel _model; + private readonly BinaryWriter _binWriter; + + public NcnnEmitter(BinaryWriter binWriter) + { + _model = new NcnnModel(); + _binWriter = binWriter; + } + + public void SaveParam(Stream paramStream) + { + using var sw = new StreamWriter(paramStream, Encoding.ASCII, leaveOpen: true); + _model.Serialize(sw); + } + + public void Input(string name) => + AddLayer("Input", name, Array.Empty(), new[] { name }, null); + + public void Softmax(string name, string input, int axis) => + AddLayer("Softmax", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = axis }, // axis + [1] = new ParamValue { Kind= ParamKind.Int, IntValue = 1 }, // fixbug0 + }); + + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null) + { + var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); + if (paramDict != null) + { + layer.ParamDict = paramDict; + } + + for (int i = 0; i < bottoms.Length; i++) + { + layer.Bottoms[i] = new NcnnTensor { Name = bottoms[i] }; + } + + for (int i = 0; i < tops.Length; i++) + { + layer.Tops[i] = new NcnnTensor { Name = tops[i] }; + } + + _model.Layers.Add(layer); + } +} diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs new file mode 100644 index 0000000000..e0f7ca488f --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -0,0 +1,97 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Reactive; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.Diagnostics; +using Nncase.IR; +using Nncase.IR.Ncnn; +using Nncase.IR.Tensors; +using Nncase.Utilities; +using static Nncase.CodeGen.CodeGenDumper; + +namespace Nncase.CodeGen.Ncnn; + +/// +/// Ncnn function builder. +/// +internal class NcnnFunctionBuilder : FunctionBuilder +{ + private readonly NcnnEmitter _emitter; + private string[]? _inputs; + private string[]? _outputs; + + public NcnnFunctionBuilder(uint id, SectionManager sectionManager) + : base(id, sectionManager) + { + _emitter = new NcnnEmitter(sectionManager.GetWriter(WellknownSectionNames.Rdata)); + } + + protected override ILinkableFunction CreateLinkableFunction(uint id, BaseFunction callable, IReadOnlyList functionRefs, Stream text) + { + return new NcnnLinkableFunction(id, callable, functionRefs, text, _inputs!, _outputs!); + } + + protected override void Compile(BaseFunction callable) + { + var visitor = new CodeGenVisitor(_emitter); + _outputs = visitor.Visit(callable).Split(','); + _inputs = visitor.Inputs.ToArray(); + } + + protected override void WriteText() + { + _emitter.SaveParam(TextWriter.BaseStream); + } + + private class CodeGenVisitor : ExprVisitor + { + private readonly NcnnEmitter _emitter; + + private int _layerId; + + public CodeGenVisitor(NcnnEmitter emitter) + { + _emitter = emitter; + } + + public List Inputs { get; } = new(); + + protected override string VisitLeafVar(Var expr) + { + var name = GetNextName(); + _emitter.Input(name); + Inputs.Add(name); + return name; + } + + protected override string VisitLeafFunction(Function expr) => ExprMemo[expr.Body]; + + protected override string VisitLeafOp(Op expr) => string.Empty; + + protected override string VisitLeafTuple(IR.Tuple expr) => StringUtility.Join(",", expr.Fields.AsValueEnumerable().Select(x => ExprMemo[x])); + + protected override string VisitLeafCall(Call expr) + { + var name = GetNextName(); + switch (expr.Target) + { + case NcnnSoftmax op: + _emitter.Softmax(name, ExprMemo[expr.Arguments[0]], op.Axis); + break; + default: + throw new NotSupportedException(); + } + + return name; + } + + private string GetNextName() => $"layer{_layerId++}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkableFunction.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkableFunction.cs new file mode 100644 index 0000000000..21a28b70d7 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkableFunction.cs @@ -0,0 +1,43 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; + +namespace Nncase.CodeGen.Ncnn; + +internal class NcnnLinkableFunction : ILinkableFunction +{ + public NcnnLinkableFunction(uint id, BaseFunction sourceFunction, IEnumerable functionRefs, Stream text, string[] inputs, string[] outputs) + { + Id = id; + SourceFunction = sourceFunction; + FunctionRefs = functionRefs; + Text = text; + Inputs = inputs; + Outputs = outputs; + Sections = new[] + { + LinkedSection.FromStrings(inputs, ".inputs"), + LinkedSection.FromStrings(outputs, ".outputs"), + }; + } + + public uint Id { get; } + + public BaseFunction SourceFunction { get; } + + public IEnumerable FunctionRefs { get; } + + public Stream Text { get; } + + public string[] Inputs { get; } + + public string[] Outputs { get; } + + public IReadOnlyList Sections { get; } +} diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkableModule.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkableModule.cs new file mode 100644 index 0000000000..5ea4a751ba --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkableModule.cs @@ -0,0 +1,26 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Nncase.CodeGen.Ncnn; + +internal class NcnnLinkableModule : LinkableModule +{ + public NcnnLinkableModule(IReadOnlyList functions, SectionManager sectionManager) + : base(functions, sectionManager) + { + } + + protected override ILinkedModule CreateLinkedModule(IReadOnlyList linkedFunctions, Stream text) + { + return new NcnnLinkedModule( + linkedFunctions, + text, + SectionManager.GetContent(WellknownSectionNames.Rdata)); + } +} diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkedModule.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkedModule.cs new file mode 100644 index 0000000000..078f9c903c --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkedModule.cs @@ -0,0 +1,31 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Nncase.CodeGen.Ncnn; + +internal class NcnnLinkedModule : ILinkedModule +{ + public NcnnLinkedModule(IReadOnlyList functions, Stream text, Stream? rdata) + { + Functions = functions; + Sections = new[] + { + new LinkedSection(text, ".text", 0, 8, (uint)text.Length), + new LinkedSection(rdata, ".rdata", 0, 8, (uint)(rdata?.Length ?? 0)), + }; + } + + public string ModuleKind => "ncnn"; + + public uint Version => 1; + + public IReadOnlyList Functions { get; } + + public IReadOnlyList Sections { get; } +} diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnModuleBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnModuleBuilder.cs new file mode 100644 index 0000000000..3234727e69 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnModuleBuilder.cs @@ -0,0 +1,32 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; + +namespace Nncase.CodeGen.Ncnn; + +/// +/// Ncnn module builder. +/// +public class NcnnModuleBuilder : ModuleBuilder +{ + /// + public override string ModuleKind => "ncnn"; + + /// + protected override ILinkableModule CreateLinkableModule(IReadOnlyList linkableFunctions) + { + return new NcnnLinkableModule(linkableFunctions, SectionManager); + } + + /// + protected override FunctionBuilder CreateFunctionBuilder(uint id) + { + return new NcnnFunctionBuilder(id, SectionManager); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs new file mode 100644 index 0000000000..b1504cfbb8 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -0,0 +1,18 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using DryIoc; +using Nncase.Hosting; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Ncnn module. +/// +internal class NcnnModule : IApplicationPart +{ + public void ConfigureServices(IRegistrator registrator) + { + registrator.RegisterManyInterface(reuse: Reuse.Singleton); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSoftmax.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSoftmax.cs new file mode 100644 index 0000000000..36a962d3d9 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSoftmax.cs @@ -0,0 +1,69 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnSoftmaxEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnSoftmax softmax) + { + var input = context.GetOrtArgumentValue(softmax, NcnnSoftmax.Input); + var dim = softmax.Axis; + return OrtKI.Softmax(input, dim).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnSoftmax target) + { + var input = context.CheckArgumentType(target, NcnnSoftmax.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnSoftmax target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnSoftmax target) + { + var inputType = context.GetArgumentType(target, NcnnSoftmax.Input); + var returnType = context.GetReturnType(); + var returnF = MetricUtility.GetFLOPs(returnType); + var inputF = MetricUtility.GetFLOPs(inputType); + var inner = inputF / returnF; + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = (inner * 2) + (inputF * (MetricUtility.SubFLOPs + MetricUtility.ExpFLOPs + MetricUtility.DivFLOPs)), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnSoftmax target) => context.GetArgumentShape(target, NcnnSoftmax.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index f2e2e7c6c4..a98aa81965 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -12,8 +12,9 @@ namespace Nncase.IR.F; public sealed class Ncnn { - public static Call NcnnUnary(UnaryOperationType unaryOp, Expr expr) - { - return new Call(new NcnnUnary(unaryOp), expr); - } + public static Call NcnnUnary(Expr expr, UnaryOperationType unaryOp) => + new Call(new NcnnUnary(unaryOp), expr); + + public static Call NcnnSoftmax(Expr expr, int axis) => + new Call(new NcnnSoftmax(axis), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSoftmax.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSoftmax.cs new file mode 100644 index 0000000000..298e755e8d --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSoftmax.cs @@ -0,0 +1,32 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Softmax expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnSoftmax : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnSoftmax), 0, "input"); + + public int Axis { get; } + + /// + public override string DisplayProperty() + { + return $"{Axis}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/NcnnApplicationPart.cs b/modules/Nncase.Modules.Ncnn/NcnnApplicationPart.cs new file mode 100644 index 0000000000..e492c65665 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/NcnnApplicationPart.cs @@ -0,0 +1,30 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading.Tasks; +using DryIoc; +using Nncase.Evaluator.Ncnn; +using Nncase.Hosting; + +namespace Nncase; + +/// +/// Ncnn application part extensions. +/// +public static class NcnnApplicationPart +{ + /// + /// Add ncnn assembly. + /// + /// Service registrator. + /// Configured service registrator. + public static IRegistrator AddNcnn(this IRegistrator registrator) + { + return registrator.RegisterModule(); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftmax.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftmax.cs new file mode 100644 index 0000000000..4954908a3c --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftmax.cs @@ -0,0 +1,34 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerSoftmax : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsSoftmax( + IsWildcard("input") with { TypePattern = IsFloat() & HasRank(x => x <= 3) }, + IsTensorConst("axis")); + + private Expr? GetReplace(Expr input, int axis) + { + var newInput = new Var(input.CheckedType); + return new Call(new Fusion("ncnn", NcnnSoftmax(newInput, axis), new[] { newInput }), input); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnary.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnary.cs index e756588078..7b1ec3f4f8 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnary.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnary.cs @@ -39,7 +39,8 @@ public partial class LowerUnary : RewriteRule { if (MapUnaryOp(unary.UnaryOp) is UnaryOperationType op) { - return NcnnUnary(op, input); + var newInput = new Var(input.CheckedType); + return new Call(new Fusion("ncnn", NcnnUnary(newInput, op), new[] { newInput }), input); } return null; diff --git a/modules/Nncase.Modules.StackVM/Nncase.Modules.StackVM.csproj b/modules/Nncase.Modules.StackVM/Nncase.Modules.StackVM.csproj index 3d14c19536..45983364c7 100644 --- a/modules/Nncase.Modules.StackVM/Nncase.Modules.StackVM.csproj +++ b/modules/Nncase.Modules.StackVM/Nncase.Modules.StackVM.csproj @@ -15,6 +15,7 @@ + diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index ba77ae58f5..ede3397fa5 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -9,9 +9,11 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Options; using Nncase.CodeGen; +using Nncase.CodeGen.Ncnn; using Nncase.CodeGen.StackVM; using Nncase.IR; using Nncase.Passes; +using Nncase.Passes.Transforms; using Nncase.Quantization; namespace Nncase.Targets; @@ -41,6 +43,11 @@ public void RegisterTargetInDependentPass(IPassManager passManager, CompileOptio /// public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions options) { + passManager.AddWithName("LowerNcnnIR").Configure(p => + { + p.Add(); + p.Add(); + }); } /// @@ -75,6 +82,7 @@ public void RegisterTargetDependentAfterQuantPass(IPassManager passManager, Comp public void RegisterTargetDependentBeforeCodeGen(IPassManager passManager, CompileOptions options) { + passManager.Add(); } /// @@ -84,6 +92,10 @@ public IModuleBuilder CreateModuleBuilder(string moduleKind, CompileOptions opti { return new StackVMModuleBuilder(); } + else if (moduleKind == "ncnn") + { + return new NcnnModuleBuilder(); + } else { throw new NotSupportedException($"{moduleKind} module is not supported."); diff --git a/modules/Nncase.Modules.StackVM/packages.lock.json b/modules/Nncase.Modules.StackVM/packages.lock.json index b69bdc40f8..6dc5922aab 100644 --- a/modules/Nncase.Modules.StackVM/packages.lock.json +++ b/modules/Nncase.Modules.StackVM/packages.lock.json @@ -165,6 +165,13 @@ "nncase.io": { "type": "Project" }, + "nncase.modules.ncnn": { + "type": "Project", + "dependencies": { + "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Passes": "[1.0.0, )" + } + }, "nncase.passes": { "type": "Project", "dependencies": { diff --git a/modules/ncnn/CMakeLists.txt b/modules/ncnn/CMakeLists.txt new file mode 100644 index 0000000000..b2d953d3b4 --- /dev/null +++ b/modules/ncnn/CMakeLists.txt @@ -0,0 +1,59 @@ +cmake_minimum_required (VERSION 3.8) + +if (BUILDING_RUNTIME) + add_definitions(-DBUILDING_RUNTIME) +endif() + +include_directories(include) +add_subdirectory(src/runtime) + +if (BUILDING_RUNTIME) + add_library(nncase_rt_modules_ncnn STATIC ${SRCS}) + target_include_directories(nncase_rt_modules_ncnn PRIVATE include) + target_link_libraries(nncase_rt_modules_ncnn PRIVATE runtime_ncnn) + set_target_properties(nncase_rt_modules_ncnn PROPERTIES + OUTPUT_NAME "nncase.rt_modules.ncnn") + + install(DIRECTORY include/nncase/kernels + DESTINATION include/nncase + COMPONENT nncase-headers + FILES_MATCHING + PATTERN "*.def" + PATTERN "*.h" + PATTERN "*.hpp" + PATTERN "*.td" + PATTERN "*.inc" + PATTERN "LICENSE.TXT" + ) + + install(DIRECTORY include/nncase/runtime + DESTINATION include/nncase + COMPONENT nncase-headers + FILES_MATCHING + PATTERN "*.def" + PATTERN "*.h" + PATTERN "*.hpp" + PATTERN "*.td" + PATTERN "*.inc" + PATTERN "LICENSE.TXT" + ) + + install(TARGETS nncase_rt_modules_ncnn EXPORT nncase_rt_modules_ncnnTargets + ARCHIVE DESTINATION lib + LIBRARY DESTINATION lib + RUNTIME DESTINATION bin + INCLUDES DESTINATION include + ) + + install(EXPORT nncase_rt_modules_ncnnTargets + DESTINATION lib/cmake/nncase_rt_modules_ncnn) +else() + add_library(nncase_modules_ncnn SHARED ${SRCS}) + target_include_directories(nncase_modules_ncnn PUBLIC include) + target_link_libraries(nncase_modules_ncnn PRIVATE simulator_ncnn) + set_target_properties(nncase_modules_ncnn PROPERTIES + OUTPUT_NAME "nncase.simulator.ncnn") + install(TARGETS nncase_modules_ncnn + COMPONENT nncase-runtime) + +endif() diff --git a/modules/ncnn/include/nncase/runtime/ncnn/compiler_defs.h b/modules/ncnn/include/nncase/runtime/ncnn/compiler_defs.h new file mode 100644 index 0000000000..2285e8abae --- /dev/null +++ b/modules/ncnn/include/nncase/runtime/ncnn/compiler_defs.h @@ -0,0 +1,28 @@ +/* Copyright 2019-2021 Canaan Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once +#include + +#if defined(_MSC_VER) +#ifdef NNCASE_MODULES_NCNN_DLL +#define NNCASE_MODULES_NCNN_API __declspec(dllexport) +#elif NNCASE_SHARED_LIBS +#define NNCASE_MODULES_NCNN_API __declspec(dllimport) +#else +#define NNCASE_MODULES_NCNN_API +#endif +#else +#define NNCASE_MODULES_NCNN_API __attribute__((visibility("default"))) +#endif diff --git a/modules/ncnn/include/nncase/runtime/ncnn/runtime_module.h b/modules/ncnn/include/nncase/runtime/ncnn/runtime_module.h new file mode 100644 index 0000000000..689b4ae62d --- /dev/null +++ b/modules/ncnn/include/nncase/runtime/ncnn/runtime_module.h @@ -0,0 +1,28 @@ +/* Copyright 2019-2021 Canaan Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once +#include "compiler_defs.h" +#include + +BEGIN_NS_NNCASE_RT_MODULE(ncnn) + +NNCASE_INLINE_VAR constexpr module_kind_t ncnn_module_kind = + to_module_kind("ncnn"); +NNCASE_INLINE_VAR constexpr uint32_t ncnn_module_version = 1; + +NNCASE_MODULES_NCNN_API result> +create_ncnn_runtime_module(); + +END_NS_NNCASE_RT_MODULE diff --git a/modules/ncnn/src/runtime/CMakeLists.txt b/modules/ncnn/src/runtime/CMakeLists.txt new file mode 100644 index 0000000000..c256595b39 --- /dev/null +++ b/modules/ncnn/src/runtime/CMakeLists.txt @@ -0,0 +1,17 @@ +cmake_minimum_required (VERSION 3.13) + +set(SRCS runtime_module.cpp + runtime_function.cpp) + +if (BUILDING_RUNTIME) + add_library(runtime_ncnn OBJECT ${SRCS}) + + target_link_libraries(runtime_ncnn PUBLIC nncaseruntime ncnn::ncnn) + set_target_properties(runtime_ncnn PROPERTIES POSITION_INDEPENDENT_CODE ON) + install(TARGETS runtime_ncnn EXPORT nncase_rt_modules_ncnnTargets) +else() + add_library(simulator_ncnn OBJECT ${SRCS}) + target_link_libraries(simulator_ncnn PRIVATE nncaseruntime ncnn::ncnn) + target_compile_definitions(simulator_ncnn PUBLIC -DNNCASE_MODULES_NCNN_DLL -DNNCASE_SIMULATOR) + set_target_properties(simulator_ncnn PROPERTIES POSITION_INDEPENDENT_CODE ON) +endif() diff --git a/modules/ncnn/src/runtime/runtime_function.cpp b/modules/ncnn/src/runtime/runtime_function.cpp new file mode 100644 index 0000000000..dee711793f --- /dev/null +++ b/modules/ncnn/src/runtime/runtime_function.cpp @@ -0,0 +1,205 @@ +/* Copyright 2019-2021 Canaan Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "runtime_function.h" +#include +#include +#include +#include +#include + +using namespace nncase; +using namespace nncase::runtime; +using namespace nncase::runtime::ncnn; + +namespace { +class DataReaderFromEmpty : public ::ncnn::DataReader { + public: + virtual int scan(const char *format, void *p) const { return 0; } + virtual size_t read(void *buf, size_t size) const { + memset(buf, 0, size); + return size; + } +}; +} // namespace + +ncnn_runtime_function::ncnn_runtime_function(runtime_module &rt_module) + : runtime_function(rt_module) {} + +ncnn_runtime_module &ncnn_runtime_function::module() const noexcept { + return static_cast(runtime_function::module()); +} + +result ncnn_runtime_function::initialize_core( + runtime_function_init_context &context) noexcept { + try_(context.read_section(".inputs", + [this](auto sr, size_t) -> result { + input_names_ = sr.read_string_array(); + return ok(); + })); + try_(context.read_section(".outputs", + [this](auto sr, size_t) -> result { + output_names_ = sr.read_string_array(); + return ok(); + })); + + auto param_mem = reinterpret_cast( + module().text().data() + context.header().entrypoint); + ::ncnn::DataReaderFromMemory paramdr(param_mem); + DataReaderFromEmpty bindr; + + CHECK_WITH_ERR(!net_.load_param(paramdr), std::errc::invalid_argument); + CHECK_WITH_ERR(!net_.load_model(bindr), std::errc::invalid_argument); + return ok(); +} + +result ncnn_runtime_function::invoke_core( + gsl::span parameters, + [[maybe_unused]] value_t return_value) noexcept { + + auto ex = net_.create_extractor(); + + // 1. Set input + for (size_t i = 0; i < parameters.size(); i++) { + try_var(t, parameters[0].as()); + ::ncnn::Mat mat; + mat.elempack = 1; + mat.elemsize = runtime::get_bytes(t->dtype()); + auto shape = t->shape(); + switch (shape.size()) { + case 1: + mat.dims = 1; + mat.w = shape[0]; + mat.h = 1; + mat.d = 1; + mat.c = 1; + mat.cstep = mat.w; + break; + case 2: + mat.dims = 2; + mat.w = shape[1]; + mat.h = shape[0]; + mat.d = 1; + mat.c = 1; + mat.cstep = (size_t)mat.w * mat.h; + break; + case 3: + mat.dims = 3; + mat.w = shape[2]; + mat.h = shape[1]; + mat.d = 1; + mat.c = shape[0]; + mat.cstep = (size_t)mat.w * mat.h; + break; + case 4: + mat.dims = 4; + mat.w = shape[3]; + mat.h = shape[2]; + mat.d = shape[1]; + mat.c = shape[0]; + mat.cstep = (size_t)mat.w * mat.h * mat.d; + break; + default: + return err(std::errc::invalid_argument); + } + + try_var(hb, t->buffer().as_host()); + try_var(map, hb.map(map_read)); + mat.data = map.buffer().data(); + + // Must clone to full fill ncnn implicit contracts: + // 1. Input is not user mananged data + // 2. Data must be aligned + auto cloned_input = mat.clone(); + CHECK_WITH_ERR(!ex.input(input_names_[i].c_str(), cloned_input), + std::errc::invalid_argument); + } + + // 2. Extract outputs + std::vector outputs; + for (size_t i = 0; i < output_names_.size(); i++) { + ::ncnn::Mat mat; + CHECK_WITH_ERR(!ex.extract(output_names_[i].c_str(), mat), + std::errc::invalid_argument); + + gsl::span data{reinterpret_cast(mat.data), + mat.total() * mat.elemsize}; + + dims_t shape; + switch (mat.dims) { + case 1: + shape = {(size_t)mat.w}; + break; + case 2: + shape = {(size_t)mat.h, (size_t)mat.w}; + break; + case 3: + shape = {(size_t)mat.c, (size_t)mat.h, (size_t)mat.w}; + break; + case 4: + shape = {(size_t)mat.c, (size_t)mat.d, (size_t)mat.h, (size_t)mat.w, + 1}; + break; + default: + return err(std::errc::invalid_argument); + } + + datatype_t dt; + switch (mat.elemsize) { + case 1: + dt = datatype_t::uint8; + break; + case 2: + dt = datatype_t::float16; + break; + case 4: + dt = datatype_t::float32; + break; + default: + return err(std::errc::invalid_argument); + } + + buffer_allocate_options options{}; + options.flags = HOST_BUFFER_ALLOCATE_SHARED; + try_var(buf, buffer_allocator::host().allocate( + mat.total() * mat.elemsize, options)); + try_var(hb, buf.as()); + + { + try_var(map, hb->map(map_write)); + auto csize = (size_t)mat.w * mat.h * mat.d * mat.elemsize; + for (size_t i = 0; i < mat.c; i++) { + void *dest = + (unsigned char *)map.buffer().data() + (size_t)i * csize; + const void *src = + (unsigned char *)mat.data + i * mat.cstep * mat.elemsize; + memcpy(dest, src, csize); + } + } + + tensor t(std::in_place, dt, shape, get_default_strides(shape), buf); + outputs.emplace_back(t); + } + + auto ret_val = output_names_.size() == 1 + ? outputs[0] + : tuple(std::in_place, std::move(outputs)); + + if (!return_value.empty()) { + try_(ret_val->copy_to(return_value)); + return ok(return_value); + } + + return ok(ret_val); +} diff --git a/modules/ncnn/src/runtime/runtime_function.h b/modules/ncnn/src/runtime/runtime_function.h new file mode 100644 index 0000000000..f5d4e97f47 --- /dev/null +++ b/modules/ncnn/src/runtime/runtime_function.h @@ -0,0 +1,45 @@ +/* Copyright 2019-2021 Canaan Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once +#include "runtime_module.h" +#include +#include +#include +#include + +BEGIN_NS_NNCASE_RT_MODULE(ncnn) + +class ncnn_runtime_function final : public runtime_function { + public: + ncnn_runtime_function(runtime_module &rt_module); + + ncnn_runtime_module &module() const noexcept; + + protected: + result + initialize_core(runtime_function_init_context &context) noexcept override; + result invoke_core(gsl::span parameters, + value_t return_value) noexcept override; + + private: + result run(gsl::span text) noexcept; + + private: + ::ncnn::Net net_; + std::vector input_names_; + std::vector output_names_; +}; + +END_NS_NNCASE_RT_MODULE diff --git a/modules/ncnn/src/runtime/runtime_module.cpp b/modules/ncnn/src/runtime/runtime_module.cpp new file mode 100644 index 0000000000..88d71305d5 --- /dev/null +++ b/modules/ncnn/src/runtime/runtime_module.cpp @@ -0,0 +1,65 @@ +/* Copyright 2019-2021 Canaan Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "runtime_module.h" +#include "runtime_function.h" +#include +#include +#include +#include + +using namespace nncase; +using namespace nncase::runtime; +using namespace nncase::runtime::ncnn; + +result ncnn_runtime_module::initialize_before_functions( + runtime_module_init_context &context) noexcept { + try_set(text_, context.get_or_read_section(".text", text_storage_, false)); + try_set(rdata_, + context.get_or_read_section(".rdata", rdata_storage_, false)); + return ok(); +} + +kernels::kernel_context &ncnn_runtime_module::kernel_context() noexcept { + auto &context = kernels::default_kernel_context(); +#ifdef NNCASE_DUMP_MANAGER + context.dump_manager = interp().dump_manager(); +#endif + return context; +} + +result> +ncnn_runtime_module::create_function() noexcept { + std::unique_ptr mod(new (std::nothrow) + ncnn_runtime_function(*this)); + if (mod) + return ok(std::move(mod)); + return err(std::errc::not_enough_memory); +} + +result> +nncase::runtime::ncnn::create_ncnn_runtime_module() { + std::unique_ptr mod(new (std::nothrow) + ncnn_runtime_module()); + if (mod) + return ok(std::move(mod)); + return err(std::errc::not_enough_memory); +} + +extern "C" { +NNCASE_MODULES_NCNN_API void +RUNTIME_MODULE_ACTIVATOR_NAME(result> &result) { + result = create_ncnn_runtime_module(); +} +} diff --git a/modules/ncnn/src/runtime/runtime_module.h b/modules/ncnn/src/runtime/runtime_module.h new file mode 100644 index 0000000000..e0f286d417 --- /dev/null +++ b/modules/ncnn/src/runtime/runtime_module.h @@ -0,0 +1,42 @@ +/* Copyright 2019-2021 Canaan Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once +#include +#include +#include + +BEGIN_NS_NNCASE_RT_MODULE(ncnn) + +class ncnn_runtime_module : public runtime_module { + public: + kernels::kernel_context &kernel_context() noexcept; + + gsl::span text() const noexcept { return text_; } + gsl::span rdata() const noexcept { return rdata_; } + + protected: + result initialize_before_functions( + runtime_module_init_context &context) noexcept override; + result> + create_function() noexcept override; + + private: + gsl::span text_; + gsl::span rdata_; + host_buffer_t text_storage_; + host_buffer_t rdata_storage_; +}; + +END_NS_NNCASE_RT_MODULE diff --git a/modules/ncnn/src/runtime/tensor_utils.h b/modules/ncnn/src/runtime/tensor_utils.h new file mode 100644 index 0000000000..00593bbfee --- /dev/null +++ b/modules/ncnn/src/runtime/tensor_utils.h @@ -0,0 +1,55 @@ +/* Copyright 2019-2021 Canaan Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once +#include +#include +#include +#include + +BEGIN_NS_NNCASE_RT_MODULE(ncnn) + +inline result<::ncnn::Mat> to_ncnn_mat(value_t value) { + try_var(t, value.as()); + ::ncnn::Mat mat; + auto shape = t->shape(); + auto elemsize = runtime::get_bytes(t->dtype()); + switch (shape.size()) { + case 1: + mat.create((int)shape[0], elemsize); + break; + case 2: + mat.create((int)shape[1], (int)shape[0], elemsize); + break; + case 3: + mat.create((int)shape[2], (int)shape[1], (int)shape[0], elemsize); + break; + case 4: + mat.create((int)shape[3], (int)shape[2], (int)shape[1], (int)shape[0], + elemsize); + break; + default: + return err(std::errc::invalid_argument); + } + + try_var(hb, t->buffer().as_host()); + try_var(map, hb.map(map_read)); + return kernels::stackvm::optimized::slice( + datatype, src_map.buffer().data() + src_start * datatype->size_bytes(), + dest_map.buffer().data() + dest_start * datatype->size_bytes(), shape, + src_strides, dest_strides, begins, ends, strides, + kernels::default_kernel_context()); +} + +END_NS_NNCASE_RT_MODULE diff --git a/src/Native/include/nncase/runtime/stream_reader.h b/src/Native/include/nncase/runtime/stream_reader.h index 59999ac150..b4fbbbccc5 100644 --- a/src/Native/include/nncase/runtime/stream_reader.h +++ b/src/Native/include/nncase/runtime/stream_reader.h @@ -66,6 +66,32 @@ class stream_reader { } } + std::string read_string() { + std::string str; + while (true) { + auto c = read(); + if (c) { + str.push_back(c); + } else { + break; + } + } + + return str; + } + + std::vector read_string_array() { + std::vector array; + while (true) { + if (peek() == '\0') { + skip(1); + break; + } + array.emplace_back(read_string()); + } + return array; + } + void skip(size_t count) { stream_.seekg(count, std::ios::cur); } private: diff --git a/src/Native/src/runtime/CMakeLists.txt b/src/Native/src/runtime/CMakeLists.txt index b892beded3..eb102dfe9b 100644 --- a/src/Native/src/runtime/CMakeLists.txt +++ b/src/Native/src/runtime/CMakeLists.txt @@ -55,7 +55,7 @@ else() target_include_directories(simulator PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) target_link_libraries(simulator PUBLIC gsl::gsl-lite) target_link_libraries(simulator PRIVATE kernels) - target_compile_definitions(simulator PUBLIC -DNNCASE_DLL -DNNCASE_SIMULATOR) + target_compile_definitions(simulator PUBLIC -DNNCASE_DLL -DNNCASE_SIMULATOR -DNNCASE_SHARED_LIBS) if (DEFAULT_BUILTIN_RUNTIMES) target_compile_definitions(simulator PRIVATE -DNNCASE_DEFAULT_BUILTIN_RUNTIMES) endif() @@ -69,6 +69,7 @@ else() target_link_libraries(nncaseruntime PUBLIC gsl::gsl-lite) set_target_properties(nncaseruntime PROPERTIES OUTPUT_NAME "Nncase.Runtime.Native") + target_compile_definitions(nncaseruntime PUBLIC -DNNCASE_SIMULATOR -DNNCASE_SHARED_LIBS) install(TARGETS nncaseruntime EXPORT nncaseTargets COMPONENT nncase-runtime diff --git a/src/Nncase.Cli/packages.lock.json b/src/Nncase.Cli/packages.lock.json index b438ba9cc6..007e490b3c 100644 --- a/src/Nncase.Cli/packages.lock.json +++ b/src/Nncase.Cli/packages.lock.json @@ -742,10 +742,18 @@ "nncase.io": { "type": "Project" }, + "nncase.modules.ncnn": { + "type": "Project", + "dependencies": { + "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Passes": "[1.0.0, )" + } + }, "nncase.modules.stackvm": { "type": "Project", "dependencies": { "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Modules.Ncnn": "[1.0.0, )", "Nncase.Passes": "[1.0.0, )" } }, diff --git a/src/Nncase.CodeGen/CodeGen/LinkedSection.cs b/src/Nncase.CodeGen/CodeGen/LinkedSection.cs index 2bbddefd90..84329d7ba2 100644 --- a/src/Nncase.CodeGen/CodeGen/LinkedSection.cs +++ b/src/Nncase.CodeGen/CodeGen/LinkedSection.cs @@ -43,6 +43,23 @@ public LinkedSection(Stream? content, string name, uint flags, uint alignment, u public ulong SizeInMemory { get; } + public static LinkedSection FromStrings(IReadOnlyCollection strings, string name) + { + var ms = new MemoryStream(); + using (var bw = new BinaryWriter(ms, Encoding.UTF8, true)) + { + foreach (string s in strings) + { + bw.Write(Encoding.UTF8.GetBytes(s)); + bw.Write((byte)0); + } + + bw.Write((byte)0); + } + + return new LinkedSection(ms, name, 0, 1, (ulong)ms.Length); + } + public void Serialize(Stream output) { if (_content != null) diff --git a/src/Nncase.Compiler/Hosting/CompilerHostBuilderExtensions.cs b/src/Nncase.Compiler/Hosting/CompilerHostBuilderExtensions.cs index 5fa0e4ba0f..629a1235cd 100644 --- a/src/Nncase.Compiler/Hosting/CompilerHostBuilderExtensions.cs +++ b/src/Nncase.Compiler/Hosting/CompilerHostBuilderExtensions.cs @@ -53,7 +53,8 @@ private static void ConfigureBuiltinModules(Container builder) .AddEGraph() .AddCodeGen() .AddPasses() - .AddStackVM(); + .AddStackVM() + .AddNcnn(); } private static void ConfigureServices(HostBuilderContext context, IServiceCollection services) diff --git a/src/Nncase.Compiler/Hosting/PluginLoader.cs b/src/Nncase.Compiler/Hosting/PluginLoader.cs index 73f10f367a..d475102773 100644 --- a/src/Nncase.Compiler/Hosting/PluginLoader.cs +++ b/src/Nncase.Compiler/Hosting/PluginLoader.cs @@ -25,6 +25,7 @@ public sealed class PluginLoader private static readonly string[] _builtinModules = new[] { "Nncase.Modules.StackVM.dll", + "Nncase.Modules.Ncnn.dll", "Nncase.Modules.K210.dll", }; diff --git a/src/Nncase.Compiler/packages.lock.json b/src/Nncase.Compiler/packages.lock.json index 639bb6a9bc..380cdda452 100644 --- a/src/Nncase.Compiler/packages.lock.json +++ b/src/Nncase.Compiler/packages.lock.json @@ -720,10 +720,18 @@ "nncase.io": { "type": "Project" }, + "nncase.modules.ncnn": { + "type": "Project", + "dependencies": { + "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Passes": "[1.0.0, )" + } + }, "nncase.modules.stackvm": { "type": "Project", "dependencies": { "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Modules.Ncnn": "[1.0.0, )", "Nncase.Passes": "[1.0.0, )" } }, diff --git a/src/Nncase.Core/IR/ExprCollector.cs b/src/Nncase.Core/IR/ExprCollector.cs index 5eff7385a6..e76084fa25 100644 --- a/src/Nncase.Core/IR/ExprCollector.cs +++ b/src/Nncase.Core/IR/ExprCollector.cs @@ -13,20 +13,27 @@ namespace Nncase.IR; public sealed class ExprCollector : ExprWalker> { - private ExprCollector() + private readonly Func? _condition; + + private ExprCollector(Func? condition = null) { + _condition = condition; } - public static IReadOnlyList Collect(Expr expr) + public static IReadOnlyList Collect(Expr expr, Func? condition = null) { var exprs = new List(); - new ExprCollector().Visit(expr, exprs); + new ExprCollector(condition).Visit(expr, exprs); return exprs; } protected override Unit DefaultVisitLeaf(Expr expr, List context) { - context.Add(expr); + if (_condition?.Invoke(expr) ?? true) + { + context.Add(expr); + } + return default; } } diff --git a/src/Nncase.Core/IR/Function.cs b/src/Nncase.Core/IR/Function.cs index 4758a87969..d86eb1012a 100644 --- a/src/Nncase.Core/IR/Function.cs +++ b/src/Nncase.Core/IR/Function.cs @@ -27,13 +27,13 @@ public sealed class Function : BaseFunction /// Initializes a new instance of the class. /// build function. /// - public Function(string name, Expr body, ReadOnlySpan parameters) - : this(name, body, parameters, new Dictionary()) + public Function(string name, Expr body, ReadOnlySpan parameters, string? moduleKind = null) + : this(name, body, parameters, new Dictionary(), moduleKind) { } - public Function(string name, Expr body, ReadOnlySpan parameters, Dictionary? varMap) - : base(name, StackVMModuleKind, ArrayUtility.Concat(body, SpanUtility.UnsafeCast(parameters))) + public Function(string name, Expr body, ReadOnlySpan parameters, Dictionary? varMap, string? moduleKind = null) + : base(name, moduleKind ?? StackVMModuleKind, ArrayUtility.Concat(body, SpanUtility.UnsafeCast(parameters))) { VarMap = varMap ?? new(); var dynamicDims = VarMap.Values.SelectMany(x => x).ToArray(); @@ -44,8 +44,8 @@ public Function(string name, Expr body, ReadOnlySpan parameters, Dictionary /// Initializes a new instance of the class. /// build function. /// - public Function(Expr body, ReadOnlySpan parameters) - : this($"func_{_globalFuncIndex++}", body, parameters, new Dictionary()) + public Function(Expr body, ReadOnlySpan parameters, string? moduleKind = null) + : this($"func_{_globalFuncIndex++}", body, parameters, new Dictionary(), moduleKind) { } diff --git a/src/Nncase.Core/Runtime/Ncnn/NcnnLayer.cs b/src/Nncase.Core/Runtime/Ncnn/NcnnLayer.cs new file mode 100644 index 0000000000..71735ee2a6 --- /dev/null +++ b/src/Nncase.Core/Runtime/Ncnn/NcnnLayer.cs @@ -0,0 +1,62 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; + +namespace Nncase.Runtime.Ncnn; + +public class NcnnTensor +{ + public string Name { get; set; } = string.Empty; + + public Shape ShapeHint { get; set; } = Shape.Unranked; + + public override string ToString() => $"{Name}: {ShapeHint}"; +} + +public class NcnnLayer +{ + public NcnnLayer(string type, string name, int bottomCount, int topCount) + { + Type = type; + Name = name; + Bottoms = new NcnnTensor[bottomCount]; + Tops = new NcnnTensor[topCount]; + } + + public string Type { get; } + + public string Name { get; } + + public NcnnTensor[] Bottoms { get; } + + public NcnnTensor[] Tops { get; } + + public ParamDict ParamDict { get; set; } = new(); + + public override string ToString() => $"[{Type}] {Name}"; + + public void Serialize(TextWriter writer) + { + writer.Write($"{Type}\t{Name}\t{Bottoms.Length} {Tops.Length} "); + + foreach (var bottom in Bottoms) + { + writer.Write($"{bottom.Name} "); + } + + foreach (var top in Tops) + { + writer.Write($"{top.Name} "); + } + + ParamDict.Serialize(writer); + writer.WriteLine(); + } +} diff --git a/src/Nncase.Importer/Ncnn/NcnnModel.cs b/src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs similarity index 71% rename from src/Nncase.Importer/Ncnn/NcnnModel.cs rename to src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs index 488e191ad4..393fa91188 100644 --- a/src/Nncase.Importer/Ncnn/NcnnModel.cs +++ b/src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs @@ -7,47 +7,20 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -using Nncase.IR; -namespace Nncase.Importer.Ncnn; +namespace Nncase.Runtime.Ncnn; -internal class NcnnTensor +public class NcnnModel { - public string Name { get; set; } = string.Empty; - - public Shape ShapeHint { get; set; } = Shape.Unranked; - - public override string ToString() => $"{Name}: {ShapeHint}"; -} + public static readonly int ExpectedMagic = 7767517; -internal class NcnnLayer -{ - public NcnnLayer(string type, string name, int bottomCount, int topCount) + public NcnnModel() { - Type = type; - Name = name; - Bottoms = new NcnnTensor[bottomCount]; - Tops = new NcnnTensor[topCount]; + Magic = ExpectedMagic; + Layers = new List(); } - public string Type { get; } - - public string Name { get; } - - public NcnnTensor[] Bottoms { get; } - - public NcnnTensor[] Tops { get; } - - public ParamDict ParamDict { get; } = new(); - - public override string ToString() => $"[{Type}] {Name}"; -} - -internal class NcnnModel -{ - public static readonly int ExpectedMagic = 7767517; - - public NcnnModel(int magic, IReadOnlyList layers) + public NcnnModel(int magic, IList layers) { Magic = magic; Layers = layers; @@ -55,7 +28,7 @@ public NcnnModel(int magic, IReadOnlyList layers) public int Magic { get; } - public IReadOnlyList Layers { get; } + public IList Layers { get; } public static NcnnModel ParseFromStream(Stream stream) { @@ -105,4 +78,19 @@ public static NcnnModel ParseFromStream(Stream stream) return new NcnnModel(magic, layers); } + + public void Serialize(TextWriter writer) + { + // 1. Magic + writer.WriteLine(Magic); + + // 2. layer_count & blob_count + writer.WriteLine($"{Layers.Count} {Layers.Select(x => x.Tops.Length).Sum()}"); + + // 3. layers + foreach (var layer in Layers) + { + layer.Serialize(writer); + } + } } diff --git a/src/Nncase.Importer/Ncnn/NcnnModelBin.cs b/src/Nncase.Core/Runtime/Ncnn/NcnnModelBin.cs similarity index 96% rename from src/Nncase.Importer/Ncnn/NcnnModelBin.cs rename to src/Nncase.Core/Runtime/Ncnn/NcnnModelBin.cs index a0d60bdc03..acacfd47c9 100644 --- a/src/Nncase.Importer/Ncnn/NcnnModelBin.cs +++ b/src/Nncase.Core/Runtime/Ncnn/NcnnModelBin.cs @@ -11,9 +11,9 @@ using System.Threading.Tasks; using Microsoft.Toolkit.HighPerformance; -namespace Nncase.Importer.Ncnn; +namespace Nncase.Runtime.Ncnn; -internal class NcnnModelBin +public class NcnnModelBin { private readonly Stream _stream; diff --git a/src/Nncase.Importer/Ncnn/ParamDict.cs b/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs similarity index 69% rename from src/Nncase.Importer/Ncnn/ParamDict.cs rename to src/Nncase.Core/Runtime/Ncnn/ParamDict.cs index 954525ea48..1fa97f65fc 100644 --- a/src/Nncase.Importer/Ncnn/ParamDict.cs +++ b/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs @@ -7,12 +7,11 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -using Google.Protobuf.WellKnownTypes; using Nncase.IR; -namespace Nncase.Importer.Ncnn; +namespace Nncase.Runtime.Ncnn; -internal enum ParamKind +public enum ParamKind { Null, IntOrFloat, @@ -23,7 +22,7 @@ internal enum ParamKind ArrayOfFloat, } -internal struct ParamValue +public struct ParamValue { public ParamKind Kind; @@ -34,12 +33,18 @@ internal struct ParamValue public Tensor? TensorValue; } -internal class ParamDict +public class ParamDict { public static readonly int NcnnMaxParamCount = 32; private readonly Dictionary _values = new(); + public ParamValue this[int index] + { + get => _values[index]; + set => _values[index] = value; + } + public void LoadFrom(ReadOnlySpan fields) { foreach (var field in fields) @@ -116,4 +121,52 @@ public void LoadFrom(ReadOnlySpan fields) public Tensor Get(int id, Tensor defaultValue) where T : unmanaged, IEquatable => _values.TryGetValue(id, out var value) ? value.TensorValue!.Cast() : defaultValue; + + public void Serialize(TextWriter writer) + { + int index = 0; + foreach (var field in _values) + { + var id = field.Key; + + var paramValue = field.Value; + var isArray = paramValue.Kind is ParamKind.ArrayOfFloat or ParamKind.ArrayOfIntOrFloat or ParamKind.ArrayOfInt; + var isFloat = paramValue.Kind is ParamKind.ArrayOfFloat or ParamKind.Float; + + if (isArray) + { + id = id - 23300; + } + + writer.Write($"{id}="); + + if (isArray) + { + if (isFloat) + { + writer.Write(string.Join(',', paramValue.TensorValue!.Cast().Select(x => x.ToString("e")))); + } + else + { + writer.Write(string.Join(',', paramValue.TensorValue!.Cast())); + } + } + else + { + if (isFloat) + { + writer.Write(paramValue.FloatValue.ToString("e")); + } + else + { + writer.Write(paramValue.IntValue); + } + } + + if (++index != _values.Count) + { + writer.Write(' '); + } + } + } } diff --git a/src/Nncase.Importer/Ncnn/Concat.cs b/src/Nncase.Importer/Ncnn/Concat.cs index 2363144d4c..c565dbfc31 100644 --- a/src/Nncase.Importer/Ncnn/Concat.cs +++ b/src/Nncase.Importer/Ncnn/Concat.cs @@ -11,6 +11,7 @@ using Nncase.IR.Buffers; using Nncase.IR.F; using Nncase.IR.NN; +using Nncase.Runtime.Ncnn; namespace Nncase.Importer.Ncnn; diff --git a/src/Nncase.Importer/Ncnn/Convolution.cs b/src/Nncase.Importer/Ncnn/Convolution.cs index 5885feef46..be28b4fd63 100644 --- a/src/Nncase.Importer/Ncnn/Convolution.cs +++ b/src/Nncase.Importer/Ncnn/Convolution.cs @@ -12,6 +12,7 @@ using Nncase.IR.Buffers; using Nncase.IR.F; using Nncase.IR.NN; +using Nncase.Runtime.Ncnn; namespace Nncase.Importer.Ncnn; diff --git a/src/Nncase.Importer/Ncnn/InnerProduct.cs b/src/Nncase.Importer/Ncnn/InnerProduct.cs index 80aeb98bb2..e051e82b85 100644 --- a/src/Nncase.Importer/Ncnn/InnerProduct.cs +++ b/src/Nncase.Importer/Ncnn/InnerProduct.cs @@ -12,6 +12,7 @@ using Nncase.IR.Buffers; using Nncase.IR.F; using Nncase.IR.NN; +using Nncase.Runtime.Ncnn; namespace Nncase.Importer.Ncnn; diff --git a/src/Nncase.Importer/Ncnn/NcnnImporter.cs b/src/Nncase.Importer/Ncnn/NcnnImporter.cs index a67542aeab..7e66018743 100644 --- a/src/Nncase.Importer/Ncnn/NcnnImporter.cs +++ b/src/Nncase.Importer/Ncnn/NcnnImporter.cs @@ -17,6 +17,7 @@ using Nncase.IR.Math; using Nncase.IR.NN; using Nncase.IR.Tensors; +using Nncase.Runtime.Ncnn; using Math = System.Math; using Tuple = Nncase.IR.Tuple; diff --git a/src/Nncase.Importer/Ncnn/Pooling.cs b/src/Nncase.Importer/Ncnn/Pooling.cs index edc486d71d..dc48d9d04b 100644 --- a/src/Nncase.Importer/Ncnn/Pooling.cs +++ b/src/Nncase.Importer/Ncnn/Pooling.cs @@ -11,6 +11,7 @@ using Nncase.IR.Buffers; using Nncase.IR.F; using Nncase.IR.NN; +using Nncase.Runtime.Ncnn; namespace Nncase.Importer.Ncnn; diff --git a/src/Nncase.Importer/Ncnn/ShuffleChannel.cs b/src/Nncase.Importer/Ncnn/ShuffleChannel.cs index dd4f760951..0d88ff7d29 100644 --- a/src/Nncase.Importer/Ncnn/ShuffleChannel.cs +++ b/src/Nncase.Importer/Ncnn/ShuffleChannel.cs @@ -11,6 +11,7 @@ using Nncase.IR.Buffers; using Nncase.IR.F; using Nncase.IR.NN; +using Nncase.Runtime.Ncnn; namespace Nncase.Importer.Ncnn; diff --git a/src/Nncase.Importer/Ncnn/Softmax.cs b/src/Nncase.Importer/Ncnn/Softmax.cs index 0e1744992b..8c6c3620f7 100644 --- a/src/Nncase.Importer/Ncnn/Softmax.cs +++ b/src/Nncase.Importer/Ncnn/Softmax.cs @@ -11,6 +11,7 @@ using Nncase.IR.Buffers; using Nncase.IR.F; using Nncase.IR.NN; +using Nncase.Runtime.Ncnn; namespace Nncase.Importer.Ncnn; diff --git a/src/Nncase.Importer/Ncnn/Split.cs b/src/Nncase.Importer/Ncnn/Split.cs index a6327ee653..6ab464c106 100644 --- a/src/Nncase.Importer/Ncnn/Split.cs +++ b/src/Nncase.Importer/Ncnn/Split.cs @@ -11,6 +11,7 @@ using Nncase.IR.Buffers; using Nncase.IR.F; using Nncase.IR.NN; +using Nncase.Runtime.Ncnn; namespace Nncase.Importer.Ncnn; diff --git a/src/Nncase.Passes/Transforms/FusionToFunctionPass.cs b/src/Nncase.Passes/Transforms/FusionToFunctionPass.cs new file mode 100644 index 0000000000..6464f21efd --- /dev/null +++ b/src/Nncase.Passes/Transforms/FusionToFunctionPass.cs @@ -0,0 +1,45 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reactive; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; +using Nncase.IR; + +namespace Nncase.Passes.Transforms; + +public sealed class FusionToFunctionPass : ModulePass +{ + protected override Task RunCoreAsync(IRModule input, RunPassContext context) + { + var count = input.Functions.Count; + for (int i = 0; i < count; i++) + { + var replacer = new FusionReplacer(input); + input.Replace(i, (BaseFunction)replacer.Rewrite(input.Functions[i])); + } + + return Task.FromResult(input); + } + + private sealed class FusionReplacer : ExprRewriter + { + private readonly IRModule _module; + + public FusionReplacer(IRModule module) + { + _module = module; + } + + protected override Expr RewriteLeafFusion(Fusion expr) + { + var func = new Function(expr.Body, expr.Parameters, expr.ModuleKind); + _module.Add(func); + return func; + } + } +} diff --git a/src/Nncase.Tests.TestFixture/packages.lock.json b/src/Nncase.Tests.TestFixture/packages.lock.json index a7d3cc2c05..9310b67143 100644 --- a/src/Nncase.Tests.TestFixture/packages.lock.json +++ b/src/Nncase.Tests.TestFixture/packages.lock.json @@ -796,10 +796,18 @@ "nncase.io": { "type": "Project" }, + "nncase.modules.ncnn": { + "type": "Project", + "dependencies": { + "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Passes": "[1.0.0, )" + } + }, "nncase.modules.stackvm": { "type": "Project", "dependencies": { "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Modules.Ncnn": "[1.0.0, )", "Nncase.Passes": "[1.0.0, )" } }, diff --git a/src/Nncase.Tests/packages.lock.json b/src/Nncase.Tests/packages.lock.json index c722181308..a430980e57 100644 --- a/src/Nncase.Tests/packages.lock.json +++ b/src/Nncase.Tests/packages.lock.json @@ -917,10 +917,18 @@ "nncase.io": { "type": "Project" }, + "nncase.modules.ncnn": { + "type": "Project", + "dependencies": { + "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Passes": "[1.0.0, )" + } + }, "nncase.modules.stackvm": { "type": "Project", "dependencies": { "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Modules.Ncnn": "[1.0.0, )", "Nncase.Passes": "[1.0.0, )" } }, From 5a256cb56d79c85c98c5eb487fd667f1160ca00e Mon Sep 17 00:00:00 2001 From: sunnycase Date: Tue, 17 Oct 2023 10:05:16 +0000 Subject: [PATCH 003/132] Apply code-format changes --- modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs | 2 +- modules/ncnn/src/runtime/tensor_utils.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 91f708479f..9b83769e9e 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -34,7 +34,7 @@ public void Softmax(string name, string input, int axis) => AddLayer("Softmax", name, new[] { input }, new[] { name }, new ParamDict { [0] = new ParamValue { Kind = ParamKind.Int, IntValue = axis }, // axis - [1] = new ParamValue { Kind= ParamKind.Int, IntValue = 1 }, // fixbug0 + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = 1 }, // fixbug0 }); private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null) diff --git a/modules/ncnn/src/runtime/tensor_utils.h b/modules/ncnn/src/runtime/tensor_utils.h index 00593bbfee..400787097a 100644 --- a/modules/ncnn/src/runtime/tensor_utils.h +++ b/modules/ncnn/src/runtime/tensor_utils.h @@ -15,8 +15,8 @@ #pragma once #include #include -#include #include +#include BEGIN_NS_NNCASE_RT_MODULE(ncnn) From 00987cc47709282c10d8cf6200fe084b5ab73dd5 Mon Sep 17 00:00:00 2001 From: sunnycase Date: Thu, 19 Oct 2023 18:42:12 +0800 Subject: [PATCH 004/132] Fix build --- modules/ncnn/src/runtime/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/ncnn/src/runtime/CMakeLists.txt b/modules/ncnn/src/runtime/CMakeLists.txt index c256595b39..70dfdb9552 100644 --- a/modules/ncnn/src/runtime/CMakeLists.txt +++ b/modules/ncnn/src/runtime/CMakeLists.txt @@ -5,12 +5,18 @@ set(SRCS runtime_module.cpp if (BUILDING_RUNTIME) add_library(runtime_ncnn OBJECT ${SRCS}) + if (NOT MSVC) + target_compile_options(runtime_ncnn PRIVATE -Wno-unused-parameter) + endif() target_link_libraries(runtime_ncnn PUBLIC nncaseruntime ncnn::ncnn) set_target_properties(runtime_ncnn PROPERTIES POSITION_INDEPENDENT_CODE ON) install(TARGETS runtime_ncnn EXPORT nncase_rt_modules_ncnnTargets) else() add_library(simulator_ncnn OBJECT ${SRCS}) + if (NOT MSVC) + target_compile_options(simulator_ncnn PRIVATE -Wno-unused-parameter) + endif() target_link_libraries(simulator_ncnn PRIVATE nncaseruntime ncnn::ncnn) target_compile_definitions(simulator_ncnn PUBLIC -DNNCASE_MODULES_NCNN_DLL -DNNCASE_SIMULATOR) set_target_properties(simulator_ncnn PROPERTIES POSITION_INDEPENDENT_CODE ON) From 94776cab9c99d8c389b693cac051a9c32b717c9b Mon Sep 17 00:00:00 2001 From: sunnycase Date: Thu, 19 Oct 2023 18:53:34 +0800 Subject: [PATCH 005/132] Fix build --- modules/ncnn/src/runtime/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ncnn/src/runtime/CMakeLists.txt b/modules/ncnn/src/runtime/CMakeLists.txt index 70dfdb9552..f5354f48c3 100644 --- a/modules/ncnn/src/runtime/CMakeLists.txt +++ b/modules/ncnn/src/runtime/CMakeLists.txt @@ -9,7 +9,7 @@ if (BUILDING_RUNTIME) target_compile_options(runtime_ncnn PRIVATE -Wno-unused-parameter) endif() - target_link_libraries(runtime_ncnn PUBLIC nncaseruntime ncnn::ncnn) + target_link_libraries(runtime_ncnn PUBLIC nncaseruntime ncnn::ncnn OpenMP::OpenMP_CXX) set_target_properties(runtime_ncnn PROPERTIES POSITION_INDEPENDENT_CODE ON) install(TARGETS runtime_ncnn EXPORT nncase_rt_modules_ncnnTargets) else() @@ -17,7 +17,7 @@ else() if (NOT MSVC) target_compile_options(simulator_ncnn PRIVATE -Wno-unused-parameter) endif() - target_link_libraries(simulator_ncnn PRIVATE nncaseruntime ncnn::ncnn) + target_link_libraries(simulator_ncnn PRIVATE nncaseruntime ncnn::ncnn OpenMP::OpenMP_CXX) target_compile_definitions(simulator_ncnn PUBLIC -DNNCASE_MODULES_NCNN_DLL -DNNCASE_SIMULATOR) set_target_properties(simulator_ncnn PROPERTIES POSITION_INDEPENDENT_CODE ON) endif() From 85a4aa6da7a559f2adeddda37d119e313788e522 Mon Sep 17 00:00:00 2001 From: sunnycase Date: Fri, 20 Oct 2023 18:04:47 +0800 Subject: [PATCH 006/132] Fix build --- .github/workflows/runtime-build.yml | 2 +- modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs | 1 - modules/ncnn/src/runtime/runtime_function.cpp | 2 +- toolchains/riscv64-unknown-linux.profile.jinja | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/runtime-build.yml b/.github/workflows/runtime-build.yml index 0665ff10fb..e105f0fa71 100644 --- a/.github/workflows/runtime-build.yml +++ b/.github/workflows/runtime-build.yml @@ -92,7 +92,7 @@ jobs: strategy: matrix: config: - - {name: riscv64-linux, shell: bash, arch: riscv64, toolchain: riscv64-unknown-linux, toolchain_env: RISCV_ROOT_PATH, toolchain_file: riscv64-unknown-linux-gnu-12.0.1, qemu: qemu-riscv64, loader_args: '-cpu;rv64,v=true,Zfh=true,vlen=128,elen=64,vext_spec=v1.0;-L', cmakeArgs: '', buildType: Release} + - {name: riscv64-linux, shell: bash, arch: riscv64, toolchain: riscv64-unknown-linux, toolchain_env: RISCV_ROOT_PATH, toolchain_file: riscv64-unknown-linux-gnu-13.2.0, qemu: qemu-riscv64, loader_args: '-cpu;rv64,v=true,Zfh=true,vlen=128,elen=64,vext_spec=v1.0;-L', cmakeArgs: '', buildType: Release} steps: - uses: actions/checkout@v3 diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index ede3397fa5..559c651c06 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -46,7 +46,6 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions passManager.AddWithName("LowerNcnnIR").Configure(p => { p.Add(); - p.Add(); }); } diff --git a/modules/ncnn/src/runtime/runtime_function.cpp b/modules/ncnn/src/runtime/runtime_function.cpp index dee711793f..aead2f5aed 100644 --- a/modules/ncnn/src/runtime/runtime_function.cpp +++ b/modules/ncnn/src/runtime/runtime_function.cpp @@ -179,7 +179,7 @@ result ncnn_runtime_function::invoke_core( { try_var(map, hb->map(map_write)); auto csize = (size_t)mat.w * mat.h * mat.d * mat.elemsize; - for (size_t i = 0; i < mat.c; i++) { + for (size_t i = 0; i < (size_t)mat.c; i++) { void *dest = (unsigned char *)map.buffer().data() + (size_t)i * csize; const void *src = diff --git a/toolchains/riscv64-unknown-linux.profile.jinja b/toolchains/riscv64-unknown-linux.profile.jinja index 6ef63e9cd4..2b924cb14d 100644 --- a/toolchains/riscv64-unknown-linux.profile.jinja +++ b/toolchains/riscv64-unknown-linux.profile.jinja @@ -22,4 +22,4 @@ compiler=gcc build_type=Release compiler.cppstd=17 compiler.libcxx=libstdc++11 -compiler.version=12 +compiler.version=13 From 748ade19485769d3ecb0989752a29bf8b7897c0c Mon Sep 17 00:00:00 2001 From: yanghaoqi Date: Mon, 27 Nov 2023 17:04:25 +0800 Subject: [PATCH 007/132] support more than 3D ncnn:softmax --- .../Passes/Rules/Ncnn/LowerSoftmax.cs | 38 ++++++++++++++++++- tests/importer/onnx_/basic/test_softmax.py | 5 ++- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftmax.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftmax.cs index 4954908a3c..9dcdd24f74 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftmax.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftmax.cs @@ -12,6 +12,7 @@ using Nncase.PatternMatch; using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; using static Nncase.IR.TypePatternUtility; using static Nncase.PatternMatch.F.NN; using static Nncase.PatternMatch.Utility; @@ -23,12 +24,45 @@ public partial class LowerSoftmax : RewriteRule { /// public override Pattern Pattern { get; } = IsSoftmax( - IsWildcard("input") with { TypePattern = IsFloat() & HasRank(x => x <= 3) }, + IsWildcard("input") with { TypePattern = IsFloat() }, IsTensorConst("axis")); + // squeeze softmax to 3D，set axis to 1 + private (List NewShape, int NewAxis) GetFixedShapeAndAxis(List oldShape, int oldAxis) + { + int positive_axis = oldAxis < 0 ? oldShape.Count + oldAxis : oldAxis; + var newShape = new List { 1, oldShape[positive_axis], 1 }; + for (int i = 0; i < positive_axis; i++) + { + newShape[0] *= oldShape[i]; + } + + for (int i = positive_axis + 1; i < oldShape.Count; i++) + { + newShape[2] *= oldShape[i]; + } + + return (newShape, 1); + } + private Expr? GetReplace(Expr input, int axis) { var newInput = new Var(input.CheckedType); - return new Call(new Fusion("ncnn", NcnnSoftmax(newInput, axis), new[] { newInput }), input); + if (input.CheckedShape.Rank > 3) + { + var (newShape, newAxis) = GetFixedShapeAndAxis(input.CheckedShape.ToValueList(), axis); + + var inRes = Reshape(input, newShape.ToArray()); + var inResO = new Var(inRes.CheckedType); + + var ncnnSoftmaxCall = new Call(new Fusion("ncnn", NcnnSoftmax(inResO, newAxis), new[] { inResO }), inRes); + + var outRes = Reshape(ncnnSoftmaxCall, input.CheckedShape.ToValueList().ToArray()); + return outRes; + } + else + { + return new Call(new Fusion("ncnn", NcnnSoftmax(newInput, axis), new[] { newInput }), input); + } } } diff --git a/tests/importer/onnx_/basic/test_softmax.py b/tests/importer/onnx_/basic/test_softmax.py index 31f98158b2..aed9275084 100644 --- a/tests/importer/onnx_/basic/test_softmax.py +++ b/tests/importer/onnx_/basic/test_softmax.py @@ -63,8 +63,9 @@ def _make_module(in_shape, axis, op_version): in_shapes = [ - [2, 3, 8, 1], - [1, 3, 8, 5], + [2, 32, 3, 3], + [2, 32, 64, 128], + [1, 113, 228, 65], ] axes = [ From 7fa6bac0072909db3485a8f973e60370a49a93c6 Mon Sep 17 00:00:00 2001 From: yanghaoqi Date: Tue, 28 Nov 2023 10:28:52 +0800 Subject: [PATCH 008/132] fix shape --- tests/importer/onnx_/basic/test_softmax.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/importer/onnx_/basic/test_softmax.py b/tests/importer/onnx_/basic/test_softmax.py index aed9275084..3f16c20bb4 100644 --- a/tests/importer/onnx_/basic/test_softmax.py +++ b/tests/importer/onnx_/basic/test_softmax.py @@ -63,9 +63,8 @@ def _make_module(in_shape, axis, op_version): in_shapes = [ - [2, 32, 3, 3], - [2, 32, 64, 128], - [1, 113, 228, 65], + [2, 32, 16, 4], + [5, 13, 28, 64], ] axes = [ From 724b20e4c24300c8f3cc3a2d465caa6fa786bf04 Mon Sep 17 00:00:00 2001 From: yanghaoqi Date: Tue, 28 Nov 2023 17:06:19 +0800 Subject: [PATCH 009/132] fix merge error --- nncase.sln | 6 ------ 1 file changed, 6 deletions(-) diff --git a/nncase.sln b/nncase.sln index 7cfc1f42c2..2421349de4 100644 --- a/nncase.sln +++ b/nncase.sln @@ -175,17 +175,14 @@ Global {E6462E82-B48F-4AFA-AE34-725EF0A9CB42}.Debug|Any CPU.Build.0 = Debug|Any CPU {E6462E82-B48F-4AFA-AE34-725EF0A9CB42}.Release|Any CPU.ActiveCfg = Release|Any CPU {E6462E82-B48F-4AFA-AE34-725EF0A9CB42}.Release|Any CPU.Build.0 = Release|Any CPU -<<<<<<< HEAD {FA535CB0-C8B9-4CA5-B3E3-8B7912E5F061}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {FA535CB0-C8B9-4CA5-B3E3-8B7912E5F061}.Debug|Any CPU.Build.0 = Debug|Any CPU {FA535CB0-C8B9-4CA5-B3E3-8B7912E5F061}.Release|Any CPU.ActiveCfg = Release|Any CPU {FA535CB0-C8B9-4CA5-B3E3-8B7912E5F061}.Release|Any CPU.Build.0 = Release|Any CPU -======= {B9A09DA2-EF1A-4C0E-A0F5-427AFBB5C769}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B9A09DA2-EF1A-4C0E-A0F5-427AFBB5C769}.Debug|Any CPU.Build.0 = Debug|Any CPU {B9A09DA2-EF1A-4C0E-A0F5-427AFBB5C769}.Release|Any CPU.ActiveCfg = Release|Any CPU {B9A09DA2-EF1A-4C0E-A0F5-427AFBB5C769}.Release|Any CPU.Build.0 = Release|Any CPU ->>>>>>> master EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -216,11 +213,8 @@ Global {E365B1B1-4D13-4839-9763-A7A7C5F32FD4} = {BCA74168-F015-4B5B-B4CD-C83AE06B9822} {98A03405-CA53-4EC4-9B18-94D1C8DF9453} = {E5A4516C-4080-4346-991D-57A7AA76ADA6} {E6462E82-B48F-4AFA-AE34-725EF0A9CB42} = {BCA74168-F015-4B5B-B4CD-C83AE06B9822} -<<<<<<< HEAD {FA535CB0-C8B9-4CA5-B3E3-8B7912E5F061} = {9859F5E8-5504-4AFE-B955-9497A0A0CD66} -======= {B9A09DA2-EF1A-4C0E-A0F5-427AFBB5C769} = {BCA74168-F015-4B5B-B4CD-C83AE06B9822} ->>>>>>> master EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9492E141-292E-4D60-9C6E-3738AB234DB2} From c4290451ed2c9682eeead1a6109d868c0d121eda Mon Sep 17 00:00:00 2001 From: curioyang Date: Tue, 28 Nov 2023 09:11:21 +0000 Subject: [PATCH 010/132] Apply code-format changes --- .../Nncase.Modules.Ncnn/packages.lock.json | 19 +++++++++++++------ src/Nncase.Studio/packages.lock.json | 8 ++++++++ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/packages.lock.json b/modules/Nncase.Modules.Ncnn/packages.lock.json index b69bdc40f8..4830d0bbf6 100644 --- a/modules/Nncase.Modules.Ncnn/packages.lock.json +++ b/modules/Nncase.Modules.Ncnn/packages.lock.json @@ -4,11 +4,11 @@ "net7.0": { "StyleCop.Analyzers": { "type": "Direct", - "requested": "[1.2.0-beta.507, )", - "resolved": "1.2.0-beta.507", - "contentHash": "/FtugDT66cKJJ+GGH7rNpG6UDrT4iIWz45M6lrXXHobDUFDHw+q5VgkbiR+6ffTO564ge7w6fQh/eoQhVdJO8Q==", + "requested": "[1.2.0-beta.435, )", + "resolved": "1.2.0-beta.435", + "contentHash": "TADk7vdGXtfTnYCV7GyleaaRTQjfoSfZXprQrVMm7cSJtJbFc1QIbWPyLvrgrfGdfHbGmUPvaN4ODKNxg2jgPQ==", "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.507" + "StyleCop.Analyzers.Unstable": "1.2.0.435" } }, "Google.OrTools.runtime.linux-arm64": { @@ -103,8 +103,8 @@ }, "StyleCop.Analyzers.Unstable": { "type": "Transitive", - "resolved": "1.2.0.507", - "contentHash": "gTY3IQdRqDJ4hbhSA3e/R48oE8b/OiKfvwkt1QdNVfrJK2gMHBV8ldaHJ885jxWZfllK66soa/sdcjh9bX49Tw==" + "resolved": "1.2.0.435", + "contentHash": "ouwPWZxbOV3SmCZxIRqHvljkSzkCyi1tDoMzQtDb/bRP8ctASV/iRJr+A2Gdj0QLaLmWnqTWDrH82/iP+X80Lg==" }, "System.Buffers": { "type": "Transitive", @@ -134,6 +134,7 @@ "Microsoft.Extensions.Options": "[6.0.0, )", "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", "NetFabric.Hyperlinq": "[3.0.0-beta48, )", + "System.CommandLine": "[2.0.0-beta4.22272.1, )", "System.Reactive": "[5.0.0, )" } }, @@ -271,6 +272,12 @@ "resolved": "1.0.2", "contentHash": "giLAHrjJe0Bh7yhNexR6pmcv02+Fi+lEPxQVdB8zvkuJCmy6rnqu8CZLIpxrUfLcWDuTCSiK0IfGmMhig3UDhA==" }, + "System.CommandLine": { + "type": "CentralTransitive", + "requested": "[2.0.0-beta4.22272.1, )", + "resolved": "2.0.0-beta4.22272.1", + "contentHash": "1uqED/q2H0kKoLJ4+hI2iPSBSEdTuhfCYADeJrAqERmiGQ2NNacYKRNEQ+gFbU4glgVyK8rxI+ZOe1onEtr/Pg==" + }, "System.Reactive": { "type": "CentralTransitive", "requested": "[5.0.0, )", diff --git a/src/Nncase.Studio/packages.lock.json b/src/Nncase.Studio/packages.lock.json index ee5a22e374..8215bc8fa9 100644 --- a/src/Nncase.Studio/packages.lock.json +++ b/src/Nncase.Studio/packages.lock.json @@ -995,10 +995,18 @@ "nncase.io": { "type": "Project" }, + "nncase.modules.ncnn": { + "type": "Project", + "dependencies": { + "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Passes": "[1.0.0, )" + } + }, "nncase.modules.stackvm": { "type": "Project", "dependencies": { "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Modules.Ncnn": "[1.0.0, )", "Nncase.Passes": "[1.0.0, )" } }, From 2e56c95cddac3343e360f8c55deb9a5d88e3e3d4 Mon Sep 17 00:00:00 2001 From: yanghaoqi Date: Wed, 29 Nov 2023 18:43:14 +0800 Subject: [PATCH 011/132] support ncnn unary --- .../CodeGen/Ncnn/NcnnEmitter.cs | 7 ++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Evaluator/Ncnn/NcnnUnary.cs | 117 ++++++++++++++++++ .../Passes/Rules/Ncnn/LowerUnary.cs | 55 +++++++- .../Targets/CPUTarget.cs | 1 + 6 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnUnary.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 9b83769e9e..3fd3a03cf1 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Nncase.IR.Ncnn; using Nncase.Runtime.Ncnn; namespace Nncase.CodeGen.Ncnn; @@ -37,6 +38,12 @@ public void Softmax(string name, string input, int axis) => [1] = new ParamValue { Kind = ParamKind.Int, IntValue = 1 }, // fixbug0 }); + public void Unary(string name, string input, UnaryOperationType opTypes) => + AddLayer("UnaryOp", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = (int)opTypes }, + }); + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index e0f7ca488f..60c968998d 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -85,6 +85,9 @@ protected override string VisitLeafCall(Call expr) case NcnnSoftmax op: _emitter.Softmax(name, ExprMemo[expr.Arguments[0]], op.Axis); break; + case NcnnUnary op: + _emitter.Unary(name, ExprMemo[expr.Arguments[0]], op.OpType); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index b1504cfbb8..281e9e9e62 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -14,5 +14,6 @@ internal class NcnnModule : IApplicationPart public void ConfigureServices(IRegistrator registrator) { registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnUnary.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnUnary.cs new file mode 100644 index 0000000000..452b3598ee --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnUnary.cs @@ -0,0 +1,117 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnUnaryEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnUnary unary) + { + var input = context.GetOrtArgumentValue(unary, NcnnUnary.Input); + var opType = unary.OpType; + + // return OrtKI. (input, dim).ToValue(); + switch (opType) + { + case UnaryOperationType.ABS: + return OrtKI.Abs(input).ToValue(); + case UnaryOperationType.NEG: + return OrtKI.Neg(input).ToValue(); + case UnaryOperationType.FLOOR: + return OrtKI.Floor(input).ToValue(); + case UnaryOperationType.CEIL: + return OrtKI.Ceil(input).ToValue(); + case UnaryOperationType.SQUARE: + return OrtKI.Square(input).ToValue(); + case UnaryOperationType.SQRT: + return OrtKI.Sqrt(input).ToValue(); + case UnaryOperationType.RSQRT: + return OrtKI.Rsqrt(input).ToValue(); + case UnaryOperationType.EXP: + return OrtKI.Exp(input).ToValue(); + case UnaryOperationType.LOG: + return OrtKI.Log(input).ToValue(); + case UnaryOperationType.SIN: + return OrtKI.Sin(input).ToValue(); + case UnaryOperationType.COS: + return OrtKI.Cos(input).ToValue(); + case UnaryOperationType.TAN: + return OrtKI.Tan(input).ToValue(); + case UnaryOperationType.ASIN: + return OrtKI.Asin(input).ToValue(); + case UnaryOperationType.ACOS: + return OrtKI.Acos(input).ToValue(); + case UnaryOperationType.ATAN: + return OrtKI.Atan(input).ToValue(); + case UnaryOperationType.RECIPROCAL: + return OrtKI.Reciprocal(input).ToValue(); + case UnaryOperationType.TANH: + return OrtKI.Tanh(input).ToValue(); + case UnaryOperationType.LOG10: + double ln10 = 2.3025850929940456840179914546844; + return OrtKI.Div(OrtKI.Log(input), OrtKISharp.Tensor.FromScalar(ln10)).ToValue(); + case UnaryOperationType.ROUND: + return OrtKI.Round(input).ToValue(); + + //TODO: trunc + default: + throw new NotSupportedException("Ncnn unary ops"); + } + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnUnary target) + { + var input = context.CheckArgumentType(target, NcnnUnary.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnUnary target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnUnary target) + { + var inputType = context.GetArgumentType(target, NcnnUnary.Input); + var returnType = context.GetReturnType(); + var returnF = MetricUtility.GetFLOPs(returnType); + var inputF = MetricUtility.GetFLOPs(inputType); + var inner = inputF / returnF; + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = (inner * 2) + (inputF * (MetricUtility.SubFLOPs + MetricUtility.ExpFLOPs + MetricUtility.DivFLOPs)), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnUnary target) => context.GetArgumentShape(target, NcnnUnary.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnary.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnary.cs index 7b1ec3f4f8..ef4f3fbcb9 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnary.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnary.cs @@ -12,6 +12,7 @@ using Nncase.PatternMatch; using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; using static Nncase.IR.TypePatternUtility; using static Nncase.PatternMatch.F.Math; using static Nncase.PatternMatch.Utility; @@ -25,22 +26,72 @@ public partial class LowerUnary : RewriteRule public override Pattern Pattern { get; } = IsUnary( target_name: "unary", _ => true, - IsWildcard("input") with { TypePattern = IsFloat() & HasRank(x => x <= 3) }); + IsWildcard("input") with { TypePattern = IsFloat() }); private static UnaryOperationType? MapUnaryOp(UnaryOp unaryOp) => unaryOp switch { UnaryOp.Abs => UnaryOperationType.ABS, + UnaryOp.Neg => UnaryOperationType.NEG, + UnaryOp.Floor => UnaryOperationType.FLOOR, + UnaryOp.Ceil => UnaryOperationType.CEIL, + UnaryOp.Square => UnaryOperationType.SQUARE, + UnaryOp.Sqrt => UnaryOperationType.SQRT, + UnaryOp.Rsqrt => UnaryOperationType.RSQRT, + UnaryOp.Exp => UnaryOperationType.EXP, + UnaryOp.Log => UnaryOperationType.LOG, + UnaryOp.Sin => UnaryOperationType.SIN, + UnaryOp.Cos => UnaryOperationType.COS, + UnaryOp.Asin => UnaryOperationType.ASIN, UnaryOp.Acos => UnaryOperationType.ACOS, + UnaryOp.Tanh => UnaryOperationType.TANH, + UnaryOp.Round => UnaryOperationType.ROUND, _ => null, + + // unsupported unary ops + // UnaryOp.TAN => UnaryOperationType.ABS, + // UnaryOp.Atan => UnaryOperationType.ATAN, + // UnaryOp.Reciprocal => UnaryOperationType.RECIPROCAL, + // UnaryOp.Log10 => UnaryOperationType.LOG10, + // UnaryOp.Trunc => UnaryOperationType.TRUNC, }; + // squeeze unary shape to 3D + private List GetFixedShape(List oldShape) + { + var newShape = new List(); + + newShape.AddRange(oldShape.GetRange(oldShape.Count - 3, 3)); + + for (int i = 0; i < oldShape.Count - 3; i++) + { + newShape[0] *= oldShape[i]; + } + + return newShape; + } + private Expr? GetReplace(Unary unary, Expr input) { if (MapUnaryOp(unary.UnaryOp) is UnaryOperationType op) { var newInput = new Var(input.CheckedType); - return new Call(new Fusion("ncnn", NcnnUnary(newInput, op), new[] { newInput }), input); + if (input.CheckedShape.Rank > 3) + { + var newShape = GetFixedShape(input.CheckedShape.ToValueList()); + + var inRes = Reshape(input, newShape.ToArray()); + var inResO = new Var(inRes.CheckedType); + + var ncnnUnaryCall = new Call(new Fusion("ncnn", NcnnUnary(inResO, op), new[] { inResO }), inRes); + + var outRes = Reshape(ncnnUnaryCall, input.CheckedShape.ToValueList().ToArray()); + return outRes; + } + else + { + return new Call(new Fusion("ncnn", NcnnUnary(newInput, op), new[] { newInput }), input); + } } return null; diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index f4f0659fef..a879737a82 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -49,6 +49,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions passManager.AddWithName("LowerNcnnIR").Configure(p => { p.Add(); + p.Add(); }); } From 5baa3a4da9646fd2ad7eb6f08df0d97b756a5931 Mon Sep 17 00:00:00 2001 From: curioyang Date: Wed, 29 Nov 2023 10:47:00 +0000 Subject: [PATCH 012/132] Apply code-format changes --- modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnUnary.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnUnary.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnUnary.cs index 452b3598ee..dde4c66e7b 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnUnary.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnUnary.cs @@ -68,7 +68,7 @@ public IValue Visit(IEvaluateContext context, NcnnUnary unary) case UnaryOperationType.ROUND: return OrtKI.Round(input).ToValue(); - //TODO: trunc + // TODO: trunc default: throw new NotSupportedException("Ncnn unary ops"); } From 615d103a687d3e1b847940545829d605652f1e59 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 8 Dec 2023 18:42:24 +0800 Subject: [PATCH 013/132] add batchnorm --- .../CodeGen/Ncnn/NcnnEmitter.cs | 23 ++++++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnBatchNorm.cs | 73 +++++++++++++++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../IR/Ncnn/NcnnBatchnorm.cs | 71 ++++++++++++++++++ .../IR/Ncnn/NcnnSoftmax.cs | 3 + .../Passes/Rules/Ncnn/LowerBatchnorm.cs | 55 ++++++++++++++ .../Targets/CPUTarget.cs | 1 + modules/ncnn/src/runtime/runtime_function.cpp | 18 +++-- 10 files changed, 243 insertions(+), 8 deletions(-) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBatchNorm.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBatchnorm.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBatchnorm.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 3fd3a03cf1..cfc255911e 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -44,6 +44,20 @@ public void Unary(string name, string input, UnaryOperationType opTypes) => [0] = new ParamValue { Kind = ParamKind.Int, IntValue = (int)opTypes }, }); + public void BatchNorm(string name, string input, int channels, float eps, float[] slopeData, float[] meanData, float[] varData, float[] biasData/*, float[] aData, float[] bData*/) + { + AddLayer("BatchNorm", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = channels }, + [1] = new ParamValue { Kind = ParamKind.Float, FloatValue = eps }, + }); + + WriteFloatArray(slopeData); + WriteFloatArray(meanData); + WriteFloatArray(varData); + WriteFloatArray(biasData); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); @@ -64,4 +78,13 @@ private void AddLayer(string type, string name, string[] bottoms, string[] tops, _model.Layers.Add(layer); } + + private void WriteFloatArray(float[] data) + { + foreach (float value in data) + { + _binWriter.Write(value); + } + + } } diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 60c968998d..697329611e 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -88,6 +88,9 @@ protected override string VisitLeafCall(Call expr) case NcnnUnary op: _emitter.Unary(name, ExprMemo[expr.Arguments[0]], op.OpType); break; + case NcnnBatchNorm op: + _emitter.BatchNorm(name, ExprMemo[expr.Arguments[0]], op.Channels, op.Eps, op.SlopeData, op.MeanData, op.VarData, op.BiasData); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBatchNorm.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBatchNorm.cs new file mode 100644 index 0000000000..e843def456 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBatchNorm.cs @@ -0,0 +1,73 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnBatchNormEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnBatchNorm batchnorm) + { + var input = context.GetOrtArgumentValue(batchnorm, NcnnBatchNorm.Input); + var eps = batchnorm.Eps; + var slopeData = batchnorm.SlopeData.ToArray(); + var meanData = batchnorm.MeanData.ToArray(); + var varData = batchnorm.VarData.ToArray(); + var biasData = batchnorm.BiasData.ToArray(); + return OrtKI.BatchNormalization(input, slopeData, biasData, meanData, varData, eps, 0.0f).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnBatchNorm target) + { + var input = context.CheckArgumentType(target, NcnnBatchNorm.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnBatchNorm target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnBatchNorm target) + { + var inputType = context.GetArgumentType(target, NcnnBatchNorm.Input); + var returnType = context.GetReturnType(); + var returnF = MetricUtility.GetFLOPs(returnType); + var inputF = MetricUtility.GetFLOPs(inputType); + var inner = inputF / returnF; + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = (inner * 2) + (inputF * (MetricUtility.SubFLOPs + MetricUtility.DivFLOPs + MetricUtility.MulFLOPs + MetricUtility.AddFLOPs)), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnBatchNorm target) => context.GetArgumentShape(target, NcnnBatchNorm.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 281e9e9e62..45516868bd 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -15,5 +15,6 @@ public void ConfigureServices(IRegistrator registrator) { registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index a98aa81965..6f52268bc6 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -17,4 +17,7 @@ public static Call NcnnUnary(Expr expr, UnaryOperationType unaryOp) => public static Call NcnnSoftmax(Expr expr, int axis) => new Call(new NcnnSoftmax(axis), expr); + + public static Call NcnnBatchNorm(Expr expr, int channels, float eps, float[] slopeData, float[] meanData, float[] varData, float[] biasData) => + new Call(new NcnnBatchNorm(channels, eps, slopeData, meanData, varData, biasData), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBatchnorm.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBatchnorm.cs new file mode 100644 index 0000000000..c968ef5108 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBatchnorm.cs @@ -0,0 +1,71 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// BatchNorm expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnBatchNorm : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnBatchNorm), 0, "input"); + + /// + /// Gets channels of Ncnn BatchNorm. + /// + public int Channels { get; } + + /// + /// Gets eps. + /// + public float Eps { get; } + + /// + /// Gets slopeData of Ncnn BatchNorm. + /// + public float[] SlopeData { get; } // 斜率数据 + + /// + /// Gets meanData of Ncnn BatchNorm. + /// + public float[] MeanData { get; } // 均值数据 + + /// + /// Gets varData of Ncnn BatchNorm. + /// + public float[] VarData { get; } // 方差数据 + + /// + /// Gets biasData of Ncnn BatchNorm. + /// + public float[] BiasData { get; } // 偏置数据 + + // /// + // /// Gets aData of Ncnn BatchNorm. + // /// + // public float[] AData { get; } + + // /// + // /// Gets bData of Ncnn BatchNorm. + // /// + // public float[] BData { get; } + + /// + public override string DisplayProperty() + { + return $"{Channels},{Eps}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSoftmax.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSoftmax.cs index 298e755e8d..c028da8923 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSoftmax.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSoftmax.cs @@ -22,6 +22,9 @@ public sealed partial class NcnnSoftmax : Op /// public static readonly ParameterInfo Input = new(typeof(NcnnSoftmax), 0, "input"); + /// + /// Gets axis of Ncnn Softmax. + /// public int Axis { get; } /// diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBatchnorm.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBatchnorm.cs new file mode 100644 index 0000000000..9f5c1dec0a --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBatchnorm.cs @@ -0,0 +1,55 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; +using static Nncase.IR.F.Math; +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerBatchNorm : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsBatchNormalization( + IsWildcard("input") with { TypePattern = HasFixedShape() }, + IsTensorConst("scale"), + IsTensorConst("bias"), + IsTensorConst("mean"), + IsTensorConst("var"), + IsTensorConst("eps")); + + private Expr? GetReplace(Expr input, Tensor scale, Tensor bias, Tensor mean, Tensor var, Tensor eps) + { + if (input.CheckedShape.Rank <= 1) + { + return null; + } + + var shape = input.CheckedShape.ToValueArray().Reverse().Take(input.CheckedShape.Rank - 1).Reverse().ToArray(); + + var newVar = Add(var, eps).Evaluate().AsTensor().ToArray(); + + + var inRes = Reshape(input, shape); + var inResO = new Var(inRes.CheckedType); + + var ncnnBatchNorm = new Call(new Fusion("ncnn", NcnnBatchNorm(inResO, shape[0], 0.0f, scale.ToArray(), mean.ToArray(), newVar.ToArray(), bias.ToArray()), new[] { inResO }), inRes); + + var outRes = Reshape(ncnnBatchNorm, input.CheckedShape); + return outRes; + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index a879737a82..9060b52db8 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -48,6 +48,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions { passManager.AddWithName("LowerNcnnIR").Configure(p => { + p.Add(); p.Add(); p.Add(); }); diff --git a/modules/ncnn/src/runtime/runtime_function.cpp b/modules/ncnn/src/runtime/runtime_function.cpp index aead2f5aed..05d134d5aa 100644 --- a/modules/ncnn/src/runtime/runtime_function.cpp +++ b/modules/ncnn/src/runtime/runtime_function.cpp @@ -18,19 +18,19 @@ #include #include #include - +#include using namespace nncase; using namespace nncase::runtime; using namespace nncase::runtime::ncnn; namespace { class DataReaderFromEmpty : public ::ncnn::DataReader { - public: - virtual int scan(const char *format, void *p) const { return 0; } - virtual size_t read(void *buf, size_t size) const { - memset(buf, 0, size); - return size; - } + public: + virtual int scan(const char *format, void *p) const { return 0; } + virtual size_t read(void *buf, size_t size) const { + memset(buf, 0, size); + return size; + } }; } // namespace @@ -56,8 +56,10 @@ result ncnn_runtime_function::initialize_core( auto param_mem = reinterpret_cast( module().text().data() + context.header().entrypoint); + auto bin_mem = reinterpret_cast( + module().rdata().data() + context.header().entrypoint); ::ncnn::DataReaderFromMemory paramdr(param_mem); - DataReaderFromEmpty bindr; + ::ncnn::DataReaderFromMemory bindr(bin_mem); CHECK_WITH_ERR(!net_.load_param(paramdr), std::errc::invalid_argument); CHECK_WITH_ERR(!net_.load_model(bindr), std::errc::invalid_argument); From 652fba1de7e3f215a2e66d265b3bce2b3245e452 Mon Sep 17 00:00:00 2001 From: curioyang Date: Fri, 8 Dec 2023 10:44:25 +0000 Subject: [PATCH 014/132] Apply code-format changes --- .../CodeGen/Ncnn/NcnnEmitter.cs | 1 - .../Passes/Rules/Ncnn/LowerBatchnorm.cs | 1 - modules/ncnn/src/runtime/runtime_function.cpp | 14 +++++++------- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index cfc255911e..7e3de7e424 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -85,6 +85,5 @@ private void WriteFloatArray(float[] data) { _binWriter.Write(value); } - } } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBatchnorm.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBatchnorm.cs index 9f5c1dec0a..2358d5b2c0 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBatchnorm.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBatchnorm.cs @@ -43,7 +43,6 @@ public partial class LowerBatchNorm : RewriteRule var newVar = Add(var, eps).Evaluate().AsTensor().ToArray(); - var inRes = Reshape(input, shape); var inResO = new Var(inRes.CheckedType); diff --git a/modules/ncnn/src/runtime/runtime_function.cpp b/modules/ncnn/src/runtime/runtime_function.cpp index 05d134d5aa..e870bef70d 100644 --- a/modules/ncnn/src/runtime/runtime_function.cpp +++ b/modules/ncnn/src/runtime/runtime_function.cpp @@ -13,24 +13,24 @@ * limitations under the License. */ #include "runtime_function.h" +#include #include #include #include #include #include -#include using namespace nncase; using namespace nncase::runtime; using namespace nncase::runtime::ncnn; namespace { class DataReaderFromEmpty : public ::ncnn::DataReader { - public: - virtual int scan(const char *format, void *p) const { return 0; } - virtual size_t read(void *buf, size_t size) const { - memset(buf, 0, size); - return size; - } + public: + virtual int scan(const char *format, void *p) const { return 0; } + virtual size_t read(void *buf, size_t size) const { + memset(buf, 0, size); + return size; + } }; } // namespace From cf35523ab5a105d402d0aba6a34dd4d63f01e229 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 22 Dec 2023 10:15:08 +0800 Subject: [PATCH 015/132] [x] binary --- .../Evaluator/Ncnn/NcnnBinary.cs | 135 ++++++++++++++++++ .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs | 78 ++++++++++ .../Passes/Rules/Ncnn/LowerBinary.cs | 111 ++++++++++++++ 3 files changed, 324 insertions(+) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs new file mode 100644 index 0000000000..7d52c5d1d2 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs @@ -0,0 +1,135 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.CostModel; +using Nncase.Evaluator; +using Nncase.IR; +using Nncase.IR.Ncnn; +using Nncase.Utilities; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnBinaryEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + + /// + public IValue Visit(IEvaluateContext context, NcnnBinary binary) + { + var inputA = context.GetOrtArgumentValue(binary, NcnnBinary.InputA); + var inputB = context.GetOrtArgumentValue(binary, NcnnBinary.InputB); + var opType = binary.OpType; + + // return OrtKI. (input, dim).ToValue(); + switch (opType) + { + case BinaryOperationType.ADD: + return OrtKI.Add(inputA, inputB).ToValue(); + case BinaryOperationType.SUB: + return OrtKI.Sub(inputA, inputB).ToValue(); + case BinaryOperationType.MUL: + return OrtKI.Mul(inputA, inputB).ToValue(); + case BinaryOperationType.DIV: + return OrtKI.Div(inputA, inputB).ToValue(); + // case BinaryOperationType.MAX: + // return System.Math.Min(inputA, inputB).ToValue(); + // case BinaryOperationType.MIN: + // return OrtKI.Min(inputA, inputB).ToValue(); + + // TODO: trunc + default: + throw new NotSupportedException("Ncnn unary ops"); + } + } + + public IRType Visit(NcnnBinary target, TensorType lhs, TensorType rhs) + { + return TypeInference.BroadcastType(lhs, rhs); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnBinary target) + { + var inputA = context.CheckArgumentType(target, NcnnBinary.InputA); + return Visit(inputA); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnBinary target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnBinary target) + { + var lhsType = context.GetArgumentType(target, NcnnBinary.InputA); + var rhsType = context.GetArgumentType(target, NcnnBinary.InputB); + var outputType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(lhsType) + CostUtility.GetMemoryAccess(rhsType) + CostUtility.GetMemoryAccess(outputType), + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(outputType, (int)MetricUtility.GetBinaryFLOPs(MapBinaryOp(target.OpType))), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnBinary target) + { + var lhs = context.GetArgumentShape(target, NcnnBinary.InputA); + var rhs = context.GetArgumentShape(target, NcnnBinary.InputB); + return ShapeExprUtility.BroadcastShape(lhs, rhs); + + } + + private static BinaryOp MapBinaryOp(BinaryOperationType binaryOp) => + binaryOp switch + { + BinaryOperationType.ADD => BinaryOp.Add, + BinaryOperationType.SUB => BinaryOp.Sub, + BinaryOperationType.MUL => BinaryOp.Mul, + BinaryOperationType.DIV => BinaryOp.Div, + BinaryOperationType.MAX => BinaryOp.Max, + BinaryOperationType.MIN => BinaryOp.Min, + BinaryOperationType.POW => BinaryOp.Pow, + + // _ => null, + + // unsupported Binary ops + // BinaryOp.Mod => + // BitwiseAnd + // BitwiseOr + // BitwiseXor + // LogicalAnd + // LogicalOr + // LogicalXor + // LeftShift + // RightShift + // => BinaryOperationType.RSUB, + // => BinaryOperationType.RDIV, + // => BinaryOperationType.RPOW, + // => BinaryOperationType.ATAN2, + // => BinaryOperationType.RATAN2, + }; + + private IRType Visit(TensorType input) + { + return input; + } + + +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs new file mode 100644 index 0000000000..9ed5c057ee --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs @@ -0,0 +1,78 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +public enum BinaryOperationType +{ + ADD = 0, + SUB = 1, + MUL = 2, + DIV = 3, + MAX = 4, + MIN = 5, + POW = 6, + + // Not support below + RSUB = 7, + RDIV = 8, + RPOW = 9, + ATAN2 = 10, + RATAN2 = 11, +} + +/// +/// Binary expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnBinary : Op +{ + /// + /// Gets inputA. + /// + public static readonly ParameterInfo InputA = new(typeof(NcnnBinary), 0, "inputA"); + + /// + /// Gets inputB. + /// + public static readonly ParameterInfo InputB = new(typeof(NcnnBinary), 1, "inputB"); + + public BinaryOperationType OpType { get; } + + /// + /// Gets the flag of which input is const. + /// + public int LorR { get; } + + // These args will never used in nncase, scaler input was convert to single constant. + // So, it was no longer be scaler. + // public int WithScaler { get; } + // public float B { get; } + + /// + /// Gets constant data. + /// + public float[] ConstInput { get; } + + /// + /// Gets shape of constant data. + /// + public int[] ConstShape { get; } + + /// + public override string DisplayProperty() + { + return $"BinaryOp.{OpType}"; + } + + + +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs new file mode 100644 index 0000000000..c74084b46e --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs @@ -0,0 +1,111 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Math; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerBinary : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsBinary( + target_name: "binary", + _ => true, + IsWildcard("inputA") with { TypePattern = IsFloat() }, + IsWildcard("inputB") with { TypePattern = IsFloat() }); + + // public IPattern Pattern { get; } = IsBinary( + // "binary", + // x => x.BinaryOp is BinaryOp.Add or BinaryOp.Sub or BinaryOp.Mul or BinaryOp.Div or BinaryOp.Mod or BinaryOp.Pow, + // IsWildcard("lhs"), + // IsTensorConst("rhs", IsScalar())); + + private static BinaryOperationType? MapBinaryOp(BinaryOp binaryOp) => + binaryOp switch + { + BinaryOp.Add => BinaryOperationType.ADD, + BinaryOp.Sub => BinaryOperationType.SUB, + BinaryOp.Mul => BinaryOperationType.MUL, + BinaryOp.Div => BinaryOperationType.DIV, + BinaryOp.Max => BinaryOperationType.MAX, + BinaryOp.Min => BinaryOperationType.MIN, + BinaryOp.Pow => BinaryOperationType.POW, + + _ => null, + + // unsupported Binary ops + // BinaryOp.Mod => + // BitwiseAnd + // BitwiseOr + // BitwiseXor + // LogicalAnd + // LogicalOr + // LogicalXor + // LeftShift + // RightShift + // => BinaryOperationType.RSUB, + // => BinaryOperationType.RDIV, + // => BinaryOperationType.RPOW, + // => BinaryOperationType.ATAN2, + // => BinaryOperationType.RATAN2, + }; + + private int[] FixShape(int[] shape, int r) + { + if (shape.Length == 1) + {return shape;} + var newShape = shape.ToList(); + for (int i = r - shape.Length; i > 0; i--) + { + newShape.Insert(0, 1); + } + return newShape.ToArray(); + } + + private Expr? GetReplace(Binary binary, Expr inputA, Expr inputB) + { + if (MapBinaryOp(binary.BinaryOp) is BinaryOperationType op) + { + var newInputA = new Var(inputA.CheckedType); + var newInputB = new Var(inputB.CheckedType); + + var r = Math.Max(inputA.CheckedShape.Rank, inputB.CheckedShape.Rank); + + if (inputA is Const) + { + var constA = ((TensorConst)inputA).Value; + var constShape = FixShape(inputA.CheckedShape.ToValueArray(), r); + return new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputB }, op, 1, constA.ToArray(), constShape), new[] { newInputB }), inputB); + } + else if (inputB is Const) + { + var constB = ((TensorConst)inputB).Value; + var constShape = FixShape(inputB.CheckedShape.ToValueArray(), r); + return new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputA }, op, 2, constB.ToArray(), constShape), new[] { newInputA }), inputA); + } + else + { + return new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputA, newInputB }, op, 0, null, null), new[] { newInputA, newInputB }), inputA, inputB); + } + } + + return null; + } +} From dfa768d518439fd3955666e261cd4009f64c19cf Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 22 Dec 2023 10:17:58 +0800 Subject: [PATCH 016/132] [x] binary --- .../CodeGen/Ncnn/NcnnEmitter.cs | 49 ++++++++++++++++++- .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 18 +++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 ++ .../Targets/CPUTarget.cs | 2 + src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs | 25 ++++++++-- 6 files changed, 92 insertions(+), 6 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 7e3de7e424..56d1bf9999 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -25,6 +25,7 @@ public NcnnEmitter(BinaryWriter binWriter) public void SaveParam(Stream paramStream) { using var sw = new StreamWriter(paramStream, Encoding.ASCII, leaveOpen: true); + // using var sw = new StreamWriter(@"/home/curio/Desktop/param.txt", false, Encoding.UTF8); _model.Serialize(sw); } @@ -58,7 +59,39 @@ public void BatchNorm(string name, string input, int channels, float eps, float[ WriteFloatArray(biasData); } - private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null) + public void Binary(string name, string inputA, string inputB, BinaryOperationType opTypes, int lOrR, float[] constInput, int[] constShape) + { + var inputList = new[] { inputA, inputB }; + + if (constInput != null && constShape != null) + { + if (lOrR == 1) + { + inputList[0] = name + "_memorydata"; + } + else + { + inputList[1] = name + "_memorydata"; + } + + var paramDict = new ParamDict(); + for (int i = 0; i < constShape.Length; i++) + { + paramDict[i] = new ParamValue { Kind = ParamKind.Int, IntValue = constShape[constShape.Length - 1 - i] }; + } + + AddLayer("MemoryData", name + "_memorydata", Array.Empty(), new[] { name + "_memorydata" }, paramDict); + + WriteFloatArray(constInput); + } + + AddLayer("BinaryOp", name, inputList, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = (int)opTypes }, + }); + } + + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); if (paramDict != null) @@ -76,7 +109,19 @@ private void AddLayer(string type, string name, string[] bottoms, string[] tops, layer.Tops[i] = new NcnnTensor { Name = tops[i] }; } - _model.Layers.Add(layer); + switch (type) + { + case "Input": + _model.ModelInputs.Add(layer); + break; + case "MemoryData": + _model.MemoryDatas.Add(layer); + break; + default: + _model.Layers.Add(layer); + break; + } + } private void WriteFloatArray(float[] data) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 697329611e..735034b64b 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -91,6 +91,24 @@ protected override string VisitLeafCall(Call expr) case NcnnBatchNorm op: _emitter.BatchNorm(name, ExprMemo[expr.Arguments[0]], op.Channels, op.Eps, op.SlopeData, op.MeanData, op.VarData, op.BiasData); break; + case NcnnBinary op: + string[]? inString; + switch (op.LorR) + { + case 0: + inString = new string[] { ExprMemo[expr.Arguments[0]], ExprMemo[expr.Arguments[1]] }; + break; + case 1: + inString = new string[] { string.Empty, ExprMemo[expr.Arguments[0]] }; + break; + case 2: + inString = new string[] { ExprMemo[expr.Arguments[0]], string.Empty }; + break; + default: + throw new NotImplementedException("Not found binary emmiter."); + } + _emitter.Binary(name, inString[0], inString[1], op.OpType, op.LorR, op.ConstInput, op.ConstShape); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 45516868bd..5b7c2f1db9 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -16,5 +16,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 6f52268bc6..d2612bb331 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -20,4 +20,7 @@ public static Call NcnnSoftmax(Expr expr, int axis) => public static Call NcnnBatchNorm(Expr expr, int channels, float eps, float[] slopeData, float[] meanData, float[] varData, float[] biasData) => new Call(new NcnnBatchNorm(channels, eps, slopeData, meanData, varData, biasData), expr); + + public static Call NcnnBinary(Expr[] inExpr, BinaryOperationType binaryOp, int lorR, float[]? constInput, int[]? constShape) => + new Call(new NcnnBinary(binaryOp, lorR, constInput, constShape), inExpr); } diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 9060b52db8..b0f5d66ade 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -51,6 +51,8 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); + p.Add(); }); } diff --git a/src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs b/src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs index 419b0c66d0..c26abe93e1 100644 --- a/src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs +++ b/src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs @@ -17,17 +17,22 @@ public class NcnnModel public NcnnModel() { Magic = ExpectedMagic; + ModelInputs = new List(); Layers = new List(); + MemoryDatas = new List(); } - public NcnnModel(int magic, IList layers) + public NcnnModel(int magic, IList layers, IList? modelInputs = null, IList? memoryDatas = null) { Magic = magic; + ModelInputs = modelInputs; Layers = layers; + MemoryDatas = memoryDatas; } public int Magic { get; } - + public IList ModelInputs { get; } + public IList MemoryDatas { get; } public IList Layers { get; } public static NcnnModel ParseFromStream(Stream stream) @@ -85,9 +90,21 @@ public void Serialize(TextWriter writer) writer.WriteLine(Magic); // 2. layer_count & blob_count - writer.WriteLine($"{Layers.Count} {Layers.Select(x => x.Tops.Length).Sum()}"); + writer.WriteLine($"{Layers.Count + MemoryDatas.Count + ModelInputs.Count} {Layers.Select(x => x.Tops.Length).Sum() + MemoryDatas.Select(x => x.Tops.Length).Sum() + ModelInputs.Select(x => x.Tops.Length).Sum()}"); + + // 3. inputs + foreach (var modelInput in ModelInputs) + { + modelInput.Serialize(writer); + } + + // 4. memorydatas + foreach (var memoryData in MemoryDatas) + { + memoryData.Serialize(writer); + } - // 3. layers + // 5. layers foreach (var layer in Layers) { layer.Serialize(writer); From 3cdee3e5a3186e899746059f803c952dc0850cf6 Mon Sep 17 00:00:00 2001 From: curioyang Date: Fri, 22 Dec 2023 02:20:10 +0000 Subject: [PATCH 017/132] Apply code-format changes --- modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs | 2 +- .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 1 + modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs | 5 +---- modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs | 3 --- .../Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs | 7 +++++-- src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs | 3 +++ 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 56d1bf9999..2b07b69857 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -25,6 +25,7 @@ public NcnnEmitter(BinaryWriter binWriter) public void SaveParam(Stream paramStream) { using var sw = new StreamWriter(paramStream, Encoding.ASCII, leaveOpen: true); + // using var sw = new StreamWriter(@"/home/curio/Desktop/param.txt", false, Encoding.UTF8); _model.Serialize(sw); } @@ -121,7 +122,6 @@ private void AddLayer(string type, string name, string[] bottoms, string[] tops, _model.Layers.Add(layer); break; } - } private void WriteFloatArray(float[] data) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 735034b64b..9a0784ef50 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -107,6 +107,7 @@ protected override string VisitLeafCall(Call expr) default: throw new NotImplementedException("Not found binary emmiter."); } + _emitter.Binary(name, inString[0], inString[1], op.OpType, op.LorR, op.ConstInput, op.ConstShape); break; default: diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs index 7d52c5d1d2..9924939d48 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs @@ -21,7 +21,6 @@ namespace Nncase.Evaluator.Ncnn; /// public class NcnnBinaryEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator { - /// public IValue Visit(IEvaluateContext context, NcnnBinary binary) { @@ -40,6 +39,7 @@ public IValue Visit(IEvaluateContext context, NcnnBinary binary) return OrtKI.Mul(inputA, inputB).ToValue(); case BinaryOperationType.DIV: return OrtKI.Div(inputA, inputB).ToValue(); + // case BinaryOperationType.MAX: // return System.Math.Min(inputA, inputB).ToValue(); // case BinaryOperationType.MIN: @@ -93,7 +93,6 @@ public Expr Visit(IShapeEvaluateContext context, NcnnBinary target) var lhs = context.GetArgumentShape(target, NcnnBinary.InputA); var rhs = context.GetArgumentShape(target, NcnnBinary.InputB); return ShapeExprUtility.BroadcastShape(lhs, rhs); - } private static BinaryOp MapBinaryOp(BinaryOperationType binaryOp) => @@ -130,6 +129,4 @@ private IRType Visit(TensorType input) { return input; } - - } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs index 9ed5c057ee..8a05bec977 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs @@ -72,7 +72,4 @@ public override string DisplayProperty() { return $"BinaryOp.{OpType}"; } - - - } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs index c74084b46e..a044ac296a 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs @@ -36,7 +36,6 @@ public partial class LowerBinary : RewriteRule // x => x.BinaryOp is BinaryOp.Add or BinaryOp.Sub or BinaryOp.Mul or BinaryOp.Div or BinaryOp.Mod or BinaryOp.Pow, // IsWildcard("lhs"), // IsTensorConst("rhs", IsScalar())); - private static BinaryOperationType? MapBinaryOp(BinaryOp binaryOp) => binaryOp switch { @@ -70,12 +69,16 @@ public partial class LowerBinary : RewriteRule private int[] FixShape(int[] shape, int r) { if (shape.Length == 1) - {return shape;} + { + return shape; + } + var newShape = shape.ToList(); for (int i = r - shape.Length; i > 0; i--) { newShape.Insert(0, 1); } + return newShape.ToArray(); } diff --git a/src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs b/src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs index c26abe93e1..6c27f40cb2 100644 --- a/src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs +++ b/src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs @@ -31,8 +31,11 @@ public NcnnModel(int magic, IList layers, IList? modelInpu } public int Magic { get; } + public IList ModelInputs { get; } + public IList MemoryDatas { get; } + public IList Layers { get; } public static NcnnModel ParseFromStream(Stream stream) From b82e98a5cf91431bada86aeeac7d15a7ae76ab6d Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 22 Dec 2023 10:22:38 +0800 Subject: [PATCH 018/132] - [X] batchnorm --- .../IR/Ncnn/NcnnBatchnorm.cs | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBatchnorm.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBatchnorm.cs index c968ef5108..386caff1d7 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBatchnorm.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBatchnorm.cs @@ -36,32 +36,22 @@ public sealed partial class NcnnBatchNorm : Op /// /// Gets slopeData of Ncnn BatchNorm. /// - public float[] SlopeData { get; } // 斜率数据 + public float[] SlopeData { get; } /// /// Gets meanData of Ncnn BatchNorm. /// - public float[] MeanData { get; } // 均值数据 + public float[] MeanData { get; } /// /// Gets varData of Ncnn BatchNorm. /// - public float[] VarData { get; } // 方差数据 + public float[] VarData { get; } /// /// Gets biasData of Ncnn BatchNorm. /// - public float[] BiasData { get; } // 偏置数据 - - // /// - // /// Gets aData of Ncnn BatchNorm. - // /// - // public float[] AData { get; } - - // /// - // /// Gets bData of Ncnn BatchNorm. - // /// - // public float[] BData { get; } + public float[] BiasData { get; } /// public override string DisplayProperty() From 55662b0830a2c06ce4d70cdc651a24660282ca11 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 29 Dec 2023 17:22:12 +0800 Subject: [PATCH 019/132] support align binary broadcast axis --- .../Passes/Rules/Ncnn/LowerBinary.cs | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs index a044ac296a..d65169f0b9 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs @@ -28,14 +28,9 @@ public partial class LowerBinary : RewriteRule public override Pattern Pattern { get; } = IsBinary( target_name: "binary", _ => true, - IsWildcard("inputA") with { TypePattern = IsFloat() }, - IsWildcard("inputB") with { TypePattern = IsFloat() }); + IsWildcard("inputA") with { TypePattern = IsFloat() & HasRank(x => x <= 3) }, + IsWildcard("inputB") with { TypePattern = IsFloat() & HasRank(x => x <= 3) }); - // public IPattern Pattern { get; } = IsBinary( - // "binary", - // x => x.BinaryOp is BinaryOp.Add or BinaryOp.Sub or BinaryOp.Mul or BinaryOp.Div or BinaryOp.Mod or BinaryOp.Pow, - // IsWildcard("lhs"), - // IsTensorConst("rhs", IsScalar())); private static BinaryOperationType? MapBinaryOp(BinaryOp binaryOp) => binaryOp switch { @@ -46,7 +41,6 @@ public partial class LowerBinary : RewriteRule BinaryOp.Max => BinaryOperationType.MAX, BinaryOp.Min => BinaryOperationType.MIN, BinaryOp.Pow => BinaryOperationType.POW, - _ => null, // unsupported Binary ops @@ -86,25 +80,28 @@ private int[] FixShape(int[] shape, int r) { if (MapBinaryOp(binary.BinaryOp) is BinaryOperationType op) { - var newInputA = new Var(inputA.CheckedType); - var newInputB = new Var(inputB.CheckedType); var r = Math.Max(inputA.CheckedShape.Rank, inputB.CheckedShape.Rank); - if (inputA is Const) { var constA = ((TensorConst)inputA).Value; var constShape = FixShape(inputA.CheckedShape.ToValueArray(), r); - return new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputB }, op, 1, constA.ToArray(), constShape), new[] { newInputB }), inputB); + var newB = Reshape(inputB, FixShape(inputB.CheckedShape.ToValueArray(), r)); + var newInputB = new Var(newB.CheckedType); + return new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputB }, op, 1, constA.ToArray(), constShape), new[] { newInputB }), newB); } else if (inputB is Const) { + var newA = Reshape(inputA, FixShape(inputA.CheckedShape.ToValueArray(), r)); + var newInputA = new Var(newA.CheckedType); var constB = ((TensorConst)inputB).Value; var constShape = FixShape(inputB.CheckedShape.ToValueArray(), r); - return new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputA }, op, 2, constB.ToArray(), constShape), new[] { newInputA }), inputA); + return new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputA }, op, 2, constB.ToArray(), constShape), newInputA), newA); } else { + var newInputA = new Var(inputA.CheckedType); + var newInputB = new Var(inputB.CheckedType); return new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputA, newInputB }, op, 0, null, null), new[] { newInputA, newInputB }), inputA, inputB); } } From f72645084b3654b437f400e12d069f282292a294 Mon Sep 17 00:00:00 2001 From: curioyang Date: Fri, 29 Dec 2023 09:24:28 +0000 Subject: [PATCH 020/132] Apply code-format changes --- modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs index d65169f0b9..4abb3fdcfe 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs @@ -80,7 +80,6 @@ private int[] FixShape(int[] shape, int r) { if (MapBinaryOp(binary.BinaryOp) is BinaryOperationType op) { - var r = Math.Max(inputA.CheckedShape.Rank, inputB.CheckedShape.Rank); if (inputA is Const) { From af0d93e310bef440fa132d266ce486bf676495cc Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 4 Jan 2024 16:19:57 +0800 Subject: [PATCH 021/132] tmp profile --- modules/ncnn/src/runtime/runtime_function.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/modules/ncnn/src/runtime/runtime_function.cpp b/modules/ncnn/src/runtime/runtime_function.cpp index e870bef70d..1cbc037ae3 100644 --- a/modules/ncnn/src/runtime/runtime_function.cpp +++ b/modules/ncnn/src/runtime/runtime_function.cpp @@ -12,13 +12,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include #include "runtime_function.h" -#include #include #include #include #include #include +#include + +typedef std::chrono::high_resolution_clock::time_point TimeVar; + +#define duration(name, a) std::cout<(a).count()/1000.0<<" ms"< ncnn_runtime_function::initialize_core( CHECK_WITH_ERR(!net_.load_param(paramdr), std::errc::invalid_argument); CHECK_WITH_ERR(!net_.load_model(bindr), std::errc::invalid_argument); + net_.opt.num_threads = omp_get_num_devices(); + return ok(); } @@ -130,6 +139,7 @@ result ncnn_runtime_function::invoke_core( // 2. Extract outputs std::vector outputs; + TimeVar t = timeNow(); for (size_t i = 0; i < output_names_.size(); i++) { ::ncnn::Mat mat; CHECK_WITH_ERR(!ex.extract(output_names_[i].c_str(), mat), @@ -193,7 +203,7 @@ result ncnn_runtime_function::invoke_core( tensor t(std::in_place, dt, shape, get_default_strides(shape), buf); outputs.emplace_back(t); } - + duration("ncnn run", timeNow() - t); auto ret_val = output_names_.size() == 1 ? outputs[0] : tuple(std::in_place, std::move(outputs)); From bf046010fc180ab1f4ae255bb3e9950da03ea12a Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 5 Jan 2024 14:11:45 +0800 Subject: [PATCH 022/132] support celu [need ncnn 0826+] --- .../CodeGen/Ncnn/NcnnEmitter.cs | 6 ++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 24 +++---- .../Evaluator/Ncnn/NcnnCelu.cs | 66 +++++++++++++++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnCelu.cs | 35 ++++++++++ .../Passes/Rules/Ncnn/LowerCelu.cs | 42 ++++++++++++ .../Targets/CPUTarget.cs | 1 + 8 files changed, 163 insertions(+), 15 deletions(-) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCelu.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCelu.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 2b07b69857..d8ec81d218 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -92,6 +92,12 @@ public void Binary(string name, string inputA, string inputB, BinaryOperationTyp }); } + public void Celu(string name, string input, float alpha) => + AddLayer("CELU", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Float, FloatValue = alpha }, // alpha + }); + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 9a0784ef50..9c06cc38dc 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -92,24 +92,18 @@ protected override string VisitLeafCall(Call expr) _emitter.BatchNorm(name, ExprMemo[expr.Arguments[0]], op.Channels, op.Eps, op.SlopeData, op.MeanData, op.VarData, op.BiasData); break; case NcnnBinary op: - string[]? inString; - switch (op.LorR) + string[]? inString = op.LorR switch { - case 0: - inString = new string[] { ExprMemo[expr.Arguments[0]], ExprMemo[expr.Arguments[1]] }; - break; - case 1: - inString = new string[] { string.Empty, ExprMemo[expr.Arguments[0]] }; - break; - case 2: - inString = new string[] { ExprMemo[expr.Arguments[0]], string.Empty }; - break; - default: - throw new NotImplementedException("Not found binary emmiter."); - } - + 0 => new string[] { ExprMemo[expr.Arguments[0]], ExprMemo[expr.Arguments[1]] }, + 1 => new string[] { string.Empty, ExprMemo[expr.Arguments[0]] }, + 2 => new string[] { ExprMemo[expr.Arguments[0]], string.Empty }, + _ => throw new NotImplementedException("Not found binary emmiter."), + }; _emitter.Binary(name, inString[0], inString[1], op.OpType, op.LorR, op.ConstInput, op.ConstShape); break; + case NcnnCelu op: + _emitter.Celu(name, ExprMemo[expr.Arguments[0]], op.Alpha); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCelu.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCelu.cs new file mode 100644 index 0000000000..be3d722b15 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCelu.cs @@ -0,0 +1,66 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnCeluEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnCelu celu) + { + var input = context.GetOrtArgumentValue(celu, NcnnCelu.Input); + var alpha = celu.Alpha; + return OrtKI.Celu(input, alpha).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnCelu target) + { + var input = context.CheckArgumentType(target, NcnnCelu.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnCelu target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnCelu target) + { + var inputType = context.GetArgumentType(target, NcnnCelu.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.DivFLOPs + MetricUtility.ExpFLOPs + MetricUtility.MulFLOPs + MetricUtility.SubFLOPs + MetricUtility.AddFLOPs + (MetricUtility.CmpFLOPs * 2)), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnCelu target) => context.GetArgumentShape(target, NcnnCelu.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 5b7c2f1db9..74f7d8ee60 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -17,5 +17,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index d2612bb331..073ee9cd53 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -23,4 +23,7 @@ public static Call NcnnBatchNorm(Expr expr, int channels, float eps, float[] slo public static Call NcnnBinary(Expr[] inExpr, BinaryOperationType binaryOp, int lorR, float[]? constInput, int[]? constShape) => new Call(new NcnnBinary(binaryOp, lorR, constInput, constShape), inExpr); + + public static Call NcnnCelu(Expr expr, float alpha) => + new Call(new NcnnCelu(alpha), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCelu.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCelu.cs new file mode 100644 index 0000000000..7629b737cb --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCelu.cs @@ -0,0 +1,35 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Celu expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnCelu : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnCelu), 0, "input"); + + /// + /// Gets axis of Ncnn Celu. + /// + public float Alpha { get; } + + /// + public override string DisplayProperty() + { + return $"{Alpha}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs new file mode 100644 index 0000000000..080cfafa66 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs @@ -0,0 +1,42 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerCelu : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsCelu( + IsWildcard("input") with { TypePattern = IsFloat() }, + IsTensorConst("alpha") with { TypePattern = IsFloatScalar() } ); + + private Expr? GetReplace(Expr input, float alpha) + { + if (alpha != 1.0) + { + return false; + } + + var newInput = new Var(input.CheckedType); + + return new Call(new Fusion("ncnn", NcnnCelu(newInput, alpha), new[] { newInput }), input); + + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index b0f5d66ade..8d98e7cffd 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -53,6 +53,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + // p.Add(); //0816ncnn not support }); } From c50e181ce1f31fb5191400a085c800d549058914 Mon Sep 17 00:00:00 2001 From: curioyang Date: Fri, 5 Jan 2024 06:13:50 +0000 Subject: [PATCH 023/132] Apply code-format changes --- .../Passes/Rules/Ncnn/LowerCelu.cs | 3 +-- modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs | 1 + modules/ncnn/src/runtime/runtime_function.cpp | 11 ++++++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs index 080cfafa66..fd784999af 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs @@ -25,7 +25,7 @@ public partial class LowerCelu : RewriteRule /// public override Pattern Pattern { get; } = IsCelu( IsWildcard("input") with { TypePattern = IsFloat() }, - IsTensorConst("alpha") with { TypePattern = IsFloatScalar() } ); + IsTensorConst("alpha") with { TypePattern = IsFloatScalar() }); private Expr? GetReplace(Expr input, float alpha) { @@ -37,6 +37,5 @@ public partial class LowerCelu : RewriteRule var newInput = new Var(input.CheckedType); return new Call(new Fusion("ncnn", NcnnCelu(newInput, alpha), new[] { newInput }), input); - } } diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 8d98e7cffd..8e1a3db02b 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -53,6 +53,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + // p.Add(); //0816ncnn not support }); } diff --git a/modules/ncnn/src/runtime/runtime_function.cpp b/modules/ncnn/src/runtime/runtime_function.cpp index 1cbc037ae3..441d86a37a 100644 --- a/modules/ncnn/src/runtime/runtime_function.cpp +++ b/modules/ncnn/src/runtime/runtime_function.cpp @@ -12,18 +12,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include #include "runtime_function.h" +#include #include #include #include #include #include -#include +#include typedef std::chrono::high_resolution_clock::time_point TimeVar; -#define duration(name, a) std::cout<(a).count()/1000.0<<" ms"<(a).count() / \ + 1000.0 \ + << " ms" << std::endl #define timeNow() std::chrono::high_resolution_clock::now() using namespace nncase; From e388fbbea26b7f44299075182172bb3b888caf36 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 5 Jan 2024 15:06:07 +0800 Subject: [PATCH 024/132] support clip --- .../CodeGen/Ncnn/NcnnEmitter.cs | 7 ++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnClip.cs | 67 +++++++++++++++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnClip.cs | 40 +++++++++++ .../Passes/Rules/Ncnn/LowerClamp.cs | 39 +++++++++++ .../Targets/CPUTarget.cs | 1 + 8 files changed, 161 insertions(+) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnClip.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnClip.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerClamp.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index d8ec81d218..0e3de4fe8b 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -98,6 +98,13 @@ public void Celu(string name, string input, float alpha) => [0] = new ParamValue { Kind = ParamKind.Float, FloatValue = alpha }, // alpha }); + public void Clip(string name, string input, float min, float max) => + AddLayer("Clip", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Float, FloatValue = min }, // min + [1] = new ParamValue { Kind = ParamKind.Float, FloatValue = max }, // max + }); + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 9c06cc38dc..6309741561 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -104,6 +104,9 @@ protected override string VisitLeafCall(Call expr) case NcnnCelu op: _emitter.Celu(name, ExprMemo[expr.Arguments[0]], op.Alpha); break; + case NcnnClip op: + _emitter.Clip(name, ExprMemo[expr.Arguments[0]], op.Min, op.Max); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnClip.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnClip.cs new file mode 100644 index 0000000000..538d1ed07b --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnClip.cs @@ -0,0 +1,67 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnClipEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnClip clip) + { + var input = context.GetOrtArgumentValue(clip, NcnnClip.Input); + var min = clip.Min; + var max = clip.Max; + return OrtKI.Clip(input, min, max).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnClip target) + { + var input = context.CheckArgumentType(target, NcnnClip.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnClip target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnClip target) + { + var inputType = context.GetArgumentType(target, NcnnClip.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * 2, + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnClip target) => context.GetArgumentShape(target, NcnnClip.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 74f7d8ee60..d11e512005 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -18,5 +18,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 073ee9cd53..4834019841 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -26,4 +26,7 @@ public static Call NcnnBinary(Expr[] inExpr, BinaryOperationType binaryOp, int l public static Call NcnnCelu(Expr expr, float alpha) => new Call(new NcnnCelu(alpha), expr); + + public static Call NcnnClip(Expr expr, float min, float max) => + new Call(new NcnnClip(min, max), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnClip.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnClip.cs new file mode 100644 index 0000000000..d64ceb265f --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnClip.cs @@ -0,0 +1,40 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Clip expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnClip : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnClip), 0, "input"); + + /// + /// Gets min of Ncnn Clip. + /// + public float Min { get; } + + /// + /// Gets max of Ncnn Clip. + /// + public float Max { get; } + + /// + public override string DisplayProperty() + { + return $"{Min}, {Max}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerClamp.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerClamp.cs new file mode 100644 index 0000000000..17586066e9 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerClamp.cs @@ -0,0 +1,39 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.Utility; + +using static Nncase.PatternMatch.F.Math; +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerClamp : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsClamp( + IsWildcard("input"), + IsTensorConst("min", t => t.Value.ElementType == DataTypes.Float32), + IsTensorConst("max", t => t.Value.ElementType == DataTypes.Float32)); + + private Expr? GetReplace(Expr input, float min, float max) + { + + var newInput = new Var(input.CheckedType); + + return new Call(new Fusion("ncnn", NcnnClip(newInput, min, max), new[] { newInput }), input); + + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 8d98e7cffd..55a4363715 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -54,6 +54,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); // p.Add(); //0816ncnn not support + p.Add(); }); } From 5d1bf67f0e59f2d06b6e17114c7d6bc2f60ae071 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Tue, 9 Jan 2024 16:39:20 +0800 Subject: [PATCH 025/132] fix runtime bug --- modules/ncnn/src/runtime/runtime_function.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ncnn/src/runtime/runtime_function.cpp b/modules/ncnn/src/runtime/runtime_function.cpp index 441d86a37a..e46db24328 100644 --- a/modules/ncnn/src/runtime/runtime_function.cpp +++ b/modules/ncnn/src/runtime/runtime_function.cpp @@ -88,7 +88,7 @@ result ncnn_runtime_function::invoke_core( // 1. Set input for (size_t i = 0; i < parameters.size(); i++) { - try_var(t, parameters[0].as()); + try_var(t, parameters[i].as()); ::ncnn::Mat mat; mat.elempack = 1; mat.elemsize = runtime::get_bytes(t->dtype()); From a64b1610f737930be7b0c3262dcc8a0e208ca13d Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Tue, 9 Jan 2024 16:45:54 +0800 Subject: [PATCH 026/132] support concat --- .../CodeGen/Ncnn/NcnnEmitter.cs | 9 ++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 10 ++ .../Evaluator/Ncnn/NcnnConcat.cs | 114 ++++++++++++++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnConcat.cs | 32 +++++ .../Passes/Rules/Ncnn/LowerConcat.cs | 80 ++++++++++++ .../Targets/CPUTarget.cs | 1 + 8 files changed, 250 insertions(+) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConcat.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConcat.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConcat.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 0e3de4fe8b..87c8dbb28f 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Microsoft.Toolkit.HighPerformance; using Nncase.IR.Ncnn; using Nncase.Runtime.Ncnn; @@ -105,6 +106,14 @@ public void Clip(string name, string input, float min, float max) => [1] = new ParamValue { Kind = ParamKind.Float, FloatValue = max }, // max }); + public void Concat(string name, string[] input, int axis) + { + AddLayer("Concat", name, input, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = axis }, // axis + }); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 6309741561..782d9872e1 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -107,6 +107,16 @@ protected override string VisitLeafCall(Call expr) case NcnnClip op: _emitter.Clip(name, ExprMemo[expr.Arguments[0]], op.Min, op.Max); break; + case NcnnConcat op: + List in_ = new(); + var t = (IR.Tuple)expr.Arguments[0]; + for (int i = 0; i < t.Fields.Length; i++) + { + in_.Add(ExprMemo[t.Fields[i]]); + } + + _emitter.Concat(name, in_.ToArray(), op.Axis); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConcat.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConcat.cs new file mode 100644 index 0000000000..0b6d56040d --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConcat.cs @@ -0,0 +1,114 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using Nncase.Utilities; +using static Nncase.IR.F.Tensors; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnConcatEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnConcat concat) + { + var inputs = context.GetArgumentValueAsTensors(concat, NcnnConcat.Input); + var axis = concat.Axis; + return OrtKI.Concat(inputs.Select(t => t.ToOrtTensor()).ToArray(), axis).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnConcat target) + { + var inputs = context.CheckArgumentType(target, NcnnConcat.Input); + return Visit(inputs.Fields, target.Axis); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnConcat target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnConcat target) => Metric.Zero; + + public Expr Visit(IShapeEvaluateContext context, NcnnConcat target) => context.GetArgumentShape(target, NcnnConcat.Input); + + private IRType? CheckType(TupleType inputs) + { + bool? allScalar = null; + DataType? allDType = null; + foreach (var (i, input) in Enumerable.Range(0, inputs.Count).Select(i => (i, inputs[i]))) + { + TensorType type; + if (input is TensorType a) + { + type = a; + } + else if (input is DistributedType { TensorType: TensorType b }) + { + type = b; + } + else + { + return new InvalidType($"The ConCat Item[{i}] Must Have TensorType But Get {input}"); + } + + if (type.Shape.IsUnranked) + { + return new TensorType(type.DType, Shape.Unranked); + } + + allScalar = (allScalar ?? type.IsScalar) & type.IsScalar; + allDType ??= type.DType; + if (allDType != type.DType) + { + return new InvalidType( + $"The ConCat Item[{i}] Must Be {allDType} But Get {type.DType.GetDisplayName()}"); + } + } + + if (allScalar == true && allDType is not null) + { + return new TensorType(allDType, new[] { inputs.Count }); + } + + return null; + } + + private TensorType GetTensorType(IRType input) => input switch + { + TensorType t => t, + DistributedType d => d.TensorType, + _ => throw new InvalidCastException(), + }; + + + private IRType Visit(IRArray inputs, int axisValue) + { + var outputShape = GetTensorType(inputs[0]).Shape.ToArray(); + + foreach (var item in inputs[1..]) + { + outputShape[axisValue] += GetTensorType(item).Shape.ToArray()[axisValue]; + } + + return new TensorType(GetTensorType(inputs[0]).DType, outputShape); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index d11e512005..cdcda79a70 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -19,5 +19,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 4834019841..fd5ab3a904 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -29,4 +29,7 @@ public static Call NcnnCelu(Expr expr, float alpha) => public static Call NcnnClip(Expr expr, float min, float max) => new Call(new NcnnClip(min, max), expr); + + public static Call NcnnConcat(Expr[] expr, int axis) => + new Call(new NcnnConcat(axis), new IR.Tuple(expr)); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConcat.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConcat.cs new file mode 100644 index 0000000000..df8503f499 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConcat.cs @@ -0,0 +1,32 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Concat expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnConcat : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnConcat), 0, "inputs", ParameterKind.Input); + + /// + /// Gets axis. + /// + public int Axis { get; } + + /// + public override string DisplayProperty() => $"Axis: {Axis}"; +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConcat.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConcat.cs new file mode 100644 index 0000000000..336d585beb --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConcat.cs @@ -0,0 +1,80 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.Utility; + +using static Nncase.PatternMatch.F.Tensors; +namespace Nncase.Passes.Rules.Ncnn; + + +[RuleGenerator] +public partial class LowerConcat : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsConcat( + "concat", + _ => true, + IsTuple("tuple", IsVArgsRepeat("tupleInputs", () => IsWildcard()))); + + // squeeze concat to 3D, get outputShape，set new axis as 1 + private static (List> NewShape, List OldOutputShape) GetFixedShapeAndOldOutputShape(IReadOnlyList tupleInputs, int oldAxis) + { + var newShapes = new List>(); + var outputShape = tupleInputs[0].CheckedShape.ToValueList(); + int positive_axis = oldAxis < 0 ? tupleInputs[0].CheckedShape.Count + oldAxis : oldAxis; + outputShape[positive_axis] = 0; + foreach (var input in tupleInputs) + { + var oldInputShape = input.CheckedShape.ToValueList(); + outputShape[positive_axis] += oldInputShape[positive_axis]; + var newShape = new List { 1, oldInputShape[positive_axis], 1 }; + for (int i = 0; i < positive_axis; i++) + { + newShape[0] *= oldInputShape[i]; + } + + for (int i = positive_axis + 1; i < oldInputShape.Count; i++) + { + newShape[2] *= oldInputShape[i]; + } + + newShapes.Add(newShape); + } + + return (newShapes, outputShape); + } + + private Expr? GetReplace(IReadOnlyList tupleInputs, IR.Tensors.Concat concat) + { + var (newInputShapes, oldOutputShape) = GetFixedShapeAndOldOutputShape(tupleInputs, concat.Axis); + + var varOfNewInputs = new List(); + var callOfNewInputs = new List(); + for (int i = 0; i < tupleInputs.Count; i++) + { + var inRes = Reshape(tupleInputs[i], newInputShapes[i].ToArray()); + var inResO = new Var(inRes.CheckedType); + varOfNewInputs.Add(inResO); + callOfNewInputs.Add(inRes); + } + + var ncnnConcatCall = new Call(new Fusion("ncnn", NcnnConcat(varOfNewInputs.ToArray(), 1), varOfNewInputs.ToArray()), callOfNewInputs.ToArray()); + + var outRes = Reshape(ncnnConcatCall, oldOutputShape.ToArray()); + + return outRes; + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 737f00dd17..562952ff2f 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -56,6 +56,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions // p.Add(); //0816ncnn not support p.Add(); + p.Add(); }); } From a13218d98a40cc154e4e92457de332663435e215 Mon Sep 17 00:00:00 2001 From: curioyang Date: Tue, 9 Jan 2024 08:48:08 +0000 Subject: [PATCH 027/132] Apply code-format changes --- modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConcat.cs | 3 +-- modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerClamp.cs | 5 +---- modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConcat.cs | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConcat.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConcat.cs index 0b6d56040d..c69d5eaf5c 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConcat.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConcat.cs @@ -10,8 +10,8 @@ using Nncase.IR; using Nncase.IR.Ncnn; using Nncase.Utilities; -using static Nncase.IR.F.Tensors; using OrtKISharp; +using static Nncase.IR.F.Tensors; namespace Nncase.Evaluator.Ncnn; @@ -99,7 +99,6 @@ public Cost Visit(ICostEvaluateContext context, NcnnConcat target) _ => throw new InvalidCastException(), }; - private IRType Visit(IRArray inputs, int axisValue) { var outputShape = GetTensorType(inputs[0]).Shape.ToArray(); diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerClamp.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerClamp.cs index 17586066e9..5cbdf3b3bc 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerClamp.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerClamp.cs @@ -10,13 +10,12 @@ using Nncase.IR.Math; using Nncase.IR.Ncnn; using Nncase.PatternMatch; - using static Nncase.IR.F.Ncnn; using static Nncase.IR.F.Tensors; using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Math; using static Nncase.PatternMatch.Utility; -using static Nncase.PatternMatch.F.Math; namespace Nncase.Passes.Rules.Ncnn; [RuleGenerator] @@ -30,10 +29,8 @@ public partial class LowerClamp : RewriteRule private Expr? GetReplace(Expr input, float min, float max) { - var newInput = new Var(input.CheckedType); return new Call(new Fusion("ncnn", NcnnClip(newInput, min, max), new[] { newInput }), input); - } } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConcat.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConcat.cs index 336d585beb..2b971f4e64 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConcat.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConcat.cs @@ -10,16 +10,14 @@ using Nncase.IR.Math; using Nncase.IR.Ncnn; using Nncase.PatternMatch; - using static Nncase.IR.F.Ncnn; using static Nncase.IR.F.Tensors; using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Tensors; using static Nncase.PatternMatch.Utility; -using static Nncase.PatternMatch.F.Tensors; namespace Nncase.Passes.Rules.Ncnn; - [RuleGenerator] public partial class LowerConcat : RewriteRule { From f1d990ff975735a6b3a7db42311538df8c15080a Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 11 Jan 2024 11:17:03 +0800 Subject: [PATCH 028/132] remove ncnn channel gap in buffer --- modules/ncnn/src/runtime/runtime_function.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/modules/ncnn/src/runtime/runtime_function.cpp b/modules/ncnn/src/runtime/runtime_function.cpp index e46db24328..9dc94c49c4 100644 --- a/modules/ncnn/src/runtime/runtime_function.cpp +++ b/modules/ncnn/src/runtime/runtime_function.cpp @@ -149,9 +149,9 @@ result ncnn_runtime_function::invoke_core( ::ncnn::Mat mat; CHECK_WITH_ERR(!ex.extract(output_names_[i].c_str(), mat), std::errc::invalid_argument); - + auto mat_size = (size_t)mat.c *mat.w * mat.h * mat.d * mat.elemsize; gsl::span data{reinterpret_cast(mat.data), - mat.total() * mat.elemsize}; + mat_size}; dims_t shape; switch (mat.dims) { @@ -189,8 +189,7 @@ result ncnn_runtime_function::invoke_core( buffer_allocate_options options{}; options.flags = HOST_BUFFER_ALLOCATE_SHARED; - try_var(buf, buffer_allocator::host().allocate( - mat.total() * mat.elemsize, options)); + try_var(buf, buffer_allocator::host().allocate(mat_size, options)); try_var(hb, buf.as()); { @@ -199,8 +198,7 @@ result ncnn_runtime_function::invoke_core( for (size_t i = 0; i < (size_t)mat.c; i++) { void *dest = (unsigned char *)map.buffer().data() + (size_t)i * csize; - const void *src = - (unsigned char *)mat.data + i * mat.cstep * mat.elemsize; + const void *src = (unsigned char *)mat.channel(i); memcpy(dest, src, csize); } } From 03755a8401865d9f7cf2688c00295b2a90961ac2 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 11 Jan 2024 11:18:30 +0800 Subject: [PATCH 029/132] format --- modules/ncnn/src/runtime/runtime_function.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ncnn/src/runtime/runtime_function.cpp b/modules/ncnn/src/runtime/runtime_function.cpp index 9dc94c49c4..f519d0499b 100644 --- a/modules/ncnn/src/runtime/runtime_function.cpp +++ b/modules/ncnn/src/runtime/runtime_function.cpp @@ -149,7 +149,7 @@ result ncnn_runtime_function::invoke_core( ::ncnn::Mat mat; CHECK_WITH_ERR(!ex.extract(output_names_[i].c_str(), mat), std::errc::invalid_argument); - auto mat_size = (size_t)mat.c *mat.w * mat.h * mat.d * mat.elemsize; + auto mat_size = (size_t)mat.c * mat.w * mat.h * mat.d * mat.elemsize; gsl::span data{reinterpret_cast(mat.data), mat_size}; From d8ff9ed932f6805eaf14cf48ccb6da2c62eafc50 Mon Sep 17 00:00:00 2001 From: curioyang Date: Thu, 11 Jan 2024 03:20:50 +0000 Subject: [PATCH 030/132] Apply code-format changes --- modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConcat.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConcat.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConcat.cs index c69d5eaf5c..f0f758fcea 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConcat.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConcat.cs @@ -1,4 +1,4 @@ -// Copyright (c) Canaan Inc. All rights reserved. +// Copyright (c) Canaan Inc. All rights reserved. // Licensed under the Apache license. See LICENSE file in the project root for full license information. using System; From 712d20d4c15a9b28b46b6e67e7b7d0603f57f8f1 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 11 Jan 2024 15:55:41 +0800 Subject: [PATCH 031/132] stash conv --- .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs new file mode 100644 index 0000000000..f39d671970 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs @@ -0,0 +1,70 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Conv expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnConv : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo InputA = new(typeof(NcnnConv), 0, "input"); + + /// + /// Gets input. + /// + public static readonly ParameterInfo Weights = new(typeof(NcnnConv), 1, "weights"); + + /// + /// Gets input. + /// + public static readonly ParameterInfo Bias = new(typeof(NcnnConv), 2, "bias"); + + public int NumOutput { get; } + public int KernelW { get; } + public int KernelH { get; } + public int DilationW { get; } + public int DilationH { get; } + public int StrideW { get; } + public int StrideH { get; } + public int PadLeft { get; } + public int PadRight { get; } + public int PadTop { get; } + public int PadBotton { get; } + public float PadValue { get; } + public int BiasTerm { get; } + public int WeightDataSize { get; } + // public int Int8ScaleTerm { get; } + public int ActivationType { get; } + public float[] ActivationParams { get; } + public int DynamicWeight { get; } + + /// + /// Gets constant data. + /// + public float[] Weight { get; } + public int[] Weight { get; } + + /// + /// Gets shape of constant data. + /// + public IR.Tensor ConstShape { get; } + + /// + public override string DisplayProperty() + { + return $"ConvOp.{OpType}"; + } +} From d23a0e9f12ae30d05e10575fb3966591730de8b3 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 11 Jan 2024 18:40:25 +0800 Subject: [PATCH 032/132] cumsum --> [ncnn op] CumulativeSum --- .../CodeGen/Ncnn/NcnnEmitter.cs | 8 +++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnCumsum.cs | 67 +++++++++++++++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnCumsum.cs | 35 ++++++++++ .../Passes/Rules/Ncnn/LowerCumsum.cs | 51 ++++++++++++++ .../Targets/CPUTarget.cs | 1 + src/Nncase.Core/Utilities/ShapeUtility.cs | 34 ++++++++++ 9 files changed, 203 insertions(+) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCumsum.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCumsum.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCumsum.cs create mode 100644 src/Nncase.Core/Utilities/ShapeUtility.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 87c8dbb28f..a579bb3cee 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -114,6 +114,14 @@ public void Concat(string name, string[] input, int axis) }); } + public void Cumsum(string name, string input, int axis) + { + AddLayer("CumulativeSum", name, new[] {input}, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = axis }, // axis + }); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 782d9872e1..ba5deacf9b 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -117,6 +117,9 @@ protected override string VisitLeafCall(Call expr) _emitter.Concat(name, in_.ToArray(), op.Axis); break; + case NcnnCumsum op: + _emitter.Cumsum(name, ExprMemo[expr.Arguments[0]], op.Axis); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCumsum.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCumsum.cs new file mode 100644 index 0000000000..c4c83ede68 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCumsum.cs @@ -0,0 +1,67 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnCumsumEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnCumsum cumsum) + { + var input = context.GetOrtArgumentValue(cumsum, NcnnCumsum.Input); + var axis = cumsum.Axis; + var axisTensor = new Tensor(new[] { axis }, new Shape(1)); + return OrtKI.CumSum(input, axisTensor.ToOrtTensor(), 0L, 0L).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnCumsum target) + { + var input = context.CheckArgumentType(target, NcnnCumsum.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnCumsum target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnCumsum target) + { + var inputType = context.GetArgumentType(target, NcnnCumsum.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * MetricUtility.AddFLOPs, + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnCumsum target) => context.GetArgumentShape(target, NcnnCumsum.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index cdcda79a70..3bbfec7d6b 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -20,5 +20,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index fd5ab3a904..c4fa808cd5 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -32,4 +32,7 @@ public static Call NcnnClip(Expr expr, float min, float max) => public static Call NcnnConcat(Expr[] expr, int axis) => new Call(new NcnnConcat(axis), new IR.Tuple(expr)); + + public static Call NcnnCumsum(Expr expr, int axis) => + new Call(new NcnnCumsum(axis), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCumsum.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCumsum.cs new file mode 100644 index 0000000000..01612188ea --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCumsum.cs @@ -0,0 +1,35 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Cumsum expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnCumsum : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnCumsum), 0, "input"); + + /// + /// Gets Axis of Ncnn Cumsum. + /// + public int Axis { get; } + + /// + public override string DisplayProperty() + { + return $"{Axis}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCumsum.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCumsum.cs new file mode 100644 index 0000000000..421bcc04b0 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCumsum.cs @@ -0,0 +1,51 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Math; +using static Nncase.PatternMatch.Utility; +using static Nncase.Utilities.ShapeUtility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerCumsum : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsCumSum( + IsWildcard("input"), + IsTensorConst("axis"), + IsTensorConst("exclusive"), + IsTensorConst("reverse")); + + private Expr? GetReplace(Expr input, Tensor axis, Tensor exclusive, Tensor reverse) + { + if (exclusive[0] == false && reverse[0] == false) + { + var inResShape = FitNcnnShape(input.CheckedShape.ToValueList(), axis[0]); + var inRes = Reshape(input, inResShape.ToArray()); + var inRes0 = new Var(inRes.CheckedType); + + var newInput = new Var(input.CheckedType); + var cumsum_ = new Call(new Fusion("ncnn", NcnnCumsum(newInput, 1), new[] { newInput }), inRes); + + var outRes = Reshape(cumsum_, input.CheckedShape.ToValueList().ToArray()); + + return outRes; + } + + return null; + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 562952ff2f..1a31354bdb 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -57,6 +57,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions // p.Add(); //0816ncnn not support p.Add(); p.Add(); + p.Add(); }); } diff --git a/src/Nncase.Core/Utilities/ShapeUtility.cs b/src/Nncase.Core/Utilities/ShapeUtility.cs new file mode 100644 index 0000000000..bc39790ec9 --- /dev/null +++ b/src/Nncase.Core/Utilities/ShapeUtility.cs @@ -0,0 +1,34 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.TIR; +using static NetFabric.Hyperlinq.ArrayExtensions; + +namespace Nncase.Utilities; + +public static class ShapeUtility +{ + public static List FitNcnnShape(List shape, int axis) + { + int positive_axis = axis < 0 ? shape.Count + axis : axis; + var newShape = new List { 1, shape[positive_axis], 1 }; + for (int i = 0; i < positive_axis; i++) + { + newShape[0] *= shape[i]; + } + + for (int i = positive_axis + 1; i < shape.Count; i++) + { + newShape[2] *= shape[i]; + } + + return newShape; + } +} From e8ffebe205b205e0d52f9cd5dc13f513dd1aa776 Mon Sep 17 00:00:00 2001 From: curioyang Date: Thu, 11 Jan 2024 10:42:36 +0000 Subject: [PATCH 033/132] Apply code-format changes --- modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs | 2 +- modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCumsum.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index a579bb3cee..e6339deabd 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -116,7 +116,7 @@ public void Concat(string name, string[] input, int axis) public void Cumsum(string name, string input, int axis) { - AddLayer("CumulativeSum", name, new[] {input}, new[] { name }, new ParamDict + AddLayer("CumulativeSum", name, new[] { input }, new[] { name }, new ParamDict { [0] = new ParamValue { Kind = ParamKind.Int, IntValue = axis }, // axis }); diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCumsum.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCumsum.cs index 421bcc04b0..0af7b4c1be 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCumsum.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCumsum.cs @@ -36,7 +36,7 @@ public partial class LowerCumsum : RewriteRule { var inResShape = FitNcnnShape(input.CheckedShape.ToValueList(), axis[0]); var inRes = Reshape(input, inResShape.ToArray()); - var inRes0 = new Var(inRes.CheckedType); + _ = new Var(inRes.CheckedType); var newInput = new Var(input.CheckedType); var cumsum_ = new Call(new Fusion("ncnn", NcnnCumsum(newInput, 1), new[] { newInput }), inRes); From 28e3fa5e7be3fc8abc03633baf9f504ec26ee6f1 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 18 Jan 2024 15:05:48 +0800 Subject: [PATCH 034/132] conv2d --> [ncnn op] conv --- .../CodeGen/Ncnn/NcnnEmitter.cs | 41 ++++++++- .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 11 +++ .../Evaluator/Ncnn/NcnnConv.cs | 88 +++++++++++++++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 4 + .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs | 88 ++++++++++++++++--- .../Passes/Rules/Ncnn/LowerConv2d.cs | 64 ++++++++++++++ .../Targets/CPUTarget.cs | 2 + 8 files changed, 283 insertions(+), 16 deletions(-) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConv.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index e6339deabd..c38ac5819f 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -26,11 +26,18 @@ public NcnnEmitter(BinaryWriter binWriter) public void SaveParam(Stream paramStream) { using var sw = new StreamWriter(paramStream, Encoding.ASCII, leaveOpen: true); - - // using var sw = new StreamWriter(@"/home/curio/Desktop/param.txt", false, Encoding.UTF8); _model.Serialize(sw); } + public void SaveBin() + { + using (var fileStream = File.Create(Directory.GetCurrentDirectory() + "/ncnn.bin")) + { + _binWriter.BaseStream.Seek(0, SeekOrigin.Begin); + _binWriter.BaseStream.CopyTo(fileStream); + } + } + public void Input(string name) => AddLayer("Input", name, Array.Empty(), new[] { name }, null); @@ -114,6 +121,36 @@ public void Concat(string name, string[] input, int axis) }); } + public void Conv(string name, string input, float[] weightsData, float[] biasData, int numOutput, int kernelW, int kernelH, int dilationW, int dilationH, int strideW, int strideH, int padLeft, int padRight, int padBottom, int padTop, int biasTerm, int weightsDataSize, int int8Flag, int actType, float[] actParams, float padValue, int dynamicFlag) + { + AddLayer("Convolution", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = numOutput }, + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = kernelW }, + [11] = new ParamValue { Kind = ParamKind.Int, IntValue = kernelH }, + [2] = new ParamValue { Kind = ParamKind.Int, IntValue = dilationW }, + [12] = new ParamValue { Kind = ParamKind.Int, IntValue = dilationH }, + [3] = new ParamValue { Kind = ParamKind.Int, IntValue = strideW }, + [13] = new ParamValue { Kind = ParamKind.Int, IntValue = strideH }, + [4] = new ParamValue { Kind = ParamKind.Int, IntValue = padLeft }, + [14] = new ParamValue { Kind = ParamKind.Int, IntValue = padTop }, + [16] = new ParamValue { Kind = ParamKind.Int, IntValue = padRight }, + [15] = new ParamValue { Kind = ParamKind.Int, IntValue = padBottom }, + [5] = new ParamValue { Kind = ParamKind.Int, IntValue = biasTerm }, + [6] = new ParamValue { Kind = ParamKind.Int, IntValue = weightsDataSize }, + [7] = new ParamValue { Kind = ParamKind.Int, IntValue = 1 }, // Group + [8] = new ParamValue { Kind = ParamKind.Int, IntValue = int8Flag }, + + // [9] = new ParamValue { Kind = ParamKind.Int, IntValue = actType }, + // [10] = new ParamValue { Kind = ParamKind.ArrayOfFloat, TensorValue = actParams }, + [18] = new ParamValue { Kind = ParamKind.Float, FloatValue = padValue }, + [19] = new ParamValue { Kind = ParamKind.Int, IntValue = dynamicFlag }, + }); + WriteFloatArray(new float[] { 0 }); // quantize flag [Not exist in ncnn op.md] + WriteFloatArray(weightsData); + WriteFloatArray(biasData); + } + public void Cumsum(string name, string input, int axis) { AddLayer("CumulativeSum", name, new[] { input }, new[] { name }, new ParamDict diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index ba5deacf9b..024298b682 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -48,6 +48,14 @@ protected override void Compile(BaseFunction callable) protected override void WriteText() { _emitter.SaveParam(TextWriter.BaseStream); + + using (var fileStream = File.Create(Directory.GetCurrentDirectory() + "/ncnn.param")) + { + TextWriter.BaseStream.Seek(0, SeekOrigin.Begin); + TextWriter.BaseStream.CopyTo(fileStream); + } + + _emitter.SaveBin(); } private class CodeGenVisitor : ExprVisitor @@ -117,6 +125,9 @@ protected override string VisitLeafCall(Call expr) _emitter.Concat(name, in_.ToArray(), op.Axis); break; + case NcnnConv op: + _emitter.Conv(name, ExprMemo[expr.Arguments[0]], op.WeightData, op.BiasData, op.NumOutput, op.KernelW, op.KernelH, op.DilationW, op.DilationH, op.StrideW, op.StrideH, op.PadLeft, op.PadTop, op.PadRight, op.PadBottom, op.BiasTerm, op.WeightDataSize, op.Int8ScaleTerm, op.ActivationType, op.ActivationParams, op.PadValue, op.DynamicWeight); + break; case NcnnCumsum op: _emitter.Cumsum(name, ExprMemo[expr.Arguments[0]], op.Axis); break; diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConv.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConv.cs new file mode 100644 index 0000000000..ec130db3c7 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConv.cs @@ -0,0 +1,88 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; +using static Nncase.IR.F.Tensors; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnConvEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnConv conv) + { + var inputs = context.GetOrtArgumentValue(conv, NcnnConv.Input); + var weights = new Tensor(conv.WeightData, new[] { conv.NumOutput, conv.WeightDataSize / (conv.KernelH * conv.KernelW), conv.KernelH, conv.KernelW }).ToOrtTensor(); + var bias = new Tensor(conv.BiasData, new[] { conv.NumOutput }).ToOrtTensor(); + var result = OrtKI.Conv(inputs, weights, bias, "NOTSET", new long[] { conv.DilationH, conv.DilationW }, 1, new long[] { conv.KernelH, conv.KernelW }, new long[] { conv.PadLeft, conv.PadTop, conv.PadRight, conv.PadBottom }, new long[] { conv.StrideH, conv.StrideW }); + return OrtKI.Clip(result, conv.ActivationParams[0], conv.ActivationParams[1]).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnConv target) + { + var inputs = context.CheckArgumentType(target, NcnnConv.Input); + return Visit(inputs, target); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnConv target) + { + var inputType = context.GetArgumentType(target, NcnnConv.Input); + var weightsType = new TensorType(DataTypes.Float32, new[] { target.WeightDataSize }); + var biasType = new TensorType(DataTypes.Float32, new[] { target.NumOutput }); + var macPerElement = (2 * target.WeightDataSize / target.NumOutput) - 1; + + var outputType = context.GetReturnType(); + + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(inputType) + CostUtility.GetMemoryAccess(weightsType) + CostUtility.GetMemoryAccess(biasType), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(outputType), + [CostFactorNames.CPUCycles] = CostUtility.GetCPUCycles(outputType, (uint)macPerElement), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnConv target) => Metric.Zero; + + public Expr Visit(IShapeEvaluateContext context, NcnnConv target) => context.GetArgumentShape(target, NcnnConv.Input); + + private TensorType GetTensorType(IRType input) => input switch + { + TensorType t => t, + DistributedType d => d.TensorType, + _ => throw new InvalidCastException(), + }; + + private IRType Visit(TensorType input, NcnnConv conv) + { + var outputShape = input.Shape.ToList(); + var kernelShape = new[] { conv.NumOutput, conv.WeightDataSize / (conv.NumOutput * conv.KernelH * conv.KernelW), conv.KernelH, conv.KernelW }; + outputShape[0] = conv.NumOutput; + + outputShape[1] = IR.TypePatternUtility.GetWindowedOutputSize( + input.Shape[1].FixedValue + conv.PadLeft + conv.PadRight, + kernelShape[2], + conv.StrideH, + conv.DilationH, + false); + outputShape[2] = IR.TypePatternUtility.GetWindowedOutputSize( + input.Shape[2].FixedValue + conv.PadTop + conv.PadBottom, + kernelShape[3], + conv.StrideW, + conv.DilationW, + false); + return new TensorType(GetTensorType(input).DType, outputShape.ToArray()); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 3bbfec7d6b..932d6b508f 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -20,6 +20,7 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index c4fa808cd5..5a0807fae0 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -33,6 +33,10 @@ public static Call NcnnClip(Expr expr, float min, float max) => public static Call NcnnConcat(Expr[] expr, int axis) => new Call(new NcnnConcat(axis), new IR.Tuple(expr)); + // In ncnn param file, lauout is [w, h] for kernel, dilation + public static Call NcnnConv(Expr expr, float[] weightsData, float[] biasData, int numOutput, int[] kernel, int[] dilation, int[] strides, int[] pads, int biasTerm, int weightsDataSize, int int8Flag, int actType, float[] actParams, float padValue, int dynamicFlag) => + new Call(new NcnnConv(weightsData, biasData, numOutput, kernel[1], kernel[0], dilation[1], dilation[0], strides[1], strides[0], pads[0], pads[1], pads[2], pads[3], padValue, biasTerm, weightsDataSize, int8Flag, actType, actParams, dynamicFlag), expr); + public static Call NcnnCumsum(Expr expr, int axis) => new Call(new NcnnCumsum(axis), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs index f39d671970..cbb104eca9 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs @@ -20,51 +20,111 @@ public sealed partial class NcnnConv : Op /// /// Gets input. /// - public static readonly ParameterInfo InputA = new(typeof(NcnnConv), 0, "input"); + public static readonly ParameterInfo Input = new(typeof(NcnnConv), 0, "input"); /// /// Gets input. /// - public static readonly ParameterInfo Weights = new(typeof(NcnnConv), 1, "weights"); + public float[] WeightData { get; } /// - /// Gets input. + /// Gets BiasData. /// - public static readonly ParameterInfo Bias = new(typeof(NcnnConv), 2, "bias"); + public float[] BiasData { get; } + /// + /// Gets NumOutput. + /// public int NumOutput { get; } + + /// + /// Gets KernelW. + /// public int KernelW { get; } + + /// + /// Gets KernelH. + /// public int KernelH { get; } + + /// + /// Gets DilationW. + /// public int DilationW { get; } + + /// + /// Gets DilationH. + /// public int DilationH { get; } + + /// + /// Gets StrideW. + /// public int StrideW { get; } + + /// + /// Gets StrideH. + /// public int StrideH { get; } + + /// + /// Gets PadLeft. + /// public int PadLeft { get; } + + /// + /// Gets PadRight. + /// public int PadRight { get; } + + /// + /// Gets PadTop. + /// public int PadTop { get; } - public int PadBotton { get; } + + /// + /// Gets PadBottom. + /// + public int PadBottom { get; } + + /// + /// Gets PadValue. + /// public float PadValue { get; } + + /// + /// Gets BiasTerm. + /// public int BiasTerm { get; } + + /// + /// Gets WeightDataSize. + /// public int WeightDataSize { get; } - // public int Int8ScaleTerm { get; } + + /// + /// Gets Int8ScaleTerm. + /// + public int Int8ScaleTerm { get; } + + /// + /// Gets ActivationType. + /// public int ActivationType { get; } - public float[] ActivationParams { get; } - public int DynamicWeight { get; } /// - /// Gets constant data. + /// Gets ActivationParams. /// - public float[] Weight { get; } - public int[] Weight { get; } + public float[] ActivationParams { get; } /// - /// Gets shape of constant data. + /// Gets DynamicWeight. /// - public IR.Tensor ConstShape { get; } + public int DynamicWeight { get; } /// public override string DisplayProperty() { - return $"ConvOp.{OpType}"; + return $""; } } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs new file mode 100644 index 0000000000..70cc555652 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs @@ -0,0 +1,64 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerConv : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsConv2D( + "conv", + conv => conv.PadMode == PadMode.Constant, + IsWildcard("input"), + IsTensorConst("weights"), + IsTensorConst("bias"), + IsTensorConst("strides"), + IsTensorConst("paddings"), + IsTensorConst("dilation"), + IsTensorConst("groups"), + IsTensorConst("fusedClamp")); + + private Expr? GetReplace(Expr input, Tensor weights, Tensor bias, int[] strides, int[] paddings, int[] dilation, int groups, float[] fusedClamp) + { + var (numOutput, kernelH, kernelW) = (weights.Shape[0], weights.Shape[2], weights.Shape[3]); + var (dilationH, dilationW) = (dilation[0], dilation.Length == 2 ? dilation[1] : dilation[0]); + var (strideH, strideW) = (strides[0], strides.Length == 2 ? strides[1] : strides[0]); + + // int[] paddingsValue = ((TensorConst)paddings).Value.ToArray(); + var (padLeft, padRight, padTop, padBottom) = (paddings[2], paddings[3], paddings[0], paddings[1]); + float padValue = 0; + int biasTerm = 1; + var weightsDataSize = weights.Shape[0] * weights.Shape[1] * weights.Shape[2] * weights.Shape[3]; + int int8ScaleTerm = 0; + + // 1:Relu 2: leaky relu 3:clip 4: sigmoid 5: mish 6: hardswish + int activationType = 3; + int dynamicWeight = 0; + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + var conv = new Call(new Fusion("ncnn", NcnnConv(inResO, weights.ToArray(), bias.ToArray(), numOutput.FixedValue, new[] { kernelH.FixedValue, kernelW.FixedValue }, new[] { dilationH, dilationW }, new[] { strideH, strideW }, new[] { padLeft, padRight, padBottom, padTop }, biasTerm, weightsDataSize.FixedValue, int8ScaleTerm, activationType, fusedClamp, padValue, dynamicWeight), new[] { inResO }), inRes); + + var outRes = Unsqueeze(conv, new[] { 0 }); + + return outRes; + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 1a31354bdb..858b494dad 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -55,8 +55,10 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); // p.Add(); //0816ncnn not support + p.Add(); p.Add(); + p.Add(); p.Add(); }); } From 9d6665cf3d64a4093c38ef1eafcfbd637fe58627 Mon Sep 17 00:00:00 2001 From: curioyang Date: Thu, 18 Jan 2024 07:08:41 +0000 Subject: [PATCH 035/132] Apply code-format changes --- modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 858b494dad..3632050909 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -55,7 +55,6 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); // p.Add(); //0816ncnn not support - p.Add(); p.Add(); p.Add(); From efe9962c7221ef0ac322e2445610ef497d41f8b0 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 18 Jan 2024 16:04:51 +0800 Subject: [PATCH 036/132] Elu --> [ncnn op] ELU --- .../CodeGen/Ncnn/NcnnEmitter.cs | 6 ++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnElu.cs | 66 +++++++++++++++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnElu.cs | 35 ++++++++++ .../Passes/Rules/Ncnn/LowerElu.cs | 36 ++++++++++ .../Targets/CPUTarget.cs | 1 + 8 files changed, 151 insertions(+) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnElu.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnElu.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerElu.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index c38ac5819f..4d79eb5ee0 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -159,6 +159,12 @@ public void Cumsum(string name, string input, int axis) }); } + public void Elu(string name, string input, float alpha) => + AddLayer("ELU", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Float, FloatValue = alpha }, // alpha + }); + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 024298b682..bb1ca5d027 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -131,6 +131,9 @@ protected override string VisitLeafCall(Call expr) case NcnnCumsum op: _emitter.Cumsum(name, ExprMemo[expr.Arguments[0]], op.Axis); break; + case NcnnElu op: + _emitter.Elu(name, ExprMemo[expr.Arguments[0]], op.Alpha); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnElu.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnElu.cs new file mode 100644 index 0000000000..5378e2727a --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnElu.cs @@ -0,0 +1,66 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnEluEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnElu elu) + { + var input = context.GetOrtArgumentValue(elu, NcnnElu.Input); + var alpha = elu.Alpha; + return OrtKI.Elu(input, alpha).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnElu target) + { + var input = context.CheckArgumentType(target, NcnnElu.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnElu target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnElu target) + { + var inputType = context.GetArgumentType(target, NcnnElu.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.ExpFLOPs + MetricUtility.MulFLOPs + MetricUtility.SubFLOPs + MetricUtility.CmpFLOPs ), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnElu target) => context.GetArgumentShape(target, NcnnElu.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 932d6b508f..c31efc1262 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -22,5 +22,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 5a0807fae0..eab1fdf62a 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -39,4 +39,7 @@ public static Call NcnnConv(Expr expr, float[] weightsData, float[] biasData, in public static Call NcnnCumsum(Expr expr, int axis) => new Call(new NcnnCumsum(axis), expr); + + public static Call NcnnElu(Expr expr, float alpha) => + new Call(new NcnnElu(alpha), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnElu.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnElu.cs new file mode 100644 index 0000000000..d901549c2f --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnElu.cs @@ -0,0 +1,35 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Gets expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnElu : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnElu), 0, "input"); + + /// + /// Gets axis of Ncnn Elu. + /// + public float Alpha { get; } + + /// + public override string DisplayProperty() + { + return $"{Alpha}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerElu.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerElu.cs new file mode 100644 index 0000000000..1fe0110338 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerElu.cs @@ -0,0 +1,36 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerElu : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsElu( + IsWildcard("input") with { TypePattern = IsFloat() }, + IsTensorConst("alpha") with { TypePattern = IsFloatScalar() }); + + private Expr? GetReplace(Expr input, float alpha) + { + var newInput = new Var(input.CheckedType); + + return new Call(new Fusion("ncnn", NcnnElu(newInput, alpha), new[] { newInput }), input); + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 858b494dad..4eb84c1fab 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -60,6 +60,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); }); } From 59a82b96acba317a618af0d2af12dd7b03fb2af9 Mon Sep 17 00:00:00 2001 From: curioyang Date: Thu, 18 Jan 2024 08:07:37 +0000 Subject: [PATCH 037/132] Apply code-format changes --- modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnElu.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnElu.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnElu.cs index 5378e2727a..e450a140b5 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnElu.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnElu.cs @@ -52,7 +52,7 @@ public Metric Visit(IMetricEvaluateContext context, NcnnElu target) return new() { [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, - [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.ExpFLOPs + MetricUtility.MulFLOPs + MetricUtility.SubFLOPs + MetricUtility.CmpFLOPs ), + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.ExpFLOPs + MetricUtility.MulFLOPs + MetricUtility.SubFLOPs + MetricUtility.CmpFLOPs), [MetricFactorNames.Parallel] = 4, }; } From f37f6392c9b03361ef6938080d1b80e266ef1df7 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 18 Jan 2024 16:49:32 +0800 Subject: [PATCH 038/132] format ncnn param --- src/Nncase.Core/Runtime/Ncnn/NcnnLayer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nncase.Core/Runtime/Ncnn/NcnnLayer.cs b/src/Nncase.Core/Runtime/Ncnn/NcnnLayer.cs index 71735ee2a6..0fe6734820 100644 --- a/src/Nncase.Core/Runtime/Ncnn/NcnnLayer.cs +++ b/src/Nncase.Core/Runtime/Ncnn/NcnnLayer.cs @@ -44,7 +44,7 @@ public NcnnLayer(string type, string name, int bottomCount, int topCount) public void Serialize(TextWriter writer) { - writer.Write($"{Type}\t{Name}\t{Bottoms.Length} {Tops.Length} "); + writer.Write($"{Type,-16} {Name,-24} {Bottoms.Length} {Tops.Length} "); foreach (var bottom in Bottoms) { From 21321ff1412f16ee42a4c520b45b049174c206fa Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 18 Jan 2024 18:18:51 +0800 Subject: [PATCH 039/132] Erf --> [ncnn op] Erf --- .../CodeGen/Ncnn/NcnnEmitter.cs | 2 + .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnErf.cs | 65 +++++++++++++++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 8 +-- .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnErf.cs | 30 +++++++++ .../Passes/Rules/Ncnn/LowerErf.cs | 45 +++++++++++++ .../Targets/CPUTarget.cs | 1 + 8 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnErf.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnErf.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerErf.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 4d79eb5ee0..16071255ac 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -165,6 +165,8 @@ public void Elu(string name, string input, float alpha) => [0] = new ParamValue { Kind = ParamKind.Float, FloatValue = alpha }, // alpha }); + public void Erf(string name, string input) => AddLayer("Erf", name, new[] { input }, new[] { name }, new ParamDict { }); + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index bb1ca5d027..c87657007a 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -134,6 +134,9 @@ protected override string VisitLeafCall(Call expr) case NcnnElu op: _emitter.Elu(name, ExprMemo[expr.Arguments[0]], op.Alpha); break; + case NcnnErf op: + _emitter.Erf(name, ExprMemo[expr.Arguments[0]]); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnErf.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnErf.cs new file mode 100644 index 0000000000..3ab6c8bef4 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnErf.cs @@ -0,0 +1,65 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnErfEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnErf erf) + { + var input = context.GetOrtArgumentValue(erf, NcnnErf.Input); + return OrtKI.Erf(input).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnErf target) + { + var input = context.CheckArgumentType(target, NcnnErf.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnErf target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnErf target) + { + var inputType = context.GetArgumentType(target, NcnnErf.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.ExpFLOPs + MetricUtility.MulFLOPs + MetricUtility.SubFLOPs + MetricUtility.CmpFLOPs ), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnErf target) => context.GetArgumentShape(target, NcnnErf.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index c31efc1262..77b80e306e 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -23,5 +23,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index eab1fdf62a..9cecbe6a6e 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -37,9 +37,9 @@ public static Call NcnnConcat(Expr[] expr, int axis) => public static Call NcnnConv(Expr expr, float[] weightsData, float[] biasData, int numOutput, int[] kernel, int[] dilation, int[] strides, int[] pads, int biasTerm, int weightsDataSize, int int8Flag, int actType, float[] actParams, float padValue, int dynamicFlag) => new Call(new NcnnConv(weightsData, biasData, numOutput, kernel[1], kernel[0], dilation[1], dilation[0], strides[1], strides[0], pads[0], pads[1], pads[2], pads[3], padValue, biasTerm, weightsDataSize, int8Flag, actType, actParams, dynamicFlag), expr); - public static Call NcnnCumsum(Expr expr, int axis) => - new Call(new NcnnCumsum(axis), expr); + public static Call NcnnCumsum(Expr expr, int axis) => new Call(new NcnnCumsum(axis), expr); - public static Call NcnnElu(Expr expr, float alpha) => - new Call(new NcnnElu(alpha), expr); + public static Call NcnnElu(Expr expr, float alpha) => new Call(new NcnnElu(alpha), expr); + + public static Call NcnnErf(Expr expr) => new Call(new NcnnErf(), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnErf.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnErf.cs new file mode 100644 index 0000000000..84fa7d9038 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnErf.cs @@ -0,0 +1,30 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Gets expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnErf : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnErf), 0, "input"); + + /// + public override string DisplayProperty() + { + return $""; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerErf.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerErf.cs new file mode 100644 index 0000000000..775d45de4e --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerErf.cs @@ -0,0 +1,45 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerErf : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsErf( + IsWildcard("input") with { TypePattern = IsFloat() }); + + private Expr? GetReplace(Expr input) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var erf = new Call(new Fusion("ncnn", NcnnErf(inResO), new[] { inResO }), inRes); + + return Unsqueeze(erf, new[] { 0 }); + + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index ef56726d93..02598873b9 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -60,6 +60,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); }); } From db7efb11ea3c880aeb9f8427e582dfa11c550a47 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 18 Jan 2024 18:20:26 +0800 Subject: [PATCH 040/132] disable lowering erf --- modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 02598873b9..4c27fc1cf9 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -60,7 +60,8 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); - p.Add(); + + // p.Add(); // need ncnn later than 20230908 }); } From 8ea444a9033fcfd59f66d7163ccc490cad705b8d Mon Sep 17 00:00:00 2001 From: curioyang Date: Thu, 18 Jan 2024 10:22:41 +0000 Subject: [PATCH 041/132] Apply code-format changes --- modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs | 2 +- modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnErf.cs | 2 +- modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerErf.cs | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index c87657007a..b133d51b5a 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -134,7 +134,7 @@ protected override string VisitLeafCall(Call expr) case NcnnElu op: _emitter.Elu(name, ExprMemo[expr.Arguments[0]], op.Alpha); break; - case NcnnErf op: + case NcnnErf: _emitter.Erf(name, ExprMemo[expr.Arguments[0]]); break; default: diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnErf.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnErf.cs index 3ab6c8bef4..bc890a5ace 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnErf.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnErf.cs @@ -51,7 +51,7 @@ public Metric Visit(IMetricEvaluateContext context, NcnnErf target) return new() { [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, - [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.ExpFLOPs + MetricUtility.MulFLOPs + MetricUtility.SubFLOPs + MetricUtility.CmpFLOPs ), + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.ExpFLOPs + MetricUtility.MulFLOPs + MetricUtility.SubFLOPs + MetricUtility.CmpFLOPs), [MetricFactorNames.Parallel] = 4, }; } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerErf.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerErf.cs index 775d45de4e..4f15d9b6a9 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerErf.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerErf.cs @@ -40,6 +40,5 @@ public partial class LowerErf : RewriteRule var erf = new Call(new Fusion("ncnn", NcnnErf(inResO), new[] { inResO }), inRes); return Unsqueeze(erf, new[] { 0 }); - } } From 19852edbd2af500b68c2473fe5ac8d50f2059555 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 18 Jan 2024 18:42:05 +0800 Subject: [PATCH 042/132] fix warning --- src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs b/src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs index 6c27f40cb2..7e1362f2ef 100644 --- a/src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs +++ b/src/Nncase.Core/Runtime/Ncnn/NcnnModel.cs @@ -25,9 +25,9 @@ public NcnnModel() public NcnnModel(int magic, IList layers, IList? modelInputs = null, IList? memoryDatas = null) { Magic = magic; - ModelInputs = modelInputs; + ModelInputs = modelInputs ?? new List(); Layers = layers; - MemoryDatas = memoryDatas; + MemoryDatas = memoryDatas ?? new List(); } public int Magic { get; } From a9641b442e75ff1177cf1fe2ae05a688ffaf6ecf Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 19 Jan 2024 14:06:05 +0800 Subject: [PATCH 043/132] Hardsigmoid --> [ncnn op] HardSigmoid --- .../CodeGen/Ncnn/NcnnEmitter.cs | 7 ++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnHardSigmoid.cs | 67 +++++++++++++++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 2 + .../IR/Ncnn/NcnnHardSigmoid.cs | 40 +++++++++++ .../Passes/Rules/Ncnn/LowerHardSigmoid.cs | 47 +++++++++++++ .../Targets/CPUTarget.cs | 1 + 8 files changed, 168 insertions(+) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSigmoid.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSigmoid.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSigmoid.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 16071255ac..2e6b480c66 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -167,6 +167,13 @@ public void Elu(string name, string input, float alpha) => public void Erf(string name, string input) => AddLayer("Erf", name, new[] { input }, new[] { name }, new ParamDict { }); + public void HardSigmoid(string name, string input, float alpha, float beta) => + AddLayer("HardSigmoid", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Float, FloatValue = alpha }, // alpha + [1] = new ParamValue { Kind = ParamKind.Float, FloatValue = beta }, // beta + }); + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index b133d51b5a..abba5bcda4 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -137,6 +137,9 @@ protected override string VisitLeafCall(Call expr) case NcnnErf: _emitter.Erf(name, ExprMemo[expr.Arguments[0]]); break; + case NcnnHardSigmoid op: + _emitter.HardSigmoid(name, ExprMemo[expr.Arguments[0]], op.Alpha, op.Beta); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSigmoid.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSigmoid.cs new file mode 100644 index 0000000000..974d318c1f --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSigmoid.cs @@ -0,0 +1,67 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnHardSigmoidEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnHardSigmoid hardSigmoid) + { + var input = context.GetOrtArgumentValue(hardSigmoid, NcnnHardSigmoid.Input); + var alpha = hardSigmoid.Alpha; + var beta = hardSigmoid.Beta; + return OrtKI.HardSigmoid(input, alpha, beta).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnHardSigmoid target) + { + var input = context.CheckArgumentType(target, NcnnHardSigmoid.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnHardSigmoid target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnHardSigmoid target) + { + var inputType = context.GetArgumentType(target, NcnnHardSigmoid.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.ExpFLOPs + MetricUtility.MulFLOPs + MetricUtility.SubFLOPs + MetricUtility.CmpFLOPs), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnHardSigmoid target) => context.GetArgumentShape(target, NcnnHardSigmoid.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 77b80e306e..4cc898a3ec 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -24,5 +24,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 9cecbe6a6e..749b311c95 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -42,4 +42,6 @@ public static Call NcnnConv(Expr expr, float[] weightsData, float[] biasData, in public static Call NcnnElu(Expr expr, float alpha) => new Call(new NcnnElu(alpha), expr); public static Call NcnnErf(Expr expr) => new Call(new NcnnErf(), expr); + + public static Call NcnnHardSigmoid(Expr expr, float alpha, float beta) => new Call(new NcnnHardSigmoid(alpha, beta), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSigmoid.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSigmoid.cs new file mode 100644 index 0000000000..6646c0ce3e --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSigmoid.cs @@ -0,0 +1,40 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Gets expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnHardSigmoid : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnHardSigmoid), 0, "input"); + + /// + /// Gets axis of Ncnn Elu. + /// + public float Alpha { get; } + + /// + /// Gets axis of Ncnn Elu. + /// + public float Beta { get; } + + /// + public override string DisplayProperty() + { + return $""; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSigmoid.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSigmoid.cs new file mode 100644 index 0000000000..c61625de6e --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSigmoid.cs @@ -0,0 +1,47 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerHardSigmoid : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsHardSigmoid( + IsWildcard("input") with { TypePattern = IsFloat() }, + IsTensorConst("alpha") with { TypePattern = IsScalar() }, + IsTensorConst("beta") with { TypePattern = IsScalar() } + ); + + private Expr? GetReplace(Expr input, float alpha, float beta) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var hardSigmoid = new Call(new Fusion("ncnn", NcnnHardSigmoid(inResO, alpha, beta), new[] { inResO }), inRes); + + return Unsqueeze(hardSigmoid, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 4c27fc1cf9..c2681f768d 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -62,6 +62,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); // p.Add(); // need ncnn later than 20230908 + p.Add(); }); } From 2b61889b916f2751e7ccb51ebf3a1938aabb3b95 Mon Sep 17 00:00:00 2001 From: curioyang Date: Fri, 19 Jan 2024 06:08:06 +0000 Subject: [PATCH 044/132] Apply code-format changes --- .../Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSigmoid.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSigmoid.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSigmoid.cs index c61625de6e..cf5895600c 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSigmoid.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSigmoid.cs @@ -26,8 +26,7 @@ public partial class LowerHardSigmoid : RewriteRule public override Pattern Pattern { get; } = IsHardSigmoid( IsWildcard("input") with { TypePattern = IsFloat() }, IsTensorConst("alpha") with { TypePattern = IsScalar() }, - IsTensorConst("beta") with { TypePattern = IsScalar() } - ); + IsTensorConst("beta") with { TypePattern = IsScalar() }); private Expr? GetReplace(Expr input, float alpha, float beta) { From d60a98c41329630036dcc6f0a1f0b04f0d9209ef Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 19 Jan 2024 14:08:44 +0800 Subject: [PATCH 045/132] fix flops --- modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSigmoid.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSigmoid.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSigmoid.cs index 974d318c1f..4553de3068 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSigmoid.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSigmoid.cs @@ -53,7 +53,7 @@ public Metric Visit(IMetricEvaluateContext context, NcnnHardSigmoid target) return new() { [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, - [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.ExpFLOPs + MetricUtility.MulFLOPs + MetricUtility.SubFLOPs + MetricUtility.CmpFLOPs), + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.MulFLOPs + MetricUtility.AddFLOPs + MetricUtility.CmpFLOPs*2), [MetricFactorNames.Parallel] = 4, }; } From 7f07de02893f94d5a2fa05f3aaed7ae14b3cbf2c Mon Sep 17 00:00:00 2001 From: curioyang Date: Fri, 19 Jan 2024 06:10:58 +0000 Subject: [PATCH 046/132] Apply code-format changes --- modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSigmoid.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSigmoid.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSigmoid.cs index 4553de3068..2a3f4b9848 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSigmoid.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSigmoid.cs @@ -53,7 +53,7 @@ public Metric Visit(IMetricEvaluateContext context, NcnnHardSigmoid target) return new() { [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, - [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.MulFLOPs + MetricUtility.AddFLOPs + MetricUtility.CmpFLOPs*2), + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.MulFLOPs + MetricUtility.AddFLOPs + (MetricUtility.CmpFLOPs * 2)), [MetricFactorNames.Parallel] = 4, }; } From 3de9d7d19d039e0f8345d8abf2606dbeacd4d5ab Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 19 Jan 2024 14:29:00 +0800 Subject: [PATCH 047/132] HardSwish --> [ncnn op] HardSwish --- .../CodeGen/Ncnn/NcnnEmitter.cs | 7 ++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnHardSwish.cs | 65 +++++++++++++++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 1 + .../IR/Ncnn/NcnnHardSigmoid.cs | 6 +- .../IR/Ncnn/NcnnHardSwish.cs | 40 ++++++++++++ .../Passes/Rules/Ncnn/LowerHardSwish.cs | 44 +++++++++++++ .../Targets/CPUTarget.cs | 1 + 9 files changed, 165 insertions(+), 3 deletions(-) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSwish.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSwish.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSwish.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 2e6b480c66..35c849a1ee 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -174,6 +174,13 @@ public void HardSigmoid(string name, string input, float alpha, float beta) => [1] = new ParamValue { Kind = ParamKind.Float, FloatValue = beta }, // beta }); + public void HardSwish(string name, string input, float alpha, float beta) => + AddLayer("HardSwish", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Float, FloatValue = alpha }, // alpha + [1] = new ParamValue { Kind = ParamKind.Float, FloatValue = beta }, // beta + }); + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index abba5bcda4..ff59518fb4 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -140,6 +140,9 @@ protected override string VisitLeafCall(Call expr) case NcnnHardSigmoid op: _emitter.HardSigmoid(name, ExprMemo[expr.Arguments[0]], op.Alpha, op.Beta); break; + case NcnnHardSwish op: + _emitter.HardSwish(name, ExprMemo[expr.Arguments[0]], op.Alpha, op.Beta); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSwish.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSwish.cs new file mode 100644 index 0000000000..2470ce121e --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSwish.cs @@ -0,0 +1,65 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnHardSwishEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnHardSwish hardSwish) + { + var input = context.GetOrtArgumentValue(hardSwish, NcnnHardSwish.Input); + return OrtKI.HardSwish(input).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnHardSwish target) + { + var input = context.CheckArgumentType(target, NcnnHardSwish.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnHardSwish target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnHardSwish target) + { + var inputType = context.GetArgumentType(target, NcnnHardSwish.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.PowFLOPs + MetricUtility.MulFLOPs + MetricUtility.AddFLOPs + MetricUtility.CmpFLOPs * 2), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnHardSwish target) => context.GetArgumentShape(target, NcnnHardSwish.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 4cc898a3ec..3f59daf428 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -25,5 +25,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 749b311c95..ba6be4449a 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -44,4 +44,5 @@ public static Call NcnnConv(Expr expr, float[] weightsData, float[] biasData, in public static Call NcnnErf(Expr expr) => new Call(new NcnnErf(), expr); public static Call NcnnHardSigmoid(Expr expr, float alpha, float beta) => new Call(new NcnnHardSigmoid(alpha, beta), expr); + public static Call NcnnHardSwish(Expr expr, float alpha, float beta) => new Call(new NcnnHardSwish(alpha, beta), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSigmoid.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSigmoid.cs index 6646c0ce3e..2a3f6bd682 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSigmoid.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSigmoid.cs @@ -23,18 +23,18 @@ public sealed partial class NcnnHardSigmoid : Op public static readonly ParameterInfo Input = new(typeof(NcnnHardSigmoid), 0, "input"); /// - /// Gets axis of Ncnn Elu. + /// Gets axis of Ncnn HardSigmoid. /// public float Alpha { get; } /// - /// Gets axis of Ncnn Elu. + /// Gets axis of Ncnn HardSigmoid. /// public float Beta { get; } /// public override string DisplayProperty() { - return $""; + return $"alpha = {Alpha}, beta = {Beta}"; } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSwish.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSwish.cs new file mode 100644 index 0000000000..aee45da3de --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSwish.cs @@ -0,0 +1,40 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Gets expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnHardSwish : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnHardSwish), 0, "input"); + + /// + /// Gets axis of Ncnn HardSwish. + /// + public float Alpha { get; } + + /// + /// Gets axis of Ncnn HardSwish. + /// + public float Beta { get; } + + /// + public override string DisplayProperty() + { + return $"alpha = {Alpha}, beta = {Beta}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSwish.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSwish.cs new file mode 100644 index 0000000000..a307282a3b --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSwish.cs @@ -0,0 +1,44 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerHardSwish : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsHardSwish( + IsWildcard("input")); + + private Expr? GetReplace(Expr input) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var hardSwish = new Call(new Fusion("ncnn", NcnnHardSwish(inResO, 1.0f / 6, 0.5f), new[] { inResO }), inRes); + + return Unsqueeze(hardSwish, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index c2681f768d..6f5977247f 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -63,6 +63,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions // p.Add(); // need ncnn later than 20230908 p.Add(); + p.Add(); }); } From 4b47f9f8b0f0d6f886aab297b61f6f4de58d71f8 Mon Sep 17 00:00:00 2001 From: curioyang Date: Fri, 19 Jan 2024 06:31:12 +0000 Subject: [PATCH 048/132] Apply code-format changes --- modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSwish.cs | 2 +- modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSwish.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSwish.cs index 2470ce121e..a372b289b9 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSwish.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnHardSwish.cs @@ -51,7 +51,7 @@ public Metric Visit(IMetricEvaluateContext context, NcnnHardSwish target) return new() { [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, - [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.PowFLOPs + MetricUtility.MulFLOPs + MetricUtility.AddFLOPs + MetricUtility.CmpFLOPs * 2), + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.PowFLOPs + MetricUtility.MulFLOPs + MetricUtility.AddFLOPs + (MetricUtility.CmpFLOPs * 2)), [MetricFactorNames.Parallel] = 4, }; } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index ba6be4449a..6e28c3e23a 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -44,5 +44,6 @@ public static Call NcnnConv(Expr expr, float[] weightsData, float[] biasData, in public static Call NcnnErf(Expr expr) => new Call(new NcnnErf(), expr); public static Call NcnnHardSigmoid(Expr expr, float alpha, float beta) => new Call(new NcnnHardSigmoid(alpha, beta), expr); + public static Call NcnnHardSwish(Expr expr, float alpha, float beta) => new Call(new NcnnHardSwish(alpha, beta), expr); } From 1e67682c6acd8b4443c882d5546136498929d614 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 19 Jan 2024 16:46:20 +0800 Subject: [PATCH 049/132] instancenorm --> [ncnn op] instancenorm --- .../CodeGen/Ncnn/NcnnEmitter.cs | 12 +++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnInstanceNorm.cs | 73 +++++++++++++++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 2 + .../IR/Ncnn/NcnnInstanceNorm.cs | 55 ++++++++++++++ .../Passes/Rules/Ncnn/LowerInstanceNorm.cs | 49 +++++++++++++ .../Targets/CPUTarget.cs | 1 + 8 files changed, 196 insertions(+) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnInstanceNorm.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnInstanceNorm.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerInstanceNorm.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 35c849a1ee..1575df93c8 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -181,6 +181,18 @@ public void HardSwish(string name, string input, float alpha, float beta) => [1] = new ParamValue { Kind = ParamKind.Float, FloatValue = beta }, // beta }); + public void InstanceNorm(string name, string input, int channels, float eps, int affine, float[] gammaData, float[] betaData) + { + AddLayer("InstanceNorm", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = channels }, // channels + [1] = new ParamValue { Kind = ParamKind.Float, FloatValue = eps }, // eps + [2] = new ParamValue { Kind = ParamKind.Int, IntValue = affine }, // affine + }); + WriteFloatArray(gammaData); + WriteFloatArray(betaData); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index ff59518fb4..43434cc089 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -143,6 +143,9 @@ protected override string VisitLeafCall(Call expr) case NcnnHardSwish op: _emitter.HardSwish(name, ExprMemo[expr.Arguments[0]], op.Alpha, op.Beta); break; + case NcnnInstanceNorm op: + _emitter.InstanceNorm(name, ExprMemo[expr.Arguments[0]], op.Channels, op.Eps, op.Affine, op.GammaData, op.BetaData); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnInstanceNorm.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnInstanceNorm.cs new file mode 100644 index 0000000000..95d0529a60 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnInstanceNorm.cs @@ -0,0 +1,73 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnInstanceNormEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnInstanceNorm instanceNorm) + { + var input = context.GetOrtArgumentValue(instanceNorm, NcnnInstanceNorm.Input); + + return OrtKI.InstanceNormalization(input, instanceNorm.GammaData, instanceNorm.BetaData, instanceNorm.Eps).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnInstanceNorm target) + { + var input = context.CheckArgumentType(target, NcnnInstanceNorm.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnInstanceNorm target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnInstanceNorm target) + { + var inputType = context.GetArgumentType(target, NcnnInstanceNorm.Input); + var gamma = new TensorType(DataTypes.Float32, new[] { target.Channels }); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2 + CostUtility.GetMemoryAccess(gamma) * 2, + // x = (x - mean)/(standard_deviation) * gamma + beta; + // mean = sum(x)/N; + // standard-deviation = sqrt(sum(square(x-mean))/N); + [MetricFactorNames.FLOPs] = + MetricUtility.GetFLOPs(inputType) * (MetricUtility.AddFLOPs + MetricUtility.SubFLOPs) + MetricUtility.DivFLOPs + // x = x-mean + MetricUtility.GetFLOPs(inputType) * (MetricUtility.PowFLOPs + MetricUtility.AddFLOPs) + MetricUtility.SqrtFLOPs + MetricUtility.DivFLOPs + + MetricUtility.GetFLOPs(inputType) * (MetricUtility.DivFLOPs + MetricUtility.MulFLOPs + MetricUtility.AddFLOPs), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnInstanceNorm target) => context.GetArgumentShape(target, NcnnInstanceNorm.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 3f59daf428..9d411e7925 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -26,5 +26,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index ba6be4449a..c6b725fb77 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -45,4 +45,6 @@ public static Call NcnnConv(Expr expr, float[] weightsData, float[] biasData, in public static Call NcnnHardSigmoid(Expr expr, float alpha, float beta) => new Call(new NcnnHardSigmoid(alpha, beta), expr); public static Call NcnnHardSwish(Expr expr, float alpha, float beta) => new Call(new NcnnHardSwish(alpha, beta), expr); + public static Call NcnnInstanceNorm(Expr expr, int channels, float eps, int affine, float[] gammaData, float[] betaData) => + new Call(new NcnnInstanceNorm(channels, eps, affine, gammaData, betaData), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnInstanceNorm.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnInstanceNorm.cs new file mode 100644 index 0000000000..4eaad36c3f --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnInstanceNorm.cs @@ -0,0 +1,55 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Gets expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnInstanceNorm : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnInstanceNorm), 0, "input"); + + /// + /// Gets Channels of Ncnn InstanceNorm. + /// + public int Channels { get; } + + /// + /// Gets Eps of Ncnn InstanceNorm. + /// + public float Eps { get; } + + /// + /// Gets Affine of Ncnn InstanceNorm. + /// + public int Affine { get; } + + /// + /// Gets GammaData of InstanceNorm. + /// + public float[] GammaData { get; } + + /// + /// Gets BetaData of InstanceNorm. + /// + public float[] BetaData { get; } + + /// + public override string DisplayProperty() + { + return $"channels={Channels}, eps={Eps}, affine={Affine}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerInstanceNorm.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerInstanceNorm.cs new file mode 100644 index 0000000000..7a9ef57da6 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerInstanceNorm.cs @@ -0,0 +1,49 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerInstanceNorm : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsInstanceNormalization( + IsWildcard("input"), + IsTensorConst("gamma"), + IsTensorConst("beta"), + IsTensorConst("epsilon") with { TypePattern = IsFloatScalar() }); + + private Expr? GetReplace(Expr input, float[] gamma, float[] beta, float epsilon) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + int affine = gamma.All(a => a != 1) && beta.All(a => a != 0) ? 1 : 0; + + var instanceNorm = new Call(new Fusion("ncnn", NcnnInstanceNorm(inResO, gamma.Length, epsilon, affine, gamma, beta), new[] { inResO }), inRes); + + return Unsqueeze(instanceNorm, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 6f5977247f..10ab718938 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -64,6 +64,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions // p.Add(); // need ncnn later than 20230908 p.Add(); p.Add(); + p.Add(); }); } From 7cbdb14f106372cff595b29909d0a92d39b2cf8f Mon Sep 17 00:00:00 2001 From: curioyang Date: Fri, 19 Jan 2024 08:48:52 +0000 Subject: [PATCH 050/132] Apply code-format changes --- .../Evaluator/Ncnn/NcnnInstanceNorm.cs | 9 +++++---- modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnInstanceNorm.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnInstanceNorm.cs index 95d0529a60..28ca42e222 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnInstanceNorm.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnInstanceNorm.cs @@ -52,14 +52,15 @@ public Metric Visit(IMetricEvaluateContext context, NcnnInstanceNorm target) return new() { - [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2 + CostUtility.GetMemoryAccess(gamma) * 2, + [MetricFactorNames.OffChipMemoryTraffic] = (CostUtility.GetMemoryAccess(returnType) * 2) + (CostUtility.GetMemoryAccess(gamma) * 2), + // x = (x - mean)/(standard_deviation) * gamma + beta; // mean = sum(x)/N; // standard-deviation = sqrt(sum(square(x-mean))/N); [MetricFactorNames.FLOPs] = - MetricUtility.GetFLOPs(inputType) * (MetricUtility.AddFLOPs + MetricUtility.SubFLOPs) + MetricUtility.DivFLOPs + // x = x-mean - MetricUtility.GetFLOPs(inputType) * (MetricUtility.PowFLOPs + MetricUtility.AddFLOPs) + MetricUtility.SqrtFLOPs + MetricUtility.DivFLOPs + - MetricUtility.GetFLOPs(inputType) * (MetricUtility.DivFLOPs + MetricUtility.MulFLOPs + MetricUtility.AddFLOPs), + (MetricUtility.GetFLOPs(inputType) * (MetricUtility.AddFLOPs + MetricUtility.SubFLOPs)) + MetricUtility.DivFLOPs + // x = x-mean + (MetricUtility.GetFLOPs(inputType) * (MetricUtility.PowFLOPs + MetricUtility.AddFLOPs)) + MetricUtility.SqrtFLOPs + MetricUtility.DivFLOPs + + (MetricUtility.GetFLOPs(inputType) * (MetricUtility.DivFLOPs + MetricUtility.MulFLOPs + MetricUtility.AddFLOPs)), [MetricFactorNames.Parallel] = 4, }; } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 0bd9793e2c..74acae044e 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -46,6 +46,7 @@ public static Call NcnnConv(Expr expr, float[] weightsData, float[] biasData, in public static Call NcnnHardSigmoid(Expr expr, float alpha, float beta) => new Call(new NcnnHardSigmoid(alpha, beta), expr); public static Call NcnnHardSwish(Expr expr, float alpha, float beta) => new Call(new NcnnHardSwish(alpha, beta), expr); + public static Call NcnnInstanceNorm(Expr expr, int channels, float eps, int affine, float[] gammaData, float[] betaData) => new Call(new NcnnInstanceNorm(channels, eps, affine, gammaData, betaData), expr); } From 3f9120d98cf3a6e3c7af877dc36c51247caca8aa Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 19 Jan 2024 16:50:46 +0800 Subject: [PATCH 051/132] fix flops --- .../Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnInstanceNorm.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnInstanceNorm.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnInstanceNorm.cs index 28ca42e222..ee54186dba 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnInstanceNorm.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnInstanceNorm.cs @@ -56,10 +56,10 @@ public Metric Visit(IMetricEvaluateContext context, NcnnInstanceNorm target) // x = (x - mean)/(standard_deviation) * gamma + beta; // mean = sum(x)/N; - // standard-deviation = sqrt(sum(square(x-mean))/N); + // standard-deviation = sqrt(sum(square(x-mean))/N + eps); [MetricFactorNames.FLOPs] = (MetricUtility.GetFLOPs(inputType) * (MetricUtility.AddFLOPs + MetricUtility.SubFLOPs)) + MetricUtility.DivFLOPs + // x = x-mean - (MetricUtility.GetFLOPs(inputType) * (MetricUtility.PowFLOPs + MetricUtility.AddFLOPs)) + MetricUtility.SqrtFLOPs + MetricUtility.DivFLOPs + + (MetricUtility.GetFLOPs(inputType) * (MetricUtility.PowFLOPs + MetricUtility.AddFLOPs)) + MetricUtility.SqrtFLOPs + MetricUtility.DivFLOPs + MetricUtility.AddFLOPs + (MetricUtility.GetFLOPs(inputType) * (MetricUtility.DivFLOPs + MetricUtility.MulFLOPs + MetricUtility.AddFLOPs)), [MetricFactorNames.Parallel] = 4, }; From c571942761285a6cdcec23ae7e9d497fedaa32d4 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Wed, 24 Jan 2024 10:22:11 +0800 Subject: [PATCH 052/132] stash layernorm --- .../CodeGen/Ncnn/NcnnEmitter.cs | 12 +++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 5 +- .../Evaluator/Ncnn/NcnnLayernorm.cs | 74 ++++++++++++++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../IR/Ncnn/NcnnLayerNorm.cs | 55 ++++++++++++++ .../Passes/Rules/Ncnn/LowerLayernorm.cs | 75 +++++++++++++++++++ .../Targets/CPUTarget.cs | 1 + 8 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLayernorm.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLayerNorm.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLayernorm.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 1575df93c8..d0b6527a9e 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -193,6 +193,18 @@ public void InstanceNorm(string name, string input, int channels, float eps, int WriteFloatArray(betaData); } + public void LayerNorm(string name, string input, int affineSize, float eps, int affine, float[] gammaData, float[] betaData) + { + AddLayer("LayerNorm", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = affineSize }, // affineSize + [1] = new ParamValue { Kind = ParamKind.Float, FloatValue = eps }, // eps + [2] = new ParamValue { Kind = ParamKind.Int, IntValue = affine }, // affine + }); + WriteFloatArray(gammaData); + WriteFloatArray(betaData); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 43434cc089..954e23bcf3 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -146,8 +146,11 @@ protected override string VisitLeafCall(Call expr) case NcnnInstanceNorm op: _emitter.InstanceNorm(name, ExprMemo[expr.Arguments[0]], op.Channels, op.Eps, op.Affine, op.GammaData, op.BetaData); break; + case NcnnLayerNorm op: + _emitter.LayerNorm(name, ExprMemo[expr.Arguments[0]], op.AffineSize, op.Eps, op.Affine, op.GammaData, op.BetaData); + break; default: - throw new NotSupportedException(); + throw new NotSupportedException("Not support in Ncnn ops emitter"); } return name; diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLayernorm.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLayernorm.cs new file mode 100644 index 0000000000..219228c94a --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLayernorm.cs @@ -0,0 +1,74 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnLayerNormEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnLayerNorm layerNorm) + { + var input = context.GetOrtArgumentValue(layerNorm, NcnnLayerNorm.Input); + + return OrtKI.LayerNormalization(input, layerNorm.GammaData, layerNorm.BetaData, layerNorm.AffineSize, layerNorm.Eps, 1L)[0].ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnLayerNorm target) + { + var input = context.CheckArgumentType(target, NcnnLayerNorm.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnLayerNorm target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnLayerNorm target) + { + var inputType = context.GetArgumentType(target, NcnnLayerNorm.Input); + var gamma = new TensorType(DataTypes.Float32, new[] { target.AffineSize }); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = (CostUtility.GetMemoryAccess(returnType) * 2) + (CostUtility.GetMemoryAccess(gamma) * 2), + + // x = (x - mean)/(standard_deviation) * gamma + beta; + // mean = sum(x)/N; + // standard-deviation = sqrt(sum(square(x-mean))/N + eps); + [MetricFactorNames.FLOPs] = + (MetricUtility.GetFLOPs(inputType) * (MetricUtility.AddFLOPs + MetricUtility.SubFLOPs)) + MetricUtility.DivFLOPs + // x = x-mean + (MetricUtility.GetFLOPs(inputType) * (MetricUtility.PowFLOPs + MetricUtility.AddFLOPs)) + MetricUtility.SqrtFLOPs + MetricUtility.DivFLOPs + MetricUtility.AddFLOPs + + (MetricUtility.GetFLOPs(inputType) * (MetricUtility.DivFLOPs + MetricUtility.MulFLOPs + MetricUtility.AddFLOPs)), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnLayerNorm target) => context.GetArgumentShape(target, NcnnLayerNorm.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 9d411e7925..710acd71e5 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -27,5 +27,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 74acae044e..ea7b3b387d 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -49,4 +49,7 @@ public static Call NcnnConv(Expr expr, float[] weightsData, float[] biasData, in public static Call NcnnInstanceNorm(Expr expr, int channels, float eps, int affine, float[] gammaData, float[] betaData) => new Call(new NcnnInstanceNorm(channels, eps, affine, gammaData, betaData), expr); + + public static Call NcnnLayerNorm(Expr expr, int affineSize, float eps, int affine, float[] gammaData, float[] betaData) => + new Call(new NcnnLayerNorm(affineSize, eps, affine, gammaData, betaData), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLayerNorm.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLayerNorm.cs new file mode 100644 index 0000000000..e85854c46a --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLayerNorm.cs @@ -0,0 +1,55 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Gets expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnLayerNorm : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnLayerNorm), 0, "input"); + + /// + /// Gets AffineSize of Ncnn LayerNorm. + /// + public int AffineSize { get; } + + /// + /// Gets Eps of Ncnn LayerNorm. + /// + public float Eps { get; } + + /// + /// Gets Affine of Ncnn LayerNorm. + /// + public int Affine { get; } + + /// + /// Gets GammaData of LayerNorm. + /// + public float[] GammaData { get; } + + /// + /// Gets BetaData of LayerNorm. + /// + public float[] BetaData { get; } + + /// + public override string DisplayProperty() + { + return $"AffineSize={AffineSize}, eps={Eps}, affine={Affine}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLayernorm.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLayernorm.cs new file mode 100644 index 0000000000..ce40849701 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLayernorm.cs @@ -0,0 +1,75 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.IR.NN; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; +using static Nncase.Utilities.ShapeUtility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerLayerNorm : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsLayerNorm( + "op", + _ => true, + IsWildcard("input"), + IsTensorConst("gamma"), + IsTensorConst("beta")); + + private Expr? GetReplace(LayerNorm op, Expr input, float[] gamma, float[] beta) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + // var newShape = FitNcnnShape(input.CheckedShape.ToValueList(), op.Axis); // newAxis = 1; + // Console.WriteLine($"op.Axis: {op.Axis}"); + // Console.WriteLine($"oldShape: {input.CheckedShape.ToValueList().Select(x => x.ToString()).Aggregate((x, y) => x + ", " + y)}"); + // Console.WriteLine($"newShape: {newShape.Select(x => x.ToString()).Aggregate((x, y) => x + "," + y)}"); + // var inRes = Reshape(input, newShape.ToArray()); + // var inResO = new Var(inRes.CheckedType); + + // int affine = gamma.All(a => a != 1) && beta.All(a => a != 0) ? 1 : 0; + // int affineSize = newShape[2]; + + // var layerNorm = new Call(new Fusion("ncnn", NcnnLayerNorm(inResO, affineSize, op.Epsilon, affine, gamma, beta), new[] { inResO }), inRes); + + // return Reshape(layerNorm, input.CheckedShape); + + // var newShape = FitNcnnShape(input.CheckedShape.ToValueList(), op.Axis); // newAxis = 1; + // Console.WriteLine($"op.Axis: {op.Axis}"); + // Console.WriteLine($"oldShape: {input.CheckedShape.ToValueList().Select(x => x.ToString()).Aggregate((x, y) => x + ", " + y)}"); + // Console.WriteLine($"newShape: {newShape.Select(x => x.ToString()).Aggregate((x, y) => x + "," + y)}"); + var inRes = Squeeze(input, new[]{0}); + var inResO = new Var(inRes.CheckedType); + + int affine = gamma.All(a => a != 1) && beta.All(a => a != 0) ? 1 : 0; + int affineSize = 1; + for(int i = 1; i < inRes.CheckedShape.Rank; i++) + { + affineSize *= inRes.CheckedShape.ToValueArray()[i]; + } + + var layerNorm = new Call(new Fusion("ncnn", NcnnLayerNorm(inResO, affineSize, op.Epsilon, affine, gamma, beta), new[] { inResO }), inRes); + + return Unsqueeze(layerNorm, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 10ab718938..5dbc54f8e6 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -65,6 +65,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); }); } From 1dfb844bf6a7512f29dc4eb45792e6d3a11f9906 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Wed, 24 Jan 2024 10:44:30 +0800 Subject: [PATCH 053/132] LRN -->[ncnn op] LRN --- .../CodeGen/Ncnn/NcnnEmitter.cs | 10 +++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnLrn.cs | 69 +++++++++++++++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 2 + .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnLrn.cs | 50 ++++++++++++++ .../Passes/Rules/Ncnn/LowerLrn.cs | 48 +++++++++++++ .../Targets/CPUTarget.cs | 1 + 8 files changed, 184 insertions(+) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLrn.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLrn.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLrn.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 1575df93c8..26d59f9914 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -193,6 +193,16 @@ public void InstanceNorm(string name, string input, int channels, float eps, int WriteFloatArray(betaData); } + public void LRN(string name, string input, float alpha, float beta, float bias, int size) => + AddLayer("LRN", name, new[] { input }, new[] { name }, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = 0 }, // region_type + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = size }, // size + [2] = new ParamValue { Kind = ParamKind.Float, FloatValue = alpha }, // alpha + [3] = new ParamValue { Kind = ParamKind.Float, FloatValue = beta }, // beta + [4] = new ParamValue { Kind = ParamKind.Float, FloatValue = bias }, // bias + }); + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 43434cc089..6052ceaf6c 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -146,6 +146,9 @@ protected override string VisitLeafCall(Call expr) case NcnnInstanceNorm op: _emitter.InstanceNorm(name, ExprMemo[expr.Arguments[0]], op.Channels, op.Eps, op.Affine, op.GammaData, op.BetaData); break; + case NcnnLRN op: + _emitter.LRN(name, ExprMemo[expr.Arguments[0]], op.Alpha, op.Beta, op.Bias, op.Size); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLrn.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLrn.cs new file mode 100644 index 0000000000..592a148dee --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLrn.cs @@ -0,0 +1,69 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnLRNEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnLRN lrn) + { + var input = context.GetOrtArgumentValue(lrn, NcnnLRN.Input); + var alpha = lrn.Alpha; + var beta = lrn.Beta; + var bias = lrn.Bias; + var size = lrn.Size; + return OrtKI.LRN(input, alpha, beta, bias, size).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnLRN target) + { + var input = context.CheckArgumentType(target, NcnnLRN.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnLRN target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnLRN target) + { + var inputType = context.GetArgumentType(target, NcnnLRN.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.ExpFLOPs + MetricUtility.MulFLOPs + MetricUtility.SubFLOPs + MetricUtility.CmpFLOPs), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnLRN target) => context.GetArgumentShape(target, NcnnLRN.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 9d411e7925..268a945394 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -27,5 +27,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 74acae044e..90a8d6edd0 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -49,4 +49,6 @@ public static Call NcnnConv(Expr expr, float[] weightsData, float[] biasData, in public static Call NcnnInstanceNorm(Expr expr, int channels, float eps, int affine, float[] gammaData, float[] betaData) => new Call(new NcnnInstanceNorm(channels, eps, affine, gammaData, betaData), expr); + + public static Call NcnnLRN(Expr expr, float alpha, float beta, float bias, int size) => new Call(new NcnnLRN(alpha, beta, bias, size), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLrn.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLrn.cs new file mode 100644 index 0000000000..baa57b8b63 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLrn.cs @@ -0,0 +1,50 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Gets expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnLRN : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnLRN), 0, "input"); + + /// + /// Gets alpha of Ncnn LRN. + /// + public float Alpha { get; } + + /// + /// Gets Beta of Ncnn LRN. + /// + public float Beta { get; } + + /// + /// Gets Bias of Ncnn LRN. + /// + public float Bias { get; } + + /// + /// Gets Size of Ncnn LRN. + /// + public int Size { get; } + + /// + public override string DisplayProperty() + { + return $"alpha={Alpha}, beta={Beta}, bias={Bias}, size={Size}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLrn.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLrn.cs new file mode 100644 index 0000000000..e735f9bef0 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLrn.cs @@ -0,0 +1,48 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerLRN : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsLRN( + IsWildcard("input"), + IsTensorConst("alpha") with { TypePattern = IsScalar() }, + IsTensorConst("beta") with { TypePattern = IsScalar() }, + IsTensorConst("bias") with { TypePattern = IsScalar() }, + IsTensorConst("size") with { TypePattern = IsIntegralScalar() }); + + private Expr? GetReplace(Expr input, float alpha, float beta, float bias, int size) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var lrn = new Call(new Fusion("ncnn", NcnnLRN(inResO, alpha, beta, bias, size), new[] { inResO }), inRes); + + return Unsqueeze(lrn, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 10ab718938..cdc38d8988 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -65,6 +65,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); }); } From 3c18eee21d789d0067af27d6637181f1e223e868 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 1 Feb 2024 15:41:41 +0800 Subject: [PATCH 054/132] fix dump manager --- src/Native/src/runtime/dump_manager.cpp | 5 +++-- src/Native/src/runtime/dump_manager_impl.h | 14 ++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Native/src/runtime/dump_manager.cpp b/src/Native/src/runtime/dump_manager.cpp index 8a54d96ce4..d1a7002d74 100644 --- a/src/Native/src/runtime/dump_manager.cpp +++ b/src/Native/src/runtime/dump_manager.cpp @@ -81,7 +81,7 @@ void dump_output_impl(dump_manager &dump_manager_, nncase::value_t value, dump_data(stream, IN_CAST(_in_type, data), value_tensor); \ return; \ } - + auto stream = dump_manager_.get_stream(path); dump( dump_manager_, value, [incr, &dump_manager_](auto &stream, auto &&value_tensor) { @@ -102,7 +102,8 @@ void dump_output_impl(dump_manager &dump_manager_, nncase::value_t value, // << (int)value_tensor->dtype()->typecode() << // std::endl; }, - path); + stream); + stream.close(); if (incr) { dump_manager_.incr_count(); } diff --git a/src/Native/src/runtime/dump_manager_impl.h b/src/Native/src/runtime/dump_manager_impl.h index 2a33f6b999..b6caed600e 100644 --- a/src/Native/src/runtime/dump_manager_impl.h +++ b/src/Native/src/runtime/dump_manager_impl.h @@ -28,29 +28,30 @@ void dump_append(dump_manager &dump_manager_, F &&f, const std::string &path) { template void dump(dump_manager &dump_manager_, nncase::value_t value, F &&f) { - dump(dump_manager_, value, f, dump_manager_.dump_path()); + auto stream = dump_manager_.get_stream(dump_manager_.dump_path()); + dump(dump_manager_, value, f, stream); + stream.close(); } template void dump(dump_manager &dump_manager_, nncase::value_t value, F &&f, - const std::string &path) { - auto stream = dump_manager_.get_stream(path); + std::ofstream &stream) { + if (value.is_a()) { auto value_tensor = value.as().unwrap(); f(stream, value_tensor); - stream.close(); } else if (value.is_a()) { // stream << "tuple" << "\n"; - stream.close(); auto value_tuple = value.as().unwrap(); for (auto &field : value_tuple->fields()) { - dump(dump_manager_, field, f, path); + dump(dump_manager_, field, f, stream); } } else { std::cout << "unknown in dump" << std::endl; stream << "unknown in dump\n"; return; } + } inline std::string to_str(const nncase::dims_t &shape) { @@ -91,6 +92,7 @@ void dump_data(std::ostream &stream, const T *data, stream << std::to_string(data[i]) << "\n"; } } + stream <<"------------------\n"<< std::endl; } END_NS_NNCASE_RUNTIME \ No newline at end of file From 74bcf7f51f309d825e6c9967ba6571e66f0c5cb9 Mon Sep 17 00:00:00 2001 From: curioyang Date: Thu, 1 Feb 2024 07:43:49 +0000 Subject: [PATCH 055/132] Apply code-format changes --- src/Native/src/runtime/dump_manager_impl.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Native/src/runtime/dump_manager_impl.h b/src/Native/src/runtime/dump_manager_impl.h index b6caed600e..a5be0f304a 100644 --- a/src/Native/src/runtime/dump_manager_impl.h +++ b/src/Native/src/runtime/dump_manager_impl.h @@ -51,7 +51,6 @@ void dump(dump_manager &dump_manager_, nncase::value_t value, F &&f, stream << "unknown in dump\n"; return; } - } inline std::string to_str(const nncase::dims_t &shape) { @@ -92,7 +91,7 @@ void dump_data(std::ostream &stream, const T *data, stream << std::to_string(data[i]) << "\n"; } } - stream <<"------------------\n"<< std::endl; + stream << "------------------\n" << std::endl; } END_NS_NNCASE_RUNTIME \ No newline at end of file From 7caf3019c08b7791282de8e89139148e0b0e24ec Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 1 Feb 2024 16:11:07 +0800 Subject: [PATCH 056/132] LSTM -->[ncnn op] LSTM --- .../CodeGen/Ncnn/NcnnEmitter.cs | 16 +++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 48 ++++--- .../Evaluator/Ncnn/NcnnLSTM.cs | 134 ++++++++++++++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnLSTM.cs | 65 +++++++++ .../Passes/Rules/Ncnn/LowerLSTM.cs | 131 +++++++++++++++++ .../Targets/CPUTarget.cs | 2 + 8 files changed, 383 insertions(+), 17 deletions(-) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLSTM.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLSTM.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLSTM.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 26d59f9914..3d717f9abe 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -203,6 +203,22 @@ public void LRN(string name, string input, float alpha, float beta, float bias, [4] = new ParamValue { Kind = ParamKind.Float, FloatValue = bias }, // bias }); + public void LSTM(string[] name, string input, int hiddenSize, int weightDataSize, int direction, float[] w, float[] r, float[] b) + { + AddLayer("LSTM", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = hiddenSize }, + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = weightDataSize }, + [2] = new ParamValue { Kind = ParamKind.Int, IntValue = direction }, + }); + WriteFloatArray(new float[] { 0 }); + WriteFloatArray(w); + WriteFloatArray(new float[] { 0 }); + WriteFloatArray(b); + WriteFloatArray(new float[] { 0 }); + WriteFloatArray(r); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 6052ceaf6c..6479161e44 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -8,6 +8,7 @@ using System.Reactive; using System.Text; using System.Threading.Tasks; +using Microsoft.Toolkit.HighPerformance; using NetFabric.Hyperlinq; using Nncase.Diagnostics; using Nncase.IR; @@ -70,6 +71,7 @@ public CodeGenVisitor(NcnnEmitter emitter) } public List Inputs { get; } = new(); + public List Outputs { get; } = new(); protected override string VisitLeafVar(Var expr) { @@ -87,17 +89,17 @@ protected override string VisitLeafVar(Var expr) protected override string VisitLeafCall(Call expr) { - var name = GetNextName(); + var names = new List { GetNextName() }; switch (expr.Target) { case NcnnSoftmax op: - _emitter.Softmax(name, ExprMemo[expr.Arguments[0]], op.Axis); + _emitter.Softmax(names[0], ExprMemo[expr.Arguments[0]], op.Axis); break; case NcnnUnary op: - _emitter.Unary(name, ExprMemo[expr.Arguments[0]], op.OpType); + _emitter.Unary(names[0], ExprMemo[expr.Arguments[0]], op.OpType); break; case NcnnBatchNorm op: - _emitter.BatchNorm(name, ExprMemo[expr.Arguments[0]], op.Channels, op.Eps, op.SlopeData, op.MeanData, op.VarData, op.BiasData); + _emitter.BatchNorm(names[0], ExprMemo[expr.Arguments[0]], op.Channels, op.Eps, op.SlopeData, op.MeanData, op.VarData, op.BiasData); break; case NcnnBinary op: string[]? inString = op.LorR switch @@ -107,13 +109,13 @@ protected override string VisitLeafCall(Call expr) 2 => new string[] { ExprMemo[expr.Arguments[0]], string.Empty }, _ => throw new NotImplementedException("Not found binary emmiter."), }; - _emitter.Binary(name, inString[0], inString[1], op.OpType, op.LorR, op.ConstInput, op.ConstShape); + _emitter.Binary(names[0], inString[0], inString[1], op.OpType, op.LorR, op.ConstInput, op.ConstShape); break; case NcnnCelu op: - _emitter.Celu(name, ExprMemo[expr.Arguments[0]], op.Alpha); + _emitter.Celu(names[0], ExprMemo[expr.Arguments[0]], op.Alpha); break; case NcnnClip op: - _emitter.Clip(name, ExprMemo[expr.Arguments[0]], op.Min, op.Max); + _emitter.Clip(names[0], ExprMemo[expr.Arguments[0]], op.Min, op.Max); break; case NcnnConcat op: List in_ = new(); @@ -123,37 +125,49 @@ protected override string VisitLeafCall(Call expr) in_.Add(ExprMemo[t.Fields[i]]); } - _emitter.Concat(name, in_.ToArray(), op.Axis); + _emitter.Concat(names[0], in_.ToArray(), op.Axis); break; case NcnnConv op: - _emitter.Conv(name, ExprMemo[expr.Arguments[0]], op.WeightData, op.BiasData, op.NumOutput, op.KernelW, op.KernelH, op.DilationW, op.DilationH, op.StrideW, op.StrideH, op.PadLeft, op.PadTop, op.PadRight, op.PadBottom, op.BiasTerm, op.WeightDataSize, op.Int8ScaleTerm, op.ActivationType, op.ActivationParams, op.PadValue, op.DynamicWeight); + _emitter.Conv(names[0], ExprMemo[expr.Arguments[0]], op.WeightData, op.BiasData, op.NumOutput, op.KernelW, op.KernelH, op.DilationW, op.DilationH, op.StrideW, op.StrideH, op.PadLeft, op.PadTop, op.PadRight, op.PadBottom, op.BiasTerm, op.WeightDataSize, op.Int8ScaleTerm, op.ActivationType, op.ActivationParams, op.PadValue, op.DynamicWeight); break; case NcnnCumsum op: - _emitter.Cumsum(name, ExprMemo[expr.Arguments[0]], op.Axis); + _emitter.Cumsum(names[0], ExprMemo[expr.Arguments[0]], op.Axis); break; case NcnnElu op: - _emitter.Elu(name, ExprMemo[expr.Arguments[0]], op.Alpha); + _emitter.Elu(names[0], ExprMemo[expr.Arguments[0]], op.Alpha); break; case NcnnErf: - _emitter.Erf(name, ExprMemo[expr.Arguments[0]]); + _emitter.Erf(names[0], ExprMemo[expr.Arguments[0]]); break; case NcnnHardSigmoid op: - _emitter.HardSigmoid(name, ExprMemo[expr.Arguments[0]], op.Alpha, op.Beta); + _emitter.HardSigmoid(names[0], ExprMemo[expr.Arguments[0]], op.Alpha, op.Beta); break; case NcnnHardSwish op: - _emitter.HardSwish(name, ExprMemo[expr.Arguments[0]], op.Alpha, op.Beta); + _emitter.HardSwish(names[0], ExprMemo[expr.Arguments[0]], op.Alpha, op.Beta); break; case NcnnInstanceNorm op: - _emitter.InstanceNorm(name, ExprMemo[expr.Arguments[0]], op.Channels, op.Eps, op.Affine, op.GammaData, op.BetaData); + _emitter.InstanceNorm(names[0], ExprMemo[expr.Arguments[0]], op.Channels, op.Eps, op.Affine, op.GammaData, op.BetaData); break; case NcnnLRN op: - _emitter.LRN(name, ExprMemo[expr.Arguments[0]], op.Alpha, op.Beta, op.Bias, op.Size); + _emitter.LRN(names[0], ExprMemo[expr.Arguments[0]], op.Alpha, op.Beta, op.Bias, op.Size); + break; + case NcnnLSTM op: + for (int i = 1; i < op.OutputSize; i++) + { + var a = GetNextName(); + Console.WriteLine($"{i} = {a}"); + names.Add(a); + } + + _emitter.LSTM(names.ToArray(), ExprMemo[expr.Arguments[0]], op.HiddenSize, op.WeightDataSize, op.Direction, op.W, op.R, op.B); break; default: throw new NotSupportedException(); } - return name; + // serialize outputs to string. + string output = string.Join(",", names.Select(x => x.ToString())); + return output; } private string GetNextName() => $"layer{_layerId++}"; diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLSTM.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLSTM.cs new file mode 100644 index 0000000000..7f0d7dc54c --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLSTM.cs @@ -0,0 +1,134 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.RNN; +using Nncase.IR.Ncnn; +using OrtKISharp; +using static Nncase.LSTMHelper; +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnLSTMEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnLSTM lstm) + { + var input = context.GetOrtArgumentValue(lstm, NcnnLSTM.Input); + // TODO: reorder w,r to iofg, b to numDirection-8*hiddensize (Now, numDirection-4*hiddensize (bw+br)) + var w = lstm.W; + var r = lstm.R; + var b = lstm.B; + var direction_ = lstm.Direction > 1 ? 2 : 1; + var mode = direction_ switch + { + 0 => LSTMDirection.Forward, + 1 => LSTMDirection.Reverse, + 2 => LSTMDirection.Bidirectional, + _ => throw new NotImplementedException(), + }; + + var seqLens = (int)input.Shape[0]; + var initH = Tensor.Zeros(new[] { direction_, seqLens, lstm.HiddenSize}); + var initC = Tensor.Zeros(new[] { direction_, seqLens, lstm.HiddenSize }); + var p = Tensor.Zeros(new[] { 1, seqLens, lstm.HiddenSize }); + var actAlpha = new[] { 0.0f }; + var actBeta = new[] { 0.0f }; + var clip = float.NaN; + var hiddenSize = lstm.HiddenSize; + var inputForget = 1; + var outputSize = 1; + var result = OrtKI.LSTM(input, w, r, b, seqLens, initH.ToOrtTensor(), initC.ToOrtTensor(), p.ToOrtTensor(), actAlpha, actBeta, new string[] { "sigmoid", "sigmoid", "tanh" }, clip, LSTMDirectionToValue(mode), hiddenSize, inputForget, 0, !clip.Equals(float.NaN), outputSize); + return Value.FromTensors(result.Select(t => t.ToTensor()).ToArray()); + + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnLSTM target) + { + var input = context.CheckArgumentType(target, NcnnLSTM.Input); + // [num_direction, batch_size:1, hidden_size] + var initH = new TensorType(DataTypes.Float32, new[] { target.Direction > 1 ? 2 : 1, 1, target.HiddenSize }); + var initC = new TensorType(DataTypes.Float32, new[] { target.Direction > 1 ? 2 : 1, 1, target.HiddenSize }); + if (input.Shape.Rank != 3) + { + // TODO: confirm ncnn input dims when direction not 1. + input.Shape.InsertAndClone(1, 1); + } + + return Visit(context, input, initH, initC, target.OutputSize); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnLSTM target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnLSTM target_) + { + var direction_ = target_.Direction > 1 ? 2 : 1; + + var xType = context.GetArgumentType(target_, NcnnLSTM.Input); + var wType = new TensorType(DataTypes.Float32, new[] { direction_, 4 * target_.HiddenSize, xType.Shape[^1] }); + var rType = new TensorType(DataTypes.Float32, new[] { direction_, 4 * target_.HiddenSize, target_.HiddenSize }); + var bType = new TensorType(DataTypes.Float32, new[] { direction_, 8 * target_.HiddenSize }); + var returnType = context.GetReturnType(); + var outputYType = (TensorType)returnType[0]; + var outputYShape = outputYType.Shape.ToValueArray().Select(s => (UInt128)s).ToArray(); + var (sequence_len, num_directions, batch_size, hidden_size) = (outputYShape[0], outputYShape[1], outputYShape[2], outputYShape[3]); + var embbeding_size = (UInt128)xType.Shape[^1].FixedValue; + + var flops = num_directions * batch_size * sequence_len * ( + MetricUtility.GetMatMulFLOPs(1, 4 * hidden_size, embbeding_size) + + // [1,embbeding_size] @ [embbeding_size, 4 * hidden_size] + + MetricUtility.GetMatMulFLOPs(1, 4 * hidden_size, hidden_size) // [1,hidden_size] @ [hidden_size, 4 * hidden_size] + + (4 * hidden_size) + + (MetricUtility.SigmoidFLOPs * hidden_size) // ft = sigmoid(g[2]) + + hidden_size // ct = init_c * ft + + (MetricUtility.SigmoidFLOPs * hidden_size) // it = sigmoid(g[0]) + + (MetricUtility.TanhFLOPs * hidden_size) // c_t = tanh(g[3]) + + hidden_size // c_t_it = it * c_t + + hidden_size // ct = ct + c_t_it + + (MetricUtility.SigmoidFLOPs * hidden_size) // ot = sigmoid(g[1]) + + (MetricUtility.TanhFLOPs * hidden_size) // tanh_ct = tanh(ct_o) + + hidden_size); // ht = tanh_ct * ot + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(xType) + CostUtility.GetMemoryAccess(wType) + CostUtility.GetMemoryAccess(rType) + CostUtility.GetMemoryAccess(bType) + returnType.Select(t => t switch + { + TensorType tensorType => CostUtility.GetMemoryAccess(tensorType), + _ => UInt128.One, + }).Sum(), + [MetricFactorNames.FLOPs] = flops, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnLSTM target) => context.GetArgumentShape(target, NcnnLSTM.Input); + + private IRType Visit(ITypeInferenceContext context, TensorType x, TensorType initH, TensorType initC, int outputSize) + { + // TODO: confirm ncnn output + var numDirections = initH.Shape[0]; + var seqLenIndex = 0; + var yType = new TensorType(DataTypes.Float32, new[] { x.Shape[0].FixedValue, initH.Shape[2].FixedValue }); + Console.WriteLine($"x.Shape[0].FixedValue: {x.Shape[0].FixedValue}, initH.Shape[2].FixedValue:{initH.Shape[2].FixedValue}"); + var result = new[] { yType, initH, initC }; + return new TupleType(result[..outputSize]); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 268a945394..b58b979e5e 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -28,5 +28,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 90a8d6edd0..566b5a1ecd 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -51,4 +51,7 @@ public static Call NcnnInstanceNorm(Expr expr, int channels, float eps, int affi new Call(new NcnnInstanceNorm(channels, eps, affine, gammaData, betaData), expr); public static Call NcnnLRN(Expr expr, float alpha, float beta, float bias, int size) => new Call(new NcnnLRN(alpha, beta, bias, size), expr); + + public static Call NcnnLSTM(Expr expr, int outputSize, int hiddenSize, int weightDataSize, int direction, float[] w, float[] b, float[] r) => + new Call(new NcnnLSTM(outputSize, hiddenSize, weightDataSize, direction, w, b, r), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLSTM.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLSTM.cs new file mode 100644 index 0000000000..d33df75b7e --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnLSTM.cs @@ -0,0 +1,65 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Gets expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnLSTM : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnLSTM), 0, "input"); + + /// + /// Gets OutputSize of Ncnn LSTM. + /// + public int OutputSize { get; } + + /// + /// Gets HiddenSize of Ncnn LSTM. + /// + public int HiddenSize { get; } + + /// + /// Gets WeightDataSize of Ncnn LSTM. + /// + public int WeightDataSize { get; } + + /// + /// Gets Direction of Ncnn LSTM. + /// + public int Direction { get; } + + /// + /// Gets W of Ncnn LSTM. + /// + public float[] W { get; } + + /// + /// Gets B of Ncnn LSTM. + /// + public float[] B { get; } + + /// + /// Gets R of Ncnn LSTM. + /// + public float[] R { get; } + + /// + public override string DisplayProperty() + { + return $"HiddenSize: {HiddenSize}, WeightDataSize: {WeightDataSize}, Direction: {Direction}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLSTM.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLSTM.cs new file mode 100644 index 0000000000..166c399e5d --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLSTM.cs @@ -0,0 +1,131 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.IR.RNN; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.RNN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerLSTM : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsLSTM( + target_name: "lstm", + call_name: "call", + _ => true, + IsWildcard("input"), + IsTensorConst("w"), + IsTensorConst("r"), + IsTensorConst("b"), + IsTensorConst(), + IsTensorConst("initialH"), + IsTensorConst("initialC"), + IsTensorConst(), + IsTensorConst(), + IsTensorConst(), + IsTensorConst(), + IsTensorConst("hiddenSize"), + IsTensorConst(), + IsTensorConst("outputSize")); + + private Expr? GetReplace(LSTM lstm, Call call, Expr input, Tensor w, Tensor r, Tensor b, Expr initialH, Expr initialC, int hiddenSize, int outputSize) + { + // x 先Squeeze掉batch size axis=1， + // ncnn : initial(H,C)默认为0，不起作用 + var inRes = Squeeze(input, new[] { 1 }); + var inResO = new Var(inRes.CheckedType); + + // args + // 0: hiddensize + // 1: weight_data_size : w.count/num_direction + var weightDataSize = w.Shape[1].FixedValue * w.Shape[2].FixedValue; + + // 2: direction [0,1,2] + int direction; + switch (lstm.Direction) + { + case LSTMDirection.Forward: + direction = 0; + break; + case LSTMDirection.Reverse: + direction = 1; + break; + default: // Bidirectional + direction = 2; + break; + } + + // bin + // reorder num_directions-IOFG - hidden_size to num_directions-IFOG - hidden_size + // weights: i f o g: + reverse i f o g + var (n_, c_, h_, w_) = (w.Shape[0].FixedValue, 4, hiddenSize, w.Shape[2].FixedValue); + var newWeights_ = Reshape(w, new[] { n_, c_, h_, w_ }); + var strides = new int[1] { 1 }; + var newWeights = Concat( + new IR.Tuple(new[] + { + Slice(newWeights_, new int[] { 0 }, new int[] { 1 }, new int[] { 1 }, strides), + Slice(newWeights_, new int[] { 2 }, new int[] { 3 }, new int[] { 1 }, strides), + Slice(newWeights_, new int[] { 1 }, new int[] { 2 }, new int[] { 1 }, strides), + Slice(newWeights_, new int[] { 3 }, new int[] { 4 }, new int[] { 1 }, strides), + }), + 1).Evaluate().AsTensor().ToArray(); + + // bias : x_b + w_b_re + var newBias_ = Reshape(b, new[] { w.Shape[0].FixedValue, 2, 4, hiddenSize }); + var biasW_ = Slice(newBias_, new[] { 0 }, new[] { 1 }, new int[] { 1 }, strides); + var biasR_ = Slice(newBias_, new[] { 1 }, new[] { 2 }, new int[] { 1 }, strides); + var biasW = Concat( + new IR.Tuple(new[] + { + Slice(biasW_, new int[] { 0 }, new int[] { 1 }, new int[] { 2 }, strides), + Slice(biasW_, new int[] { 2 }, new int[] { 3 }, new int[] { 2 }, strides), + Slice(biasW_, new int[] { 1 }, new int[] { 2 }, new int[] { 2 }, strides), + Slice(biasW_, new int[] { 3 }, new int[] { 4 }, new int[] { 2 }, strides), + }), + 2); + var biasR = Concat( + new IR.Tuple(new[] + { + Slice(biasR_, new int[] { 0 }, new int[] { 1 }, new int[] { 2 },strides), + Slice(biasR_, new int[] { 2 }, new int[] { 3 }, new int[] { 2 },strides), + Slice(biasR_, new int[] { 1 }, new int[] { 2 }, new int[] { 2 },strides), + Slice(biasR_, new int[] { 3 }, new int[] { 4 }, new int[] { 2 },strides), + }), + 2); + float[] newBias = (biasW + biasR).Evaluate().AsTensor().ToArray(); + + // R: 处理同weights + var newR_ = Reshape(r, new[] { n_, c_, h_, hiddenSize }); + float[] newR = Concat( + new IR.Tuple(new[] + { + Slice(newR_, new int[] { 0 }, new int[] { 1 }, new int[] { 1 }, strides), + Slice(newR_, new int[] { 2 }, new int[] { 3 }, new int[] { 1 }, strides), + Slice(newR_, new int[] { 1 }, new int[] { 2 }, new int[] { 1 }, strides), + Slice(newR_, new int[] { 3 }, new int[] { 4 }, new int[] { 1 }, strides), + }), + 1).Evaluate().AsTensor().ToArray(); + + var lstm_ = new Call(new Fusion("ncnn", NcnnLSTM(inResO, outputSize, hiddenSize, weightDataSize, direction, newWeights, newBias, newR), new[] { inResO }), inRes); + return new IR.Tuple(new[] { Unsqueeze(lstm_[0], new[] { 1,1 }), lstm_[1], lstm_[2] }); + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index cdc38d8988..37df62a6a8 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -16,6 +16,7 @@ using Nncase.Passes; using Nncase.Passes.Transforms; using Nncase.Quantization; +using Nncase.Passes.Rules.Neutral; namespace Nncase.Targets; @@ -66,6 +67,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); }); } From 01ce002fe1e929802a862ec410daf85ccf5c2b14 Mon Sep 17 00:00:00 2001 From: curioyang Date: Thu, 1 Feb 2024 08:13:43 +0000 Subject: [PATCH 057/132] Apply code-format changes --- .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 1 + modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLSTM.cs | 10 ++++++---- .../Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLSTM.cs | 10 +++++----- modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 6479161e44..b749f85d54 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -71,6 +71,7 @@ public CodeGenVisitor(NcnnEmitter emitter) } public List Inputs { get; } = new(); + public List Outputs { get; } = new(); protected override string VisitLeafVar(Var expr) diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLSTM.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLSTM.cs index 7f0d7dc54c..8e27b74956 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLSTM.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLSTM.cs @@ -8,10 +8,11 @@ using System.Threading.Tasks; using Nncase.CostModel; using Nncase.IR; -using Nncase.IR.RNN; using Nncase.IR.Ncnn; +using Nncase.IR.RNN; using OrtKISharp; using static Nncase.LSTMHelper; + namespace Nncase.Evaluator.Ncnn; /// @@ -23,6 +24,7 @@ public class NcnnLSTMEvaluator : IEvaluator, ITypeInferencer public IValue Visit(IEvaluateContext context, NcnnLSTM lstm) { var input = context.GetOrtArgumentValue(lstm, NcnnLSTM.Input); + // TODO: reorder w,r to iofg, b to numDirection-8*hiddensize (Now, numDirection-4*hiddensize (bw+br)) var w = lstm.W; var r = lstm.R; @@ -37,7 +39,7 @@ public IValue Visit(IEvaluateContext context, NcnnLSTM lstm) }; var seqLens = (int)input.Shape[0]; - var initH = Tensor.Zeros(new[] { direction_, seqLens, lstm.HiddenSize}); + var initH = Tensor.Zeros(new[] { direction_, seqLens, lstm.HiddenSize }); var initC = Tensor.Zeros(new[] { direction_, seqLens, lstm.HiddenSize }); var p = Tensor.Zeros(new[] { 1, seqLens, lstm.HiddenSize }); var actAlpha = new[] { 0.0f }; @@ -48,13 +50,13 @@ public IValue Visit(IEvaluateContext context, NcnnLSTM lstm) var outputSize = 1; var result = OrtKI.LSTM(input, w, r, b, seqLens, initH.ToOrtTensor(), initC.ToOrtTensor(), p.ToOrtTensor(), actAlpha, actBeta, new string[] { "sigmoid", "sigmoid", "tanh" }, clip, LSTMDirectionToValue(mode), hiddenSize, inputForget, 0, !clip.Equals(float.NaN), outputSize); return Value.FromTensors(result.Select(t => t.ToTensor()).ToArray()); - } /// public IRType Visit(ITypeInferenceContext context, NcnnLSTM target) { var input = context.CheckArgumentType(target, NcnnLSTM.Input); + // [num_direction, batch_size:1, hidden_size] var initH = new TensorType(DataTypes.Float32, new[] { target.Direction > 1 ? 2 : 1, 1, target.HiddenSize }); var initC = new TensorType(DataTypes.Float32, new[] { target.Direction > 1 ? 2 : 1, 1, target.HiddenSize }); @@ -124,7 +126,7 @@ public Metric Visit(IMetricEvaluateContext context, NcnnLSTM target_) private IRType Visit(ITypeInferenceContext context, TensorType x, TensorType initH, TensorType initC, int outputSize) { // TODO: confirm ncnn output - var numDirections = initH.Shape[0]; + _ = initH.Shape[0]; var seqLenIndex = 0; var yType = new TensorType(DataTypes.Float32, new[] { x.Shape[0].FixedValue, initH.Shape[2].FixedValue }); Console.WriteLine($"x.Shape[0].FixedValue: {x.Shape[0].FixedValue}, initH.Shape[2].FixedValue:{initH.Shape[2].FixedValue}"); diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLSTM.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLSTM.cs index 166c399e5d..588bcae200 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLSTM.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLSTM.cs @@ -105,10 +105,10 @@ public partial class LowerLSTM : RewriteRule var biasR = Concat( new IR.Tuple(new[] { - Slice(biasR_, new int[] { 0 }, new int[] { 1 }, new int[] { 2 },strides), - Slice(biasR_, new int[] { 2 }, new int[] { 3 }, new int[] { 2 },strides), - Slice(biasR_, new int[] { 1 }, new int[] { 2 }, new int[] { 2 },strides), - Slice(biasR_, new int[] { 3 }, new int[] { 4 }, new int[] { 2 },strides), + Slice(biasR_, new int[] { 0 }, new int[] { 1 }, new int[] { 2 }, strides), + Slice(biasR_, new int[] { 2 }, new int[] { 3 }, new int[] { 2 }, strides), + Slice(biasR_, new int[] { 1 }, new int[] { 2 }, new int[] { 2 }, strides), + Slice(biasR_, new int[] { 3 }, new int[] { 4 }, new int[] { 2 }, strides), }), 2); float[] newBias = (biasW + biasR).Evaluate().AsTensor().ToArray(); @@ -126,6 +126,6 @@ public partial class LowerLSTM : RewriteRule 1).Evaluate().AsTensor().ToArray(); var lstm_ = new Call(new Fusion("ncnn", NcnnLSTM(inResO, outputSize, hiddenSize, weightDataSize, direction, newWeights, newBias, newR), new[] { inResO }), inRes); - return new IR.Tuple(new[] { Unsqueeze(lstm_[0], new[] { 1,1 }), lstm_[1], lstm_[2] }); + return new IR.Tuple(new[] { Unsqueeze(lstm_[0], new[] { 1, 1 }), lstm_[1], lstm_[2] }); } } diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 37df62a6a8..aa280b4ea1 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -14,9 +14,9 @@ using Nncase.CodeGen.StackVM; using Nncase.IR; using Nncase.Passes; +using Nncase.Passes.Rules.Neutral; using Nncase.Passes.Transforms; using Nncase.Quantization; -using Nncase.Passes.Rules.Neutral; namespace Nncase.Targets; From 28c1de81fa0da3504e1dc6af574581856dd077dd Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 2 Feb 2024 15:46:24 +0800 Subject: [PATCH 058/132] Pad -->[ncnn op] Padding --- .../CodeGen/Ncnn/NcnnEmitter.cs | 19 +++++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Evaluator/Ncnn/NcnnPadding.cs | 85 +++++++++++++++++++ .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../IR/Ncnn/NcnnPadding.cs | 75 ++++++++++++++++ .../Passes/Rules/Ncnn/LowerPadding.cs | 61 +++++++++++++ .../Targets/CPUTarget.cs | 1 + 8 files changed, 248 insertions(+) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPadding.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPadding.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 3d717f9abe..a81dc16ed7 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -219,6 +219,25 @@ public void LSTM(string[] name, string input, int hiddenSize, int weightDataSize WriteFloatArray(r); } + public void Padding(string[] name, string input, int top, int bottom, int left, int right, int type, float value, int front, int behind) + { + AddLayer("Padding", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = top }, + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = bottom }, + [2] = new ParamValue { Kind = ParamKind.Int, IntValue = left }, + [3] = new ParamValue { Kind = ParamKind.Int, IntValue = right }, + [4] = new ParamValue { Kind = ParamKind.Int, IntValue = type }, + [5] = new ParamValue { Kind = ParamKind.Float, FloatValue = value }, + // [6] for perChannelPadDataSize. + [7] = new ParamValue { Kind = ParamKind.Int, IntValue = front }, + [8] = new ParamValue { Kind = ParamKind.Int, IntValue = behind }, + }); + + // TODO: confirm padValue is a tensor. + WriteFloatArray(new float[] { value }); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index b749f85d54..9d9b825ade 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -162,6 +162,9 @@ protected override string VisitLeafCall(Call expr) _emitter.LSTM(names.ToArray(), ExprMemo[expr.Arguments[0]], op.HiddenSize, op.WeightDataSize, op.Direction, op.W, op.R, op.B); break; + case NcnnPadding op: + _emitter.Padding(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Top, op.Bottom, op.Left, op.Right, op.Type, op.Value, op.Front, op.Behind); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index b58b979e5e..924950533f 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -29,5 +29,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs new file mode 100644 index 0000000000..f3fc9586e6 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs @@ -0,0 +1,85 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnPaddingEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnPadding padding) + { + var input = context.GetOrtArgumentValue(padding, NcnnPadding.Input); + // ncnn not support N + var pads = new Tensor(new int[] { 0, 0, padding.Front, padding.Top, padding.Left, padding.Behind, padding.Bottom, padding.Right }); + return OrtKI.Pad(input, pads.ToOrtTensor(), padding.Value, padding.Type switch + { + 0 => "Constant", + 1 => "Reflect", + 2 => "Edge", + _ => "Symmetric", + }).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnPadding target) + { + var input = context.CheckArgumentType(target, NcnnPadding.Input); + var c = target.Front + target.Behind; + var h = target.Left + target.Right; + var w = target.Top + target.Bottom; + return Visit(input, c, h, w); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnPadding target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnPadding target) + { + var inputType = context.GetArgumentType(target, NcnnPadding.Input); + var outputType = context.GetReturnType(); + + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(inputType), + [CostFactorNames.MemoryStore] = outputType is TensorType outT ? CostUtility.GetMemoryAccess(outT) : CostUtility.GetMemoryAccess(inputType), + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnPadding target) => context.GetArgumentShape(target, NcnnPadding.Input); + + private IRType Visit(TensorType input, int c, int h, int w) + { + if (input.Shape.Count != 3) + { + throw new ArgumentException($"{nameof(NcnnPadding)} only supports 3-dims input."); + } + + var newShape = new int[] { 0, 0, 0 }; // Must be 3-dims without BatchSize. + newShape[0] = input.Shape[0].FixedValue + c; + newShape[1] = input.Shape[1].FixedValue + h; + newShape[2] = input.Shape[2].FixedValue + w; + return new TensorType(input.DType, newShape); + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 566b5a1ecd..ef4cc64ed6 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -54,4 +54,7 @@ public static Call NcnnInstanceNorm(Expr expr, int channels, float eps, int affi public static Call NcnnLSTM(Expr expr, int outputSize, int hiddenSize, int weightDataSize, int direction, float[] w, float[] b, float[] r) => new Call(new NcnnLSTM(outputSize, hiddenSize, weightDataSize, direction, w, b, r), expr); + + public static Call NcnnPadding(Expr expr, int top, int bottom, int left, int right, int type, float value, int front, int behind) => + new Call(new NcnnPadding(top, bottom, left, right, type, value, front, behind), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPadding.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPadding.cs new file mode 100644 index 0000000000..1f6d11ece5 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPadding.cs @@ -0,0 +1,75 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Gets expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnPadding : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnPadding), 0, "input"); + + /// + /// Gets Top of Ncnn Padding. + /// + public int Top { get; } + + /// + /// Gets Bottom of Ncnn Padding. + /// + public int Bottom { get; } + + /// + /// Gets Left of Ncnn Padding. + /// + public int Left { get; } + + /// + /// Gets Right of Ncnn Padding. + /// + public int Right { get; } + + /// + /// Gets Type of Ncnn Padding. + /// + public int Type { get; } + + /// + /// Gets Value of Ncnn Padding. + /// + public float Value { get; } + + // /// + // /// Gets PerChannelPadSize of Ncnn Padding. Do not need in onnx model. + // /// + // public int PerChannelPadSize { get; } + + /// + /// Gets Front of Ncnn Padding. + /// + public int Front { get; } + + /// + /// Gets Behind of Ncnn Padding. + /// + public int Behind { get; } + + /// + public override string DisplayProperty() + { + return $"Top: {Top}, Bottom: {Bottom}, Left: {Left}, Right: {Right}, Type: {Type}, Value: {Value}, Front: {Front}, Behind: {Behind}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPadding.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPadding.cs new file mode 100644 index 0000000000..920239c2c9 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPadding.cs @@ -0,0 +1,61 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerPadding : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsPad( + "pad", + "padCall", + x => true, + IsWildcard("input"), + IsTensorConst("pads"), + IsTensorConst("padValue")); + + private Expr? GetReplace(IR.NN.Pad pad, Expr input, Expr pads, Expr padValue) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + var paddings = pads.Evaluate().AsTensor().ToArray(); + var (front, behind, top, bottom, left, right) = + (paddings[2], paddings[3], paddings[4], paddings[5], paddings[6], paddings[7]); + var padV = padValue.Evaluate().AsTensor().ToArray()[0]; // TODO: IsScaler or not. + int type = pad.PadMode switch + { + PadMode.Constant => 0, + PadMode.Edge => 1, + PadMode.Reflect => 2, + _ => throw new NotImplementedException("Ncnn not support other pad mode"), + }; + + var erf = new Call(new Fusion("ncnn", NcnnPadding(inResO, top, bottom, left, right, type, padV, front, behind), new[] { inResO }), inRes); + + return Unsqueeze(erf, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index aa280b4ea1..620dcd99de 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -68,6 +68,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); }); } From ee2605f768bf15dcce86c6252dfd5fc6cdc226b7 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 5 Feb 2024 10:14:05 +0800 Subject: [PATCH 059/132] ReduceWindows2D -->[ncnn op] Pooling --- .../Nncase.Modules.Ncnn/ArgsStruct/args.cs | 51 ++++++++++ .../CodeGen/Ncnn/NcnnEmitter.cs | 23 +++++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Evaluator/Ncnn/NcnnPooling.cs | 99 +++++++++++++++++++ .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 4 + .../IR/Ncnn/NcnnPooling.cs | 36 +++++++ .../Passes/Rules/Ncnn/LowerPooling.cs | 73 ++++++++++++++ .../Targets/CPUTarget.cs | 1 + 9 files changed, 291 insertions(+) create mode 100644 modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPooling.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPooling.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs diff --git a/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs new file mode 100644 index 0000000000..ab9d3bacac --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs @@ -0,0 +1,51 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +namespace Nncase.ArgsStruct; +/// +/// Ncnn Pooling arguments. +/// +public record PoolingArgs +{ + public PoolingArgs() + { + } + + public PoolingArgs(int poolingType, int kernelW, int kernelH, int strideW, int strideH, int padLeft, int padRight, int padTop, int padBottom, bool globalPooling, int padMode, bool avgPoolCountIncludePad, bool adaptivePooling, int outW, int outH, bool ceilMode) + { + PoolingType = poolingType; + KernelH = kernelH; + KernelW = kernelW; + StrideH = strideH; + StrideW = strideW; + PadLeft = padLeft; + PadRight = padRight; + PadTop = padTop; + PadBottom = padBottom; + GlobalPooling = globalPooling; + PadMode = padMode; + AvgPoolCountIncludePad = avgPoolCountIncludePad; + AdaptivePooling = adaptivePooling; + OutW = outW; + OutH = outH; + CeilMode = ceilMode; + } + + public int PoolingType { get; } + public int KernelW { get; } + public int KernelH { get; } + public int StrideW { get; } + public int StrideH { get; } + public int PadLeft { get; } + public int PadRight { get; } + public int PadTop { get; } + public int PadBottom { get; } + public bool GlobalPooling { get; } + public int PadMode { get; } + public bool AvgPoolCountIncludePad { get; } + public bool AdaptivePooling { get; } + public int OutW { get; } + public int OutH { get; } + public bool CeilMode { get; } + +} diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index a81dc16ed7..b571aeb4f0 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -7,6 +7,7 @@ using System.Text; using System.Threading.Tasks; using Microsoft.Toolkit.HighPerformance; +using Nncase.ArgsStruct; using Nncase.IR.Ncnn; using Nncase.Runtime.Ncnn; @@ -238,6 +239,28 @@ public void Padding(string[] name, string input, int top, int bottom, int left, WriteFloatArray(new float[] { value }); } + public void Pooling(string[] name, string input, PoolingArgs poolingArgs) + { + AddLayer("Pooling", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.PoolingType }, + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.KernelW }, + [11] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.KernelH }, + [2] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.StrideW }, + [12] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.StrideH }, + [3] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.PadLeft }, + [14] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.PadRight }, + [13] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.PadTop }, + [15] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.PadBottom }, + [4] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.GlobalPooling ? 1 : 0 }, + [5] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.PadMode }, + [6] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.AvgPoolCountIncludePad ? 1 : 0 }, + [7] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.AdaptivePooling ? 1 : 0 }, + // [8] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.OutH }, + // [18] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.OutH }, + }); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 9d9b825ade..fd798cfda8 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -165,6 +165,9 @@ protected override string VisitLeafCall(Call expr) case NcnnPadding op: _emitter.Padding(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Top, op.Bottom, op.Left, op.Right, op.Type, op.Value, op.Front, op.Behind); break; + case NcnnPooling op: + _emitter.Pooling(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Args); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 924950533f..dd27af6936 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -30,5 +30,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPooling.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPooling.cs new file mode 100644 index 0000000000..7260433bfb --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPooling.cs @@ -0,0 +1,99 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using Nncase.IR.Tensors; +using OrtKISharp; +using static Nncase.Evaluator.EvaluatorUtil; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnPoolingEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnPooling pooling) + { + var input = context.GetOrtArgumentValue(pooling, NcnnPooling.Input); + var kernelSize = new long[] { pooling.Args.KernelH, pooling.Args.KernelW }; + var stride = new long[] { pooling.Args.StrideH, pooling.Args.StrideW }; + var dilation = new long[] { 1, 1 }; + var padData = new long[] { pooling.Args.PadTop, pooling.Args.PadLeft, pooling.Args.PadBottom, pooling.Args.PadRight }; + // var ceilMode = pooling.Args.PadMode == 0 ? 1 : 0; + long countIncludePad = pooling.Args.AvgPoolCountIncludePad ? 1 : 0; + + return (pooling.Args.PoolingType switch + { + 0 => OrtKI.MaxPool(input, "NOTSET", pooling.Args.CeilMode?1:0, dilation, kernelSize, padData, countIncludePad, stride)[0], + 1 => OrtKI.AveragePool(input, "NOTSET", pooling.Args.CeilMode?1:0, countIncludePad, kernelSize, padData, stride), + _ => throw + new NotImplementedException($"Pooling type {pooling.Args.PoolingType} is not supported."), + }).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnPooling target) + { + var input = context.CheckArgumentType(target, NcnnPooling.Input); + return Visit(target, input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnPooling target) + { + var inputType = context.GetArgumentType(target, NcnnPooling.Input); + var kernelSize = new int[] { target.Args.KernelH, target.Args.KernelW }; + var outputType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) + CostUtility.GetMemoryAccess(outputType), + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(outputType, kernelSize[0] * kernelSize[1]), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnPooling target) + { + var inputType = context.GetArgumentType(target, NcnnPooling.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.DivFLOPs + MetricUtility.ExpFLOPs + MetricUtility.MulFLOPs + MetricUtility.SubFLOPs + MetricUtility.AddFLOPs + (MetricUtility.CmpFLOPs * 2)), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnPooling target) => context.GetArgumentShape(target, NcnnPooling.Input); + + private IRType Visit(NcnnPooling pooling, TensorType input) + { + // var input = context.GetOrtArgumentValue(pooling, NcnnPooling.Input); + var kernelSize = new long[] { pooling.Args.KernelH, pooling.Args.KernelW }; + var stride = new long[] { pooling.Args.StrideH, pooling.Args.StrideW }; + var padData = new long[] { pooling.Args.PadTop, pooling.Args.PadBottom, pooling.Args.PadLeft, pooling.Args.PadRight }; + + long countIncludePad = pooling.Args.AvgPoolCountIncludePad ? 1 : 0; + var newInput = new TensorType(input.DType, input.Shape.InsertAndClone(0, 1)); + var output_ = TypeInference.ReduceWindow2DType(newInput, kernelSize, stride, + Tensor.From(padData, new[] { 2, 2, }), pooling.Args.CeilMode); + if (output_ is TensorType t) + { + var newShape = t.Shape.ToArray(); + return new TensorType(t.DType, newShape[1..]); + } + + return output_; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index ef4cc64ed6..95993ea3d6 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Nncase.ArgsStruct; using Nncase.IR.Ncnn; namespace Nncase.IR.F; @@ -57,4 +58,7 @@ public static Call NcnnLSTM(Expr expr, int outputSize, int hiddenSize, int weigh public static Call NcnnPadding(Expr expr, int top, int bottom, int left, int right, int type, float value, int front, int behind) => new Call(new NcnnPadding(top, bottom, left, right, type, value, front, behind), expr); + + public static Call NcnnPooling(Expr expr, PoolingArgs poolingArgs) => + new Call(new NcnnPooling(poolingArgs), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPooling.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPooling.cs new file mode 100644 index 0000000000..fb9744e8e1 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPooling.cs @@ -0,0 +1,36 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; +using Nncase.ArgsStruct; + +namespace Nncase.IR.Ncnn; + +/// +/// Pooling expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnPooling : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnPooling), 0, "input"); + + /// + /// Gets PoolingArgs of Ncnn Pooling. + /// + public PoolingArgs Args { get; } + + /// + public override string DisplayProperty() + { + return $"{Args.PoolingType}, Kernel: {Args.KernelW}-{Args.KernelH}, Stride: {Args.StrideW}-{Args.StrideH}, Padding: {Args.PadLeft}-{Args.PadRight}-{Args.PadTop}-{Args.PadBottom}, CeilMode: {Args.CeilMode}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs new file mode 100644 index 0000000000..452ee976d1 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs @@ -0,0 +1,73 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; +using Nncase.ArgsStruct; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; +using Nncase.IR.NN; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerPooling : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsReduceWindow2D( + "pdp", + _ => true, + IsWildcard("input"), + IsWildcard("initValue"), + IsWildcard("filter"), + IsWildcard("stride"), + IsTensorConst("padding"), + IsWildcard("dilation"), + IsWildcard("ceilMode"), + IsWildcard("countIncludePad")); + + private Expr? GetReplace(ReduceWindow2D pdp, Expr input, float initValue, Expr filter, Expr stride, Expr padding, Expr dilation, bool ceilMode, bool countIncludePad) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + var poolingType = pdp.ReduceOp switch + { + ReduceOp.Max => 0, + ReduceOp.Mean => 1, + _ => throw new NotImplementedException($"{pdp.ReduceOp} not suppor in ncnn!"), + }; + + var kernel_ = filter.Evaluate().AsTensor().ToArray(); + var (kernelW, kernelH) = (kernel_[1], kernel_[0]); + + var stride_ = stride.Evaluate().AsTensor().ToArray(); + var (strideW, strideH) = (stride_[1], stride_[0]); + + var padding_ = padding.Evaluate().AsTensor().ToArray(); + var (padLeft, padTop, padRight, padBottom) = (padding_[2], padding_[3], padding_[0], padding_[1]); + + // Globalpool has been converted to pool, reflected in the size of the kernel. + var args = new PoolingArgs(poolingType, kernelW, kernelH, strideW, strideH, padLeft, padRight, padTop, padBottom, false, ceilMode ? 0 : 1, countIncludePad, false, 0, 0, ceilMode); + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var pool = new Call(new Fusion("ncnn", NcnnPooling(inResO, args), new[] { inResO }), inRes); + return Unsqueeze(pool, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 620dcd99de..efbcb9721e 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -69,6 +69,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); }); } From 91097a67bedc5653cca0a7d008553f81d847796f Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 5 Feb 2024 11:05:15 +0800 Subject: [PATCH 060/132] PRelu -->[ncnn op] PRelu --- .../Nncase.Modules.Ncnn/ArgsStruct/args.cs | 18 ++++- .../CodeGen/Ncnn/NcnnEmitter.cs | 12 ++++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnLSTM.cs | 1 - .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Evaluator/Ncnn/NcnnPRelu.cs | 66 +++++++++++++++++++ .../Evaluator/Ncnn/NcnnPadding.cs | 1 + .../Evaluator/Ncnn/NcnnPooling.cs | 10 +-- .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnPRelu.cs | 35 ++++++++++ .../IR/Ncnn/NcnnPooling.cs | 2 +- .../Passes/Rules/Ncnn/LowerPRelu.cs | 47 +++++++++++++ .../Passes/Rules/Ncnn/LowerPooling.cs | 5 +- .../Targets/CPUTarget.cs | 1 + 14 files changed, 193 insertions(+), 12 deletions(-) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPRelu.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPRelu.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPRelu.cs diff --git a/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs index ab9d3bacac..2335976e8c 100644 --- a/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs +++ b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs @@ -1,4 +1,4 @@ -// Copyright (c) Canaan Inc. All rights reserved. +// Copyright (c) Canaan Inc. All rights reserved. // Licensed under the Apache license. See LICENSE file in the project root for full license information. namespace Nncase.ArgsStruct; @@ -32,20 +32,34 @@ public PoolingArgs(int poolingType, int kernelW, int kernelH, int strideW, int s } public int PoolingType { get; } + public int KernelW { get; } + public int KernelH { get; } + public int StrideW { get; } + public int StrideH { get; } + public int PadLeft { get; } + public int PadRight { get; } + public int PadTop { get; } + public int PadBottom { get; } + public bool GlobalPooling { get; } + public int PadMode { get; } + public bool AvgPoolCountIncludePad { get; } + public bool AdaptivePooling { get; } + public int OutW { get; } + public int OutH { get; } - public bool CeilMode { get; } + public bool CeilMode { get; } } diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index b571aeb4f0..502b913823 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -230,6 +230,7 @@ public void Padding(string[] name, string input, int top, int bottom, int left, [3] = new ParamValue { Kind = ParamKind.Int, IntValue = right }, [4] = new ParamValue { Kind = ParamKind.Int, IntValue = type }, [5] = new ParamValue { Kind = ParamKind.Float, FloatValue = value }, + // [6] for perChannelPadDataSize. [7] = new ParamValue { Kind = ParamKind.Int, IntValue = front }, [8] = new ParamValue { Kind = ParamKind.Int, IntValue = behind }, @@ -256,11 +257,22 @@ public void Pooling(string[] name, string input, PoolingArgs poolingArgs) [5] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.PadMode }, [6] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.AvgPoolCountIncludePad ? 1 : 0 }, [7] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.AdaptivePooling ? 1 : 0 }, + // [8] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.OutH }, // [18] = new ParamValue { Kind = ParamKind.Int, IntValue = poolingArgs.OutH }, }); } + public void PRelu(string[] name, string input, float[] slope) + { + AddLayer("PRelu", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = slope.Length }, + }); + + WriteFloatArray(slope); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index fd798cfda8..c193635a96 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -168,6 +168,9 @@ protected override string VisitLeafCall(Call expr) case NcnnPooling op: _emitter.Pooling(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Args); break; + case NcnnPRelu op: + _emitter.PRelu(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Slope); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLSTM.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLSTM.cs index 8e27b74956..7f2f7576b1 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLSTM.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnLSTM.cs @@ -127,7 +127,6 @@ private IRType Visit(ITypeInferenceContext context, TensorType x, TensorType ini { // TODO: confirm ncnn output _ = initH.Shape[0]; - var seqLenIndex = 0; var yType = new TensorType(DataTypes.Float32, new[] { x.Shape[0].FixedValue, initH.Shape[2].FixedValue }); Console.WriteLine($"x.Shape[0].FixedValue: {x.Shape[0].FixedValue}, initH.Shape[2].FixedValue:{initH.Shape[2].FixedValue}"); var result = new[] { yType, initH, initC }; diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index dd27af6936..c4476bc333 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -31,5 +31,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPRelu.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPRelu.cs new file mode 100644 index 0000000000..f23c7f71b5 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPRelu.cs @@ -0,0 +1,66 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnPReluEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnPRelu pRelu) + { + var input = context.GetOrtArgumentValue(pRelu, NcnnPRelu.Input); + var slope = pRelu.Slope; + return OrtKI.PRelu(input, slope).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnPRelu target) + { + var input = context.CheckArgumentType(target, NcnnPRelu.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnPRelu target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnPRelu target) + { + var inputType = context.GetArgumentType(target, NcnnPRelu.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.CmpFLOPs + MetricUtility.MulFLOPs), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnPRelu target) => context.GetArgumentShape(target, NcnnPRelu.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs index f3fc9586e6..99a2502b59 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs @@ -23,6 +23,7 @@ public class NcnnPaddingEvaluator : IEvaluator, ITypeInferencer(new int[] { 0, 0, padding.Front, padding.Top, padding.Left, padding.Behind, padding.Bottom, padding.Right }); return OrtKI.Pad(input, pads.ToOrtTensor(), padding.Value, padding.Type switch diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPooling.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPooling.cs index 7260433bfb..5cebbca4a2 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPooling.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPooling.cs @@ -28,14 +28,15 @@ public IValue Visit(IEvaluateContext context, NcnnPooling pooling) var kernelSize = new long[] { pooling.Args.KernelH, pooling.Args.KernelW }; var stride = new long[] { pooling.Args.StrideH, pooling.Args.StrideW }; var dilation = new long[] { 1, 1 }; - var padData = new long[] { pooling.Args.PadTop, pooling.Args.PadLeft, pooling.Args.PadBottom, pooling.Args.PadRight }; + var padData = new long[] { pooling.Args.PadTop, pooling.Args.PadLeft, pooling.Args.PadBottom, pooling.Args.PadRight }; + // var ceilMode = pooling.Args.PadMode == 0 ? 1 : 0; long countIncludePad = pooling.Args.AvgPoolCountIncludePad ? 1 : 0; return (pooling.Args.PoolingType switch { - 0 => OrtKI.MaxPool(input, "NOTSET", pooling.Args.CeilMode?1:0, dilation, kernelSize, padData, countIncludePad, stride)[0], - 1 => OrtKI.AveragePool(input, "NOTSET", pooling.Args.CeilMode?1:0, countIncludePad, kernelSize, padData, stride), + 0 => OrtKI.MaxPool(input, "NOTSET", pooling.Args.CeilMode ? 1 : 0, dilation, kernelSize, padData, countIncludePad, stride)[0], + 1 => OrtKI.AveragePool(input, "NOTSET", pooling.Args.CeilMode ? 1 : 0, countIncludePad, kernelSize, padData, stride), _ => throw new NotImplementedException($"Pooling type {pooling.Args.PoolingType} is not supported."), }).ToValue(); @@ -83,8 +84,7 @@ private IRType Visit(NcnnPooling pooling, TensorType input) var kernelSize = new long[] { pooling.Args.KernelH, pooling.Args.KernelW }; var stride = new long[] { pooling.Args.StrideH, pooling.Args.StrideW }; var padData = new long[] { pooling.Args.PadTop, pooling.Args.PadBottom, pooling.Args.PadLeft, pooling.Args.PadRight }; - - long countIncludePad = pooling.Args.AvgPoolCountIncludePad ? 1 : 0; + _ = pooling.Args.AvgPoolCountIncludePad ? 1 : 0; var newInput = new TensorType(input.DType, input.Shape.InsertAndClone(0, 1)); var output_ = TypeInference.ReduceWindow2DType(newInput, kernelSize, stride, Tensor.From(padData, new[] { 2, 2, }), pooling.Args.CeilMode); diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 95993ea3d6..a9572331d6 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -61,4 +61,7 @@ public static Call NcnnPadding(Expr expr, int top, int bottom, int left, int rig public static Call NcnnPooling(Expr expr, PoolingArgs poolingArgs) => new Call(new NcnnPooling(poolingArgs), expr); + + public static Call NcnnPRelu(Expr expr, float[] slope) => + new Call(new NcnnPRelu(slope), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPRelu.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPRelu.cs new file mode 100644 index 0000000000..9dbb838cce --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPRelu.cs @@ -0,0 +1,35 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// PRelu expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnPRelu : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnPRelu), 0, "input"); + + /// + /// Gets Slope of Ncnn PRelu. + /// + public float[] Slope { get; } + + /// + public override string DisplayProperty() + { + return $""; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPooling.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPooling.cs index fb9744e8e1..77522cc0cd 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPooling.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPooling.cs @@ -7,8 +7,8 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -using Nncase.PatternMatch; using Nncase.ArgsStruct; +using Nncase.PatternMatch; namespace Nncase.IR.Ncnn; diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPRelu.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPRelu.cs new file mode 100644 index 0000000000..772eeebc36 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPRelu.cs @@ -0,0 +1,47 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.IR.NN; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerPRelu : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsPRelu( + + // "prelu", + IsWildcard("input"), + IsTensorConst("slope")); + + private Expr? GetReplace(Expr input, Tensor slope) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var pRelu = new Call(new Fusion("ncnn", NcnnPRelu(inResO, slope.ToArray()), new[] { inResO }), input); + return Unsqueeze(pRelu, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs index 452ee976d1..6d0f8b1849 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs @@ -6,18 +6,17 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Nncase.ArgsStruct; using Nncase.IR; using Nncase.IR.Math; using Nncase.IR.Ncnn; +using Nncase.IR.NN; using Nncase.PatternMatch; -using Nncase.ArgsStruct; - using static Nncase.IR.F.Ncnn; using static Nncase.IR.F.Tensors; using static Nncase.IR.TypePatternUtility; using static Nncase.PatternMatch.F.NN; using static Nncase.PatternMatch.Utility; -using Nncase.IR.NN; namespace Nncase.Passes.Rules.Ncnn; diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index efbcb9721e..c4280d5806 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -70,6 +70,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); // x86 need ncnn later than 20231115 }); } From b11c3c469c52648ccc71403ecd0b4d67e573732e Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 5 Feb 2024 14:39:10 +0800 Subject: [PATCH 061/132] fix name --- .../CodeGen/Ncnn/NcnnEmitter.cs | 4 ++-- .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 4 ++-- .../Evaluator/Ncnn/NcnnPRelu.cs | 22 +++++++++---------- .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 4 ++-- .../IR/Ncnn/{NcnnPRelu.cs => NcnnPReLU.cs} | 8 +++---- .../Ncnn/{LowerPRelu.cs => LowerPReLU.cs} | 6 ++--- .../Targets/CPUTarget.cs | 2 +- 7 files changed, 25 insertions(+), 25 deletions(-) rename modules/Nncase.Modules.Ncnn/IR/Ncnn/{NcnnPRelu.cs => NcnnPReLU.cs} (85%) rename modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/{LowerPRelu.cs => LowerPReLU.cs} (87%) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 502b913823..6904f2b6b6 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -263,9 +263,9 @@ public void Pooling(string[] name, string input, PoolingArgs poolingArgs) }); } - public void PRelu(string[] name, string input, float[] slope) + public void PReLU(string[] name, string input, float[] slope) { - AddLayer("PRelu", name[0], new[] { input }, name, new ParamDict + AddLayer("PReLU", name[0], new[] { input }, name, new ParamDict { [0] = new ParamValue { Kind = ParamKind.Int, IntValue = slope.Length }, }); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index c193635a96..8db0ac993e 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -168,8 +168,8 @@ protected override string VisitLeafCall(Call expr) case NcnnPooling op: _emitter.Pooling(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Args); break; - case NcnnPRelu op: - _emitter.PRelu(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Slope); + case NcnnPReLU op: + _emitter.PReLU(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Slope); break; default: throw new NotSupportedException(); diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPRelu.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPRelu.cs index f23c7f71b5..3e5fd294c7 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPRelu.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPRelu.cs @@ -14,27 +14,27 @@ namespace Nncase.Evaluator.Ncnn; /// -/// Evaluator for . +/// Evaluator for . /// -public class NcnnPReluEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +public class NcnnPReluEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator { /// - public IValue Visit(IEvaluateContext context, NcnnPRelu pRelu) + public IValue Visit(IEvaluateContext context, NcnnPReLU pReLU) { - var input = context.GetOrtArgumentValue(pRelu, NcnnPRelu.Input); - var slope = pRelu.Slope; + var input = context.GetOrtArgumentValue(pReLU, NcnnPReLU.Input); + var slope = pReLU.Slope; return OrtKI.PRelu(input, slope).ToValue(); } /// - public IRType Visit(ITypeInferenceContext context, NcnnPRelu target) + public IRType Visit(ITypeInferenceContext context, NcnnPReLU target) { - var input = context.CheckArgumentType(target, NcnnPRelu.Input); + var input = context.CheckArgumentType(target, NcnnPReLU.Input); return Visit(input); } /// - public Cost Visit(ICostEvaluateContext context, NcnnPRelu target) + public Cost Visit(ICostEvaluateContext context, NcnnPReLU target) { var ret = context.GetReturnType(); return new() @@ -44,9 +44,9 @@ public Cost Visit(ICostEvaluateContext context, NcnnPRelu target) }; } - public Metric Visit(IMetricEvaluateContext context, NcnnPRelu target) + public Metric Visit(IMetricEvaluateContext context, NcnnPReLU target) { - var inputType = context.GetArgumentType(target, NcnnPRelu.Input); + var inputType = context.GetArgumentType(target, NcnnPReLU.Input); var returnType = context.GetReturnType(); return new() @@ -57,7 +57,7 @@ public Metric Visit(IMetricEvaluateContext context, NcnnPRelu target) }; } - public Expr Visit(IShapeEvaluateContext context, NcnnPRelu target) => context.GetArgumentShape(target, NcnnPRelu.Input); + public Expr Visit(IShapeEvaluateContext context, NcnnPReLU target) => context.GetArgumentShape(target, NcnnPReLU.Input); private IRType Visit(TensorType input) { diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index a9572331d6..22511dfd53 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -62,6 +62,6 @@ public static Call NcnnPadding(Expr expr, int top, int bottom, int left, int rig public static Call NcnnPooling(Expr expr, PoolingArgs poolingArgs) => new Call(new NcnnPooling(poolingArgs), expr); - public static Call NcnnPRelu(Expr expr, float[] slope) => - new Call(new NcnnPRelu(slope), expr); + public static Call NcnnPReLU(Expr expr, float[] slope) => + new Call(new NcnnPReLU(slope), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPRelu.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPReLU.cs similarity index 85% rename from modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPRelu.cs rename to modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPReLU.cs index 9dbb838cce..e1cde8c8ca 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPRelu.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPReLU.cs @@ -12,18 +12,18 @@ namespace Nncase.IR.Ncnn; /// -/// PRelu expression. +/// PReLU expression. /// [PatternFunctionalGenerator] -public sealed partial class NcnnPRelu : Op +public sealed partial class NcnnPReLU : Op { /// /// Gets input. /// - public static readonly ParameterInfo Input = new(typeof(NcnnPRelu), 0, "input"); + public static readonly ParameterInfo Input = new(typeof(NcnnPReLU), 0, "input"); /// - /// Gets Slope of Ncnn PRelu. + /// Gets Slope of Ncnn PReLU. /// public float[] Slope { get; } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPRelu.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPReLU.cs similarity index 87% rename from modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPRelu.cs rename to modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPReLU.cs index 772eeebc36..df06a62d66 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPRelu.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPReLU.cs @@ -21,7 +21,7 @@ namespace Nncase.Passes.Rules.Ncnn; [RuleGenerator] -public partial class LowerPRelu : RewriteRule +public partial class LowerPReLU : RewriteRule { /// public override Pattern Pattern { get; } = IsPRelu( @@ -41,7 +41,7 @@ public partial class LowerPRelu : RewriteRule var inRes = Squeeze(input, new[] { 0 }); var inResO = new Var(inRes.CheckedType); - var pRelu = new Call(new Fusion("ncnn", NcnnPRelu(inResO, slope.ToArray()), new[] { inResO }), input); - return Unsqueeze(pRelu, new[] { 0 }); + var pReLU = new Call(new Fusion("ncnn", NcnnPReLU(inResO, slope.ToArray()), new[] { inResO }), input); + return Unsqueeze(pReLU, new[] { 0 }); } } diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index c4280d5806..7015c06911 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -70,7 +70,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); - p.Add(); // x86 need ncnn later than 20231115 + p.Add(); }); } From bdc2cfaf6e543a2923600fbc495464fa194b8139 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Tue, 6 Feb 2024 18:36:57 +0800 Subject: [PATCH 062/132] Reduce -->[ncnn op] Reduction --- .../Nncase.Modules.Ncnn/ArgsStruct/args.cs | 26 +++ .../CodeGen/Ncnn/NcnnEmitter.cs | 25 +++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Evaluator/Ncnn/NcnnReduction.cs | 97 ++++++++ .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../IR/Ncnn/NcnnReduction.cs | 36 +++ .../Passes/Rules/Ncnn/LowerPooling.cs | 2 +- .../Passes/Rules/Ncnn/LowerReduction.cs | 210 ++++++++++++++++++ .../Targets/CPUTarget.cs | 14 +- 10 files changed, 414 insertions(+), 3 deletions(-) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReduction.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReduction.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReduction.cs diff --git a/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs index 2335976e8c..086750971d 100644 --- a/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs +++ b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs @@ -63,3 +63,29 @@ public PoolingArgs(int poolingType, int kernelW, int kernelH, int strideW, int s public bool CeilMode { get; } } + +public record ReductionArgs +{ + public ReductionArgs() + { + } + + public ReductionArgs(int opType, int reduceAll, float coeff, long[] axes, int keepdims) + { + OpType = opType; + ReduceAll = reduceAll; + Coeff = coeff; + Axes = axes; + Keepdims = keepdims; + } + + public int OpType { get; } + + public int ReduceAll { get; } + + public float Coeff { get; } + + public long[] Axes { get; } + + public int Keepdims { get; } +} diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 6904f2b6b6..02f8e137d6 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using Microsoft.Toolkit.HighPerformance; using Nncase.ArgsStruct; +using Nncase.IR; using Nncase.IR.Ncnn; using Nncase.Runtime.Ncnn; @@ -273,6 +274,30 @@ public void PReLU(string[] name, string input, float[] slope) WriteFloatArray(slope); } + public void Reduction(string[] name, string input, ReductionArgs reductionArgs) + { + var args = new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = reductionArgs.OpType }, + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = reductionArgs.ReduceAll }, + }; + + if (reductionArgs.Axes.Length > 0) + { + var axesSizeAndData = new List { reductionArgs.Axes.Length }; + foreach (var item in reductionArgs.Axes) + { + axesSizeAndData.Add(item); + } + + args.Add(-3, new ParamValue { Kind = ParamKind.ArrayOfInt, TensorValue = axesSizeAndData.ToArray() }); + } + + args.Add(4, new ParamValue { Kind = ParamKind.Int, IntValue = reductionArgs.Keepdims }); + args.Add(5, new ParamValue { Kind = ParamKind.Int, IntValue = 1 }); + AddLayer("Reduction", name[0], new[] { input }, name, args); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 8db0ac993e..cbd1d64d72 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -171,6 +171,9 @@ protected override string VisitLeafCall(Call expr) case NcnnPReLU op: _emitter.PReLU(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Slope); break; + case NcnnReduction op: + _emitter.Reduction(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Args); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index c4476bc333..7de97b68ad 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -32,5 +32,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReduction.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReduction.cs new file mode 100644 index 0000000000..f02b078cc9 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReduction.cs @@ -0,0 +1,97 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using Nncase.IR.Tensors; +using OrtKISharp; +using static Nncase.Evaluator.EvaluatorUtil; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnReductionEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnReduction reduction) + { + var input = context.GetOrtArgumentValue(reduction, NcnnReduction.Input); + var axes = reduction.Args.Axes; + var keepDims = reduction.Args.Keepdims; + + return (reduction.Args.OpType switch + { + 0 => OrtKI.ReduceSum(input, axes, keepDims, 0), + 3 => OrtKI.ReduceMean(input, axes.Cast().ToArray(), keepDims), + 4 => OrtKI.ReduceMax(input, axes.Cast().ToArray(), keepDims), + 5 => OrtKI.ReduceMin(input, axes.Cast().ToArray(), keepDims), + 6 => OrtKI.ReduceProd(input, axes.Cast().ToArray(), keepDims), + _ => throw + new NotImplementedException($"Reduction opType {reduction.Args.OpType} is not supported."), + }).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnReduction target) + { + var input = context.CheckArgumentType(target, NcnnReduction.Input); + return Visit(target, input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnReduction target) + { + var inputType = context.GetArgumentType(target, NcnnReduction.Input); + var returnType = context.GetReturnType(); + var rF = MetricUtility.GetFLOPs(returnType); + var iF = MetricUtility.GetFLOPs(inputType); + var inner = iF / rF; + _ = iF / inner; + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) + CostUtility.GetMemoryAccess(returnType), + [MetricFactorNames.FLOPs] = iF, + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnReduction target) + { + var inputType = context.GetArgumentType(target, NcnnReduction.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.DivFLOPs + MetricUtility.ExpFLOPs + MetricUtility.MulFLOPs + MetricUtility.SubFLOPs + MetricUtility.AddFLOPs + (MetricUtility.CmpFLOPs * 2)), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnReduction target) => context.GetArgumentShape(target, NcnnReduction.Input); + + private IRType Visit(NcnnReduction reduction, TensorType input) + { + var newInput = new TensorType(input.DType, input.Shape.InsertAndClone(0, 1)); + var axis = reduction.Args.Axes; + + var newAxis = axis.Select(x => x > 0 ? x + 1 : x).ToArray(); + + var output_ = TypeInference.ReduceType(newInput, reduction.Args.Keepdims, newAxis); + if (output_ is TensorType t) + { + var newShape = t.Shape.ToArray(); + return new TensorType(t.DType, newShape[1..]); + } + + return output_; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 22511dfd53..78b0ba081f 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -64,4 +64,7 @@ public static Call NcnnPooling(Expr expr, PoolingArgs poolingArgs) => public static Call NcnnPReLU(Expr expr, float[] slope) => new Call(new NcnnPReLU(slope), expr); + + public static Call NcnnReduction(Expr expr, ReductionArgs reductionArgs) => + new Call(new NcnnReduction(reductionArgs), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReduction.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReduction.cs new file mode 100644 index 0000000000..057f3bdf74 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReduction.cs @@ -0,0 +1,36 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.ArgsStruct; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Reduction expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnReduction : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnReduction), 0, "input"); + + /// + /// Gets Slope of Ncnn Reduction. + /// + public ReductionArgs Args { get; } + + /// + public override string DisplayProperty() + { + return $"opType:{Args.OpType}, reduceAll:{Args.ReduceAll}, coeff:{Args.Coeff}, axes:{string.Join(",", Args.Axes)}, keepdims:{Args.Keepdims}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs index 6d0f8b1849..ae4804f621 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs @@ -48,7 +48,7 @@ public partial class LowerPooling : RewriteRule { ReduceOp.Max => 0, ReduceOp.Mean => 1, - _ => throw new NotImplementedException($"{pdp.ReduceOp} not suppor in ncnn!"), + _ => throw new NotImplementedException($"{pdp.ReduceOp} not support in ncnn!"), }; var kernel_ = filter.Evaluate().AsTensor().ToArray(); diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReduction.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReduction.cs new file mode 100644 index 0000000000..ef09adf357 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReduction.cs @@ -0,0 +1,210 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; +using Nncase.ArgsStruct; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.IR.NN; +using Nncase.PatternMatch; +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Math; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerReduction : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsReduce( + "reduce", + "reduceCall", + _ => true, + IsWildcard("input") with { TypePattern = IsFloat() }, + IsTensorConst("axis"), + IsTensorConst("initValue"), + IsTensorConst("keepDims")); + + public virtual int GetOpT() + { + return -1; + } + + private Expr? GetReplace(Reduce reduce, Expr input, long[] axis, Expr initValue, bool keepDims) + { + // TODO: split input + if ((input.CheckedShape.ToList()[0] != 1 && input.CheckedShape.Rank == 4) || input.CheckedShape.Rank > 4 || input.CheckedDataType == DataTypes.Float16) + { + return null; + } + + var otherOpType = GetOpT(); + var reductionType = otherOpType == -1 + ? reduce.ReduceOp switch + { + ReduceOp.Sum => 0, + ReduceOp.Mean => 3, + ReduceOp.Max => 4, + ReduceOp.Min => 5, + ReduceOp.Prod => 6, + _ => throw new NotImplementedException($"{reduce.ReduceOp} not support in ncnn!"), + } + : otherOpType; + + var newAxis = axis; + if (axis.Length == input.CheckedShape.Rank) + { + // 排除batch维度 + newAxis = newAxis.Remove(0); + newAxis = newAxis.Remove(-input.CheckedShape.Rank); + } + + for (int i = 0; i < newAxis.Length; i++) + { + if (newAxis[i] == 0 || newAxis[i] > 4 || newAxis[i] < -3) + { + return null; + } + + newAxis[i] = newAxis[i] > 0 ? newAxis[i] - 1 : newAxis[i]; + } + + var args = new ReductionArgs(reductionType, axis.Length > 0 ? 0 : 1, 0, newAxis, keepDims ? 1 : 0); + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var pool = new Call(new Fusion("ncnn", NcnnReduction(inResO, args), new[] { inResO }), inRes); + return Unsqueeze(pool, new[] { 0 }); + } +} + +[RuleGenerator] +public partial class LowerReductionSumSquare : LowerReduction +{ + /// + public override Pattern Pattern { get; } = + IsReduce( + "reduce", + "reduceCall", + ReduceOp.Sum, + IsUnary( + "square", + UnaryOp.Square, + IsWildcard("input") with { TypePattern = IsFloat() }), + IsTensorConst("axis"), + IsTensorConst("initValue"), + IsTensorConst("keepDims")); + + public override int GetOpT() + { + return 2; + } +} + +[RuleGenerator] +public partial class LowerReductionL1 : LowerReduction +{ + /// + public override Pattern Pattern { get; } = + IsReduce( + "reduce", + "reduceCall", + ReduceOp.Sum, + IsUnary( + "abs", + UnaryOp.Abs, + IsWildcard("input") with { TypePattern = IsFloat() }), + IsTensorConst("axis"), + IsTensorConst("initValue"), + IsTensorConst("keepDims")); + + public override int GetOpT() + { + return 7; + } +} + +[RuleGenerator] +public class LowerReductionL2 : LowerReduction +{ + /// + public override Pattern Pattern { get; } = + IsUnary( + "sqrt", + UnaryOp.Sqrt, + IsReduce( + "reduce", + "reduceCall", + ReduceOp.Sum, + IsUnary( + "square", + UnaryOp.Square, + IsWildcard("input") with { TypePattern = IsFloat() }), + IsTensorConst("axis"), + IsTensorConst("initValue"), + IsTensorConst("keepDims"))); + + public override int GetOpT() + { + return 8; + } +} + +[RuleGenerator] +public partial class LowerReductionLogSum : LowerReduction +{ + /// + public override Pattern Pattern { get; } = + IsUnary( + "log", + UnaryOp.Log, + IsReduce( + "reduce", + "reduceCall", + ReduceOp.Sum, + IsWildcard("input") with { TypePattern = IsFloat() }, + IsTensorConst("axis"), + IsTensorConst("initValue"), + IsTensorConst("keepDims"))); + + public override int GetOpT() + { + return 9; + } +} + +[RuleGenerator] +public partial class LowerReductionLogSumExp : LowerReduction +{ + /// + public override Pattern Pattern { get; } = + IsUnary( + "log", + UnaryOp.Log, + IsReduce( + "reduce", + "reduceCall", + ReduceOp.Sum, + IsUnary( + "exp", + UnaryOp.Exp, + IsWildcard("input") with { TypePattern = IsFloat() }), + IsTensorConst("axis"), + IsTensorConst("initValue"), + IsTensorConst("keepDims"))); + + public override int GetOpT() + { + return 10; + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 7015c06911..488c9cea21 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -49,10 +49,19 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions { passManager.AddWithName("LowerNcnnIR").Configure(p => { + p.Add(); + + // Fold reduce + p.Add(); + p.Add(); + p.Add(); + p.Add(); + p.Add(); + + // single op p.Add(); p.Add(); p.Add(); - p.Add(); p.Add(); // p.Add(); //0816ncnn not support @@ -70,7 +79,8 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); - p.Add(); + p.Add(); + p.Add(); }); } From 62b05f357ad9650057186c448ef9fe97e03d3c06 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Wed, 7 Feb 2024 18:39:00 +0800 Subject: [PATCH 063/132] Reshape -->[ncnn op] Reshape --- .../CodeGen/Ncnn/NcnnEmitter.cs | 19 +++++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Evaluator/Ncnn/NcnnReshape.cs | 82 +++++++++++++++++++ .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../IR/Ncnn/NcnnReshape.cs | 35 ++++++++ .../Passes/Rules/Ncnn/LowerPReLU.cs | 2 - .../Passes/Rules/Ncnn/LowerReshape.cs | 37 +++++++++ .../Targets/CPUTarget.cs | 1 + 9 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReshape.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReshape.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReshape.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 02f8e137d6..46ab189923 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using DryIoc.ImTools; using Microsoft.Toolkit.HighPerformance; using Nncase.ArgsStruct; using Nncase.IR; @@ -298,6 +299,24 @@ public void Reduction(string[] name, string input, ReductionArgs reductionArgs) AddLayer("Reduction", name[0], new[] { input }, name, args); } + public void Reshape(string[] name, string input, int[] newshape) + { + if (newshape.Length == 4) + { + newshape[2] *= newshape[3]; + newshape.RemoveAt(3); + } + + var args = new ParamDict(); + const int i = 0; + foreach (var item in newshape.Reverse()) + { + args.Add(i, new ParamValue { Kind = ParamKind.Int, IntValue = item }); + } + + AddLayer("Reshape", name[0], new[] { input }, name, args); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index cbd1d64d72..8f3a924f6e 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -174,6 +174,9 @@ protected override string VisitLeafCall(Call expr) case NcnnReduction op: _emitter.Reduction(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Args); break; + case NcnnReshape op: + _emitter.Reshape(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Shape); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 7de97b68ad..69a4f66dcd 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -33,5 +33,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReshape.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReshape.cs new file mode 100644 index 0000000000..4083749b91 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReshape.cs @@ -0,0 +1,82 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnReshapeEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnReshape reshape) + { + var input = context.GetOrtArgumentValue(reshape, NcnnReshape.Input); + var newShape = reshape.Shape; + return OrtKI.Reshape(input, newShape, 0).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnReshape target) + { + var input = context.CheckArgumentType(target, NcnnReshape.Input); + var newShape = target.Shape; + return Visit(input, newShape); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnReshape target) + { + return new() + { + [CostFactorNames.CPUCycles] = 1, + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnReshape target) + { + _ = context.GetArgumentType(target, NcnnReshape.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = 0, + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnReshape target) => context.GetArgumentShape(target, NcnnReshape.Input); + + private IRType Visit(TensorType input, int[] newShape) + { + var inputSize = input.Shape.Aggregate((a, b) => a * b).FixedValue; + var outputShape = newShape; + int negAxis = -1; + for (int i = 0; i < newShape.Length; i++) + { + if (newShape[i] == -1) + { + negAxis = i; + } + } + + if (negAxis != -1) + { + int otherSize = newShape.Aggregate(1, (currentProduct, num) => num == 0 ? currentProduct : currentProduct * (num > 0 ? num : 1)); + outputShape[negAxis] = inputSize / otherSize; + } + + return new TensorType(input.DType, outputShape); + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 78b0ba081f..b7a012ed9c 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -67,4 +67,7 @@ public static Call NcnnPReLU(Expr expr, float[] slope) => public static Call NcnnReduction(Expr expr, ReductionArgs reductionArgs) => new Call(new NcnnReduction(reductionArgs), expr); + + public static Call NcnnReshape(Expr expr, int[] shape) => + new Call(new NcnnReshape(shape), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReshape.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReshape.cs new file mode 100644 index 0000000000..9db6d0bd7c --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReshape.cs @@ -0,0 +1,35 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Reshape expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnReshape : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnReshape), 0, "input"); + + /// + /// Gets Slope of Ncnn Reshape. + /// + public int[] Shape { get; } + + /// + public override string DisplayProperty() + { + return $"shape:{string.Join(",", Shape)}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPReLU.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPReLU.cs index df06a62d66..afafc54dae 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPReLU.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPReLU.cs @@ -25,8 +25,6 @@ public partial class LowerPReLU : RewriteRule { /// public override Pattern Pattern { get; } = IsPRelu( - - // "prelu", IsWildcard("input"), IsTensorConst("slope")); diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReshape.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReshape.cs new file mode 100644 index 0000000000..d5fa17ba02 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReshape.cs @@ -0,0 +1,37 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using Nncase.IR; +using Nncase.PatternMatch; +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerReshape : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsReshape( + IsWildcard("input"), + IsTensorConst("shape")); + + private Expr? GetReplace(Expr input, Expr shape) + { + // TODO: split input + // if shape.Length == input.Length means that batchSize need reshape. Ncnn can't support. + if (input.CheckedShape.ToList()[0] != 1 || input.CheckedShape.Count == shape.CheckedShape.Count) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + var outputShape = shape.Evaluate().AsTensor().ToArray(); + var reshape = new Call(new Fusion("ncnn", NcnnReshape(inResO, outputShape), new[] { inResO }), input); + + return reshape; + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 488c9cea21..ee631cd93f 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -81,6 +81,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); }); } From 24d81b81c5683efd9633b6fb08f0f277e278eab6 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 8 Feb 2024 12:13:47 +0800 Subject: [PATCH 064/132] SELU -->[ncnn op] SELU --- .../CodeGen/Ncnn/NcnnEmitter.cs | 9 +++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Evaluator/Ncnn/NcnnSELU.cs | 70 +++++++++++++++++++ .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnCelu.cs | 2 +- .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnElu.cs | 2 +- .../IR/Ncnn/NcnnHardSigmoid.cs | 4 +- .../IR/Ncnn/NcnnHardSwish.cs | 4 +- .../IR/Ncnn/NcnnReduction.cs | 2 +- .../IR/Ncnn/NcnnReshape.cs | 2 +- .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnSELU.cs | 40 +++++++++++ .../Passes/Rules/Ncnn/LowerSELU.cs | 40 +++++++++++ .../Targets/CPUTarget.cs | 1 + 14 files changed, 175 insertions(+), 8 deletions(-) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSELU.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSELU.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSELU.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 46ab189923..ae264d5efe 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -317,6 +317,15 @@ public void Reshape(string[] name, string input, int[] newshape) AddLayer("Reshape", name[0], new[] { input }, name, args); } + public void SELU(string[] name, string input, float alpha, float gamma) + { + AddLayer("SELU", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Float, FloatValue = alpha }, + [1] = new ParamValue { Kind = ParamKind.Float, FloatValue = gamma }, + }); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 8f3a924f6e..f80daa6b69 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -177,6 +177,9 @@ protected override string VisitLeafCall(Call expr) case NcnnReshape op: _emitter.Reshape(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Shape); break; + case NcnnSELU op: + _emitter.SELU(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Alpha, op.Gamma); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 69a4f66dcd..fb7ab2b56d 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -34,5 +34,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSELU.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSELU.cs new file mode 100644 index 0000000000..1b9e4f07e1 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSELU.cs @@ -0,0 +1,70 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnSELUEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnSELU selu) + { + var input = context.GetOrtArgumentValue(selu, NcnnSELU.Input); + var alpha = selu.Alpha; + var gamma = selu.Gamma; + return OrtKI.Selu(input, alpha, gamma).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnSELU target) + { + var input = context.CheckArgumentType(target, NcnnSELU.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnSELU target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnSELU target) + { + var inputType = context.GetArgumentType(target, NcnnSELU.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = + MetricUtility.GetFLOPs(inputType) * (MetricUtility.ExpFLOPs + (MetricUtility.MulFLOPs * 2) + + MetricUtility.SubFLOPs + MetricUtility.AddFLOPs + + (MetricUtility.CmpFLOPs * 2)), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnSELU target) => context.GetArgumentShape(target, NcnnSELU.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index b7a012ed9c..1ff343e81e 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -70,4 +70,7 @@ public static Call NcnnReduction(Expr expr, ReductionArgs reductionArgs) => public static Call NcnnReshape(Expr expr, int[] shape) => new Call(new NcnnReshape(shape), expr); + + public static Call NcnnSELU(Expr expr, float alpha, float gamma) => + new Call(new NcnnSELU(alpha, gamma), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCelu.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCelu.cs index 7629b737cb..2755fc7d35 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCelu.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCelu.cs @@ -23,7 +23,7 @@ public sealed partial class NcnnCelu : Op public static readonly ParameterInfo Input = new(typeof(NcnnCelu), 0, "input"); /// - /// Gets axis of Ncnn Celu. + /// Gets Alpha of Ncnn Celu. /// public float Alpha { get; } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnElu.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnElu.cs index d901549c2f..48f285306e 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnElu.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnElu.cs @@ -23,7 +23,7 @@ public sealed partial class NcnnElu : Op public static readonly ParameterInfo Input = new(typeof(NcnnElu), 0, "input"); /// - /// Gets axis of Ncnn Elu. + /// Gets Alpha of Ncnn Elu. /// public float Alpha { get; } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSigmoid.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSigmoid.cs index 2a3f6bd682..4843fa26b6 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSigmoid.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSigmoid.cs @@ -23,12 +23,12 @@ public sealed partial class NcnnHardSigmoid : Op public static readonly ParameterInfo Input = new(typeof(NcnnHardSigmoid), 0, "input"); /// - /// Gets axis of Ncnn HardSigmoid. + /// Gets Alpha of Ncnn HardSigmoid. /// public float Alpha { get; } /// - /// Gets axis of Ncnn HardSigmoid. + /// Gets Beta of Ncnn HardSigmoid. /// public float Beta { get; } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSwish.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSwish.cs index aee45da3de..4ae10054cc 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSwish.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnHardSwish.cs @@ -23,12 +23,12 @@ public sealed partial class NcnnHardSwish : Op public static readonly ParameterInfo Input = new(typeof(NcnnHardSwish), 0, "input"); /// - /// Gets axis of Ncnn HardSwish. + /// Gets Alpha of Ncnn HardSwish. /// public float Alpha { get; } /// - /// Gets axis of Ncnn HardSwish. + /// Gets Beta of Ncnn HardSwish. /// public float Beta { get; } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReduction.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReduction.cs index 057f3bdf74..4cfd9ba943 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReduction.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReduction.cs @@ -24,7 +24,7 @@ public sealed partial class NcnnReduction : Op public static readonly ParameterInfo Input = new(typeof(NcnnReduction), 0, "input"); /// - /// Gets Slope of Ncnn Reduction. + /// Gets ReductionArgs of Ncnn Reduction. /// public ReductionArgs Args { get; } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReshape.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReshape.cs index 9db6d0bd7c..6988d22673 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReshape.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnReshape.cs @@ -23,7 +23,7 @@ public sealed partial class NcnnReshape : Op public static readonly ParameterInfo Input = new(typeof(NcnnReshape), 0, "input"); /// - /// Gets Slope of Ncnn Reshape. + /// Gets Shape of Ncnn Reshape. /// public int[] Shape { get; } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSELU.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSELU.cs new file mode 100644 index 0000000000..8743b3488b --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSELU.cs @@ -0,0 +1,40 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// SELU expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnSELU : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnSELU), 0, "input"); + + /// + /// Gets Alpha of Ncnn SELU. + /// + public float Alpha { get; } + + /// + /// Gets Gamma of Ncnn SELU. + /// + public float Gamma { get; } + + /// + public override string DisplayProperty() + { + return $"Alpha:{Alpha}, Gamma:{Gamma}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSELU.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSELU.cs new file mode 100644 index 0000000000..e54e5b01c9 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSELU.cs @@ -0,0 +1,40 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerSELU : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsSelu( + IsWildcard("input"), + IsTensorConst("alpha"), + IsTensorConst("gamma")); + + private Expr? GetReplace(Expr input, float alpha, float gamma) + { + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var selu = new Call(new Fusion("ncnn", NcnnSELU(inResO, alpha, gamma), new[] { inResO }), inRes); + + return Unsqueeze(selu, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index ee631cd93f..7144b4938c 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -82,6 +82,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); }); } From 066356d5fa7edecfa1a7c0bebf923099d7a4ecb2 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 8 Feb 2024 12:29:10 +0800 Subject: [PATCH 065/132] add paramdict func --- src/Nncase.Core/Runtime/Ncnn/ParamDict.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs b/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs index d4bd445871..ee3ae1aa42 100644 --- a/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs +++ b/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs @@ -45,6 +45,11 @@ public ParamValue this[int index] set => _values[index] = value; } + public void Add(int index, ParamValue paramValue) + { + _values[index] = paramValue; + } + public void LoadFrom(ReadOnlySpan fields) { foreach (var field in fields) From f75351cbd434f46f286210c284078802e58d337f Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 19 Feb 2024 13:55:21 +0800 Subject: [PATCH 066/132] Sigmoid --> [ncnn op] Sigmoid --- .../CodeGen/Ncnn/NcnnEmitter.cs | 5 ++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Evaluator/Ncnn/NcnnSigmoid.cs | 65 +++++++++++++++++++ .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../IR/Ncnn/NcnnSigmoid.cs | 30 +++++++++ .../Passes/Rules/Ncnn/LowerSigmoid.cs | 34 ++++++++++ .../Targets/CPUTarget.cs | 1 + 8 files changed, 142 insertions(+) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSigmoid.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSigmoid.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSigmoid.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index ae264d5efe..8530f6597f 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -326,6 +326,11 @@ public void SELU(string[] name, string input, float alpha, float gamma) }); } + public void Sigmoid(string[] name, string input) + { + AddLayer("Sigmoid", name[0], new[] { input }, name); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index f80daa6b69..d09d8ea029 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -180,6 +180,9 @@ protected override string VisitLeafCall(Call expr) case NcnnSELU op: _emitter.SELU(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Alpha, op.Gamma); break; + case NcnnSigmoid op: + _emitter.Sigmoid(names.ToArray(), ExprMemo[expr.Arguments[0]]); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index fb7ab2b56d..6dc3ca7908 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -35,5 +35,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSigmoid.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSigmoid.cs new file mode 100644 index 0000000000..6c5184a934 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSigmoid.cs @@ -0,0 +1,65 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnSigmoidEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnSigmoid sigmoid) + { + var input = context.GetOrtArgumentValue(sigmoid, NcnnSigmoid.Input); + return OrtKI.Sigmoid(input).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnSigmoid target) + { + var input = context.CheckArgumentType(target, NcnnSigmoid.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnSigmoid target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnSigmoid target) + { + var inputType = context.GetArgumentType(target, NcnnSigmoid.Input); + + return new() + { + // y = 1 / (1 + exp(-x)). + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.AddFLOPs + MetricUtility.ExpFLOPs + MetricUtility.DivFLOPs + MetricUtility.MulFLOPs), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnSigmoid target) => context.GetArgumentShape(target, NcnnSigmoid.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 1ff343e81e..1757812b55 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -73,4 +73,7 @@ public static Call NcnnReshape(Expr expr, int[] shape) => public static Call NcnnSELU(Expr expr, float alpha, float gamma) => new Call(new NcnnSELU(alpha, gamma), expr); + + public static Call NcnnSigmoid(Expr expr) => + new Call(new NcnnSigmoid(), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSigmoid.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSigmoid.cs new file mode 100644 index 0000000000..df65b4b1b6 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSigmoid.cs @@ -0,0 +1,30 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Sigmoid expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnSigmoid : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnSigmoid), 0, "input"); + + /// + public override string DisplayProperty() + { + return $""; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSigmoid.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSigmoid.cs new file mode 100644 index 0000000000..94e46aedc8 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSigmoid.cs @@ -0,0 +1,34 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerSigmoid : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsSigmoid( + IsWildcard("input")); + + private Expr? GetReplace(Expr input) + { + var newInput = new Var(input.CheckedType); + return new Call(new Fusion("ncnn", NcnnSigmoid(newInput), new[] { newInput }), input); + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 7144b4938c..aebb64d5c7 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -83,6 +83,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); }); } From d603a6a56d2f4e2c69e7ebb5c20e674b3c8cf65c Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Tue, 20 Feb 2024 15:53:15 +0800 Subject: [PATCH 067/132] Slice --> [ncnn op] Crop --- .../Nncase.Modules.Ncnn/ArgsStruct/args.cs | 84 +++++++++++++++++-- .../CodeGen/Ncnn/NcnnEmitter.cs | 25 ++++++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnCrop.cs | 75 +++++++++++++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnCrop.cs | 37 ++++++++ .../Passes/Rules/Ncnn/LowerCrop.cs | 78 +++++++++++++++++ .../Targets/CPUTarget.cs | 1 + tests/importer/onnx_/basic/test_slice.py | 29 +++---- 10 files changed, 310 insertions(+), 26 deletions(-) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCrop.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCrop.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCrop.cs diff --git a/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs index 086750971d..b8ec204623 100644 --- a/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs +++ b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs @@ -2,16 +2,15 @@ // Licensed under the Apache license. See LICENSE file in the project root for full license information. namespace Nncase.ArgsStruct; + /// /// Ncnn Pooling arguments. /// public record PoolingArgs { - public PoolingArgs() - { - } - - public PoolingArgs(int poolingType, int kernelW, int kernelH, int strideW, int strideH, int padLeft, int padRight, int padTop, int padBottom, bool globalPooling, int padMode, bool avgPoolCountIncludePad, bool adaptivePooling, int outW, int outH, bool ceilMode) + public PoolingArgs(int poolingType, int kernelW, int kernelH, int strideW, int strideH, int padLeft, int padRight, + int padTop, int padBottom, bool globalPooling, int padMode, bool avgPoolCountIncludePad, bool adaptivePooling, + int outW, int outH, bool ceilMode) { PoolingType = poolingType; KernelH = kernelH; @@ -66,10 +65,6 @@ public PoolingArgs(int poolingType, int kernelW, int kernelH, int strideW, int s public record ReductionArgs { - public ReductionArgs() - { - } - public ReductionArgs(int opType, int reduceAll, float coeff, long[] axes, int keepdims) { OpType = opType; @@ -89,3 +84,74 @@ public ReductionArgs(int opType, int reduceAll, float coeff, long[] axes, int ke public int Keepdims { get; } } + +public record CropArgs +{ + public CropArgs(int[] starts, int[] ends, int[] axes) + { + Woffset = 0; + Hoffset = 0; + Doffset = 0; + Coffset = 0; + Outw = 0; + Outh = 0; + Outd = 0; + Outc = 0; + Woffset2 = 0; + Hoffset2 = 0; + Doffset2 = 0; + Coffset2 = 0; + Starts = starts; + Ends = ends; + Axes = axes; + } + + public CropArgs(int woffset, int hoffset, int doffset, int coffset, int outw, int outh, int outd, int outc, int woffset2, int hoffset2, int doffset2, int coffset2) + { + Woffset = woffset; + Hoffset = hoffset; + Doffset = doffset; + Coffset = coffset; + Outw = outw; + Outh = outh; + Outd = outd; + Outc = outc; + Woffset2 = woffset2; + Hoffset2 = hoffset2; + Doffset2 = doffset2; + Coffset2 = coffset2; + Starts = null; + Ends = null; + Axes = null; + } + + public int Woffset { get; } + + public int Hoffset { get; } + + public int Doffset { get; } + + public int Coffset { get; } + + public int Outw { get; } + + public int Outh { get; } + + public int Outd { get; } + + public int Outc { get; } + + public int Woffset2 { get; } + + public int Hoffset2 { get; } + + public int Doffset2 { get; } + + public int Coffset2 { get; } + + public int[]? Starts { get; } + + public int[]? Ends { get; } + + public int[]? Axes { get; } +} diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 8530f6597f..1c0a348f97 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -326,6 +326,31 @@ public void SELU(string[] name, string input, float alpha, float gamma) }); } + public void Crop(string[] name, string input, CropArgs cropArgs) + { + + var args = new ParamDict(); + // TODO: if need to fit torch crop, add other args into paramDict. + if (cropArgs.Axes.Length > 0) + { + var startData = new List { cropArgs.Axes.Length }; + var endData = new List { cropArgs.Axes.Length }; + var axisData = new List { cropArgs.Axes.Length }; + for (int i = 0; i < cropArgs.Axes.Length; i++) + { + startData.Add(cropArgs.Starts[i]); + endData.Add(cropArgs.Ends[i]); + axisData.Add(cropArgs.Axes[i]); + } + + args.Add(-9, new ParamValue { Kind = ParamKind.ArrayOfInt, TensorValue = startData.ToArray() }); + args.Add(-10, new ParamValue { Kind = ParamKind.ArrayOfInt, TensorValue = endData.ToArray() }); + args.Add(-11, new ParamValue { Kind = ParamKind.ArrayOfInt, TensorValue = axisData.ToArray() }); + } + + AddLayer("Crop", name[0], new[] { input }, name, args); + } + public void Sigmoid(string[] name, string input) { AddLayer("Sigmoid", name[0], new[] { input }, name); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index d09d8ea029..f93beb1103 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -183,6 +183,9 @@ protected override string VisitLeafCall(Call expr) case NcnnSigmoid op: _emitter.Sigmoid(names.ToArray(), ExprMemo[expr.Arguments[0]]); break; + case NcnnCrop op: + _emitter.Crop(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Args); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCrop.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCrop.cs new file mode 100644 index 0000000000..627fb08151 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCrop.cs @@ -0,0 +1,75 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.ArgsStruct; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnCropEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnCrop crop) + { + var input = context.GetOrtArgumentValue(crop, NcnnCrop.Input); + var starts = crop.Args.Starts; + var ends = crop.Args.Ends; + var axes = crop.Args.Axes; + var steps = Enumerable.Repeat(1, (starts ?? Array.Empty()).Length).ToArray(); + return OrtKI.Slice(input, starts ?? Array.Empty(), ends ?? Array.Empty(), axes ?? Array.Empty(), steps).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnCrop target) + { + var input = context.CheckArgumentType(target, NcnnCrop.Input); + return Visit(input, target.Args); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnCrop target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnCrop target) + { + var inputType = context.GetArgumentType(target, NcnnCrop.Input); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) * 2, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnCrop target) => context.GetArgumentShape(target, NcnnCrop.Input); + + private IRType Visit(TensorType input, CropArgs args) + { + var outputShape = input.Shape.ToArray(); + for (int i = 0; i < args.Axes.Length; i++) + { + var tStart = args.Starts[i] >= 0 ? args.Starts[i] : args.Starts[i] + outputShape[args.Axes[i]].FixedValue; + var tEnd = args.Ends[i] >= 0 ? args.Ends[i] : args.Ends[i] + outputShape[args.Axes[i]].FixedValue; + outputShape[args.Axes[i] < 0 ? args.Axes[i] + outputShape.Length : args.Axes[i]] = System.Math.Abs(tEnd - tStart); + } + + return new TensorType(input.DType, outputShape); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 6dc3ca7908..62d6e48b9e 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -36,5 +36,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 1757812b55..cac2ee7130 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -76,4 +76,7 @@ public static Call NcnnSELU(Expr expr, float alpha, float gamma) => public static Call NcnnSigmoid(Expr expr) => new Call(new NcnnSigmoid(), expr); + + public static Call NcnnCrop(Expr expr, CropArgs args) => + new Call(new NcnnCrop(args), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCrop.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCrop.cs new file mode 100644 index 0000000000..423b605dc2 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCrop.cs @@ -0,0 +1,37 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.ArgsStruct; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Crop expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnCrop : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnCrop), 0, "input"); + + /// + /// Gets Args of Crop. + /// + public CropArgs Args { get; } + + /// + public override string DisplayProperty() + { + return + $"starts:{string.Join(",", (Args.Starts ?? Array.Empty()).Select(x => x.ToString()))}, ends: {string.Join(",", (Args.Ends ?? Array.Empty()).Select(x => x.ToString()))}, axes: {string.Join(",", (Args.Axes ?? Array.Empty()).Select(x => x.ToString()))}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCrop.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCrop.cs new file mode 100644 index 0000000000..cbd6479dd9 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCrop.cs @@ -0,0 +1,78 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; +using Nncase.ArgsStruct; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerCrop : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsSlice( + IsWildcard("input"), + IsTensorConst("starts"), + IsTensorConst("ends"), + IsTensorConst("axes"), + IsTensorConst("steps")); + + private Expr? GetReplace(Expr input, int[] starts, int[] ends, int[] axes, int[] steps) + { + if (steps.All(x => x != 1)) + { + Console.WriteLine("ncnn not support slice with step"); + return null; + } + + if (input.CheckedShape.Count > 4 || input.CheckedShape[0].FixedValue != 1) + { + Console.WriteLine("ncnn not support more than 4D or batchSize > 1"); + return null; + } + + var tStart = starts.ToList(); + var tEnds = ends.ToList(); + var tAxes = axes.ToList(); + + for (int i = 0; i < axes.Length; i++) + { + if (axes[i] == 0) + { + tStart.RemoveAt(i); + tEnds.RemoveAt(i); + tAxes.RemoveAt(i); + } + + break; + } + + for (int i = 0; i < tAxes.Count; i++) + { + tAxes[i] -= 1; + } + + var args = new CropArgs(tStart.ToArray(), tEnds.ToArray(), tAxes.ToArray()); + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + var crop = new Call(new Fusion("ncnn", NcnnCrop(inResO, args), new[] { inResO }), inRes); + return Unsqueeze(crop, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index aebb64d5c7..bd6b15aac4 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -84,6 +84,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); }); } diff --git a/tests/importer/onnx_/basic/test_slice.py b/tests/importer/onnx_/basic/test_slice.py index 0d10f1f204..9b1ddc883a 100644 --- a/tests/importer/onnx_/basic/test_slice.py +++ b/tests/importer/onnx_/basic/test_slice.py @@ -158,19 +158,15 @@ def _make_module(in_shape, start, end, axes, step, outshape, op_version, value_f return model_def -in_shapes = [ - [20, 10, 5] -] - -starts_ends_axes_steps_outshapes = [ - [[0, 0], [3, 10], [0, 1], [1, 1], [3, 10, 5]], - [[0, 0, 3], [20, 10, 4], None, None, [20, 10, 1]], - [[0, 0, 3], [20, 10, 4], [0, 1, 2], None, [20, 10, 1]], - [[1], [1000], [1], [1], [20, 9, 5]], - [[0], [-1], [1], [1], [20, 9, 5]], - [[0, 0, 3], [20, 10, 4], [0, -2, -1], None, [20, 10, 1]], - [[20, 10, 4], [0, 0, 1], [0, 1, 2], [-1, -3, -2], [19, 3, 2]], - [[-1], [-9223372036854775807], [0], [-1], [20, 10, 5]] +in_shapes_starts_ends_axes_steps_outshapes = [ + [[1, 20, 10, 5], [0, 0], [3, 10], [1, 2], [1, 1], [1, 3, 10, 5]], + [[1, 20, 10, 5], [0, 0, 0, 3], [1, 20, 10, 4], None, None, [1, 20, 10, 1]], + [[1, 20, 10, 5], [0, 0, 0, 3], [1, 20, 10, 4], [0, 1, 2, 3], None, [1, 20, 10, 1]], + [[1, 20, 10, 5], [1], [1000], [2], [1], [1, 20, 9, 5]], + [[1, 20, 10, 5], [0], [-1], [1], [1], [1, 19, 10, 5]], + [[1, 20, 10, 5], [0, 0, 3], [20, 10, 4], [-3, -2, -1], None, [1, 20, 10, 1]], + [[1, 20, 10, 5], [ 20, 10, 4], [ 0, 0, 1], [1, 2, 3], [-1, -3, -2], [1, 20, 4, 2]], + [[1, 20, 10, 5], [-1], [-9223372036854775807], [1], [-1], [1, 20, 10, 5]] ] @@ -185,11 +181,10 @@ def _make_module(in_shape, start, end, axes, step, outshape, op_version, value_f ] -@pytest.mark.parametrize('in_shape', in_shapes) -@pytest.mark.parametrize('start_end_axes_step_outshape', starts_ends_axes_steps_outshapes) +@pytest.mark.parametrize('in_shape_start_end_axis_step_outshape', in_shapes_starts_ends_axes_steps_outshapes) @pytest.mark.parametrize('op_versions_and_value_format', op_versions_and_value_formats) -def test_slice(in_shape, start_end_axes_step_outshape, op_versions_and_value_format, request): - start, end, axes, step, outshape = start_end_axes_step_outshape +def test_slice(in_shape_start_end_axis_step_outshape, op_versions_and_value_format, request): + in_shape, start, end, axes, step, outshape = in_shape_start_end_axis_step_outshape op_version, value_format = op_versions_and_value_format if op_version != 1 or (op_version == 1 and step is not None and all([x == 1 for x in step])): model_def = _make_module(in_shape, start, end, axes, step, From 3fc3c77becd7bba1832a6ec751fe9774d8dfedf1 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Tue, 20 Feb 2024 16:33:14 +0800 Subject: [PATCH 068/132] fix shape error --- .../Passes/Rules/Ncnn/LowerSigmoid.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSigmoid.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSigmoid.cs index 94e46aedc8..79fe8a7169 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSigmoid.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSigmoid.cs @@ -28,6 +28,16 @@ public partial class LowerSigmoid : RewriteRule private Expr? GetReplace(Expr input) { + if (input.CheckedShape.Count > 3) + { + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var sigmoid = new Call(new Fusion("ncnn", NcnnSigmoid(inResO), new[] { inResO }), inRes); + return Unsqueeze(sigmoid, new[] { 0 }); + } + + // if input has shape [1] var newInput = new Var(input.CheckedType); return new Call(new Fusion("ncnn", NcnnSigmoid(newInput), new[] { newInput }), input); } From 823fb98fabcaf8b9200c71f2100ee52314aa2a1c Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Tue, 20 Feb 2024 16:38:37 +0800 Subject: [PATCH 069/132] Softplus --> [ncnn op] Softplus --- .../CodeGen/Ncnn/NcnnEmitter.cs | 7 +- .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 5 +- .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Evaluator/Ncnn/NcnnSoftplus.cs | 65 +++++++++++++++++++ .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../IR/Ncnn/NcnnSoftplus.cs | 30 +++++++++ .../Passes/Rules/Ncnn/LowerSoftplus.cs | 42 ++++++++++++ .../Targets/CPUTarget.cs | 1 + 8 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSoftplus.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSoftplus.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftplus.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 1c0a348f97..e22ae03a84 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -328,8 +328,8 @@ public void SELU(string[] name, string input, float alpha, float gamma) public void Crop(string[] name, string input, CropArgs cropArgs) { - var args = new ParamDict(); + // TODO: if need to fit torch crop, add other args into paramDict. if (cropArgs.Axes.Length > 0) { @@ -356,6 +356,11 @@ public void Sigmoid(string[] name, string input) AddLayer("Sigmoid", name[0], new[] { input }, name); } + public void Softplus(string[] name, string input) + { + AddLayer("Softplus", name[0], new[] { input }, name); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index f93beb1103..1cb7530e0a 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -180,12 +180,15 @@ protected override string VisitLeafCall(Call expr) case NcnnSELU op: _emitter.SELU(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Alpha, op.Gamma); break; - case NcnnSigmoid op: + case NcnnSigmoid: _emitter.Sigmoid(names.ToArray(), ExprMemo[expr.Arguments[0]]); break; case NcnnCrop op: _emitter.Crop(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Args); break; + case NcnnSoftplus: + _emitter.Softplus(names.ToArray(), ExprMemo[expr.Arguments[0]]); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 62d6e48b9e..e1834a4f84 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -37,5 +37,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSoftplus.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSoftplus.cs new file mode 100644 index 0000000000..ff936aa387 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSoftplus.cs @@ -0,0 +1,65 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnSoftplusEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnSoftplus softplus) + { + var input = context.GetOrtArgumentValue(softplus, NcnnSoftplus.Input); + return OrtKI.Softplus(input).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnSoftplus target) + { + var input = context.CheckArgumentType(target, NcnnSoftplus.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnSoftplus target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnSoftplus target) + { + var inputType = context.GetArgumentType(target, NcnnSoftplus.Input); + + return new() + { + // y = log(exp(x)+1). + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) * 2, + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(inputType) * (MetricUtility.AddFLOPs + MetricUtility.ExpFLOPs + MetricUtility.LogFLOPs), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnSoftplus target) => context.GetArgumentShape(target, NcnnSoftplus.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index cac2ee7130..81657a4490 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -79,4 +79,7 @@ public static Call NcnnSigmoid(Expr expr) => public static Call NcnnCrop(Expr expr, CropArgs args) => new Call(new NcnnCrop(args), expr); + + public static Call NcnnSoftplus(Expr expr) => + new Call(new NcnnSoftplus(), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSoftplus.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSoftplus.cs new file mode 100644 index 0000000000..1619e28e01 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSoftplus.cs @@ -0,0 +1,30 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Softplus expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnSoftplus : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnSoftplus), 0, "input"); + + /// + public override string DisplayProperty() + { + return $""; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftplus.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftplus.cs new file mode 100644 index 0000000000..c3ff475253 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftplus.cs @@ -0,0 +1,42 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerSoftplus : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsSoftplus( + IsWildcard("input")); + + private Expr? GetReplace(Expr input) + { + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var softplus = new Call(new Fusion("ncnn", NcnnSoftplus(inResO), new[] { inResO }), inRes); + return Unsqueeze(softplus, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index bd6b15aac4..bd957f3daf 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -85,6 +85,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); }); } From dbe2c73d60aac4a8ec5f5bf63f42c02aade95796 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Wed, 21 Feb 2024 15:40:27 +0800 Subject: [PATCH 070/132] Split --> [ncnn op] Slice --- .../CodeGen/Ncnn/NcnnEmitter.cs | 11 +++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 4 + .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Evaluator/Ncnn/NcnnSlice.cs | 74 +++++++++++++++++++ .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnSlice.cs | 34 +++++++++ .../Passes/Rules/Ncnn/LowerSlice.cs | 60 +++++++++++++++ .../Targets/CPUTarget.cs | 1 + tests/importer/onnx_/basic/test_split.py | 14 ++-- 9 files changed, 196 insertions(+), 6 deletions(-) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSlice.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSlice.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSlice.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index e22ae03a84..2f69034bbd 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -361,6 +361,17 @@ public void Softplus(string[] name, string input) AddLayer("Softplus", name[0], new[] { input }, name); } + public void Slice(string[] name, string input, int[] slices, int axis) + { + var sliceData = new List { slices.Length }; + sliceData.AddRange(slices); + AddLayer("Slice", name[0], new[] { input }, name, new ParamDict + { + [-0] = new ParamValue { Kind = ParamKind.ArrayOfInt, TensorValue = sliceData.ToArray() }, + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = axis }, + }); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 1cb7530e0a..e7a8c8f48d 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -189,6 +189,10 @@ protected override string VisitLeafCall(Call expr) case NcnnSoftplus: _emitter.Softplus(names.ToArray(), ExprMemo[expr.Arguments[0]]); break; + case NcnnSlice op: + names.AddRange(op.Slices.Select(i => GetNextName())); + _emitter.Slice(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Slices, op.Axis); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index e1834a4f84..fedf2a537f 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -38,5 +38,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSlice.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSlice.cs new file mode 100644 index 0000000000..4a3fccffcb --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSlice.cs @@ -0,0 +1,74 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnSliceEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnSlice slice) + { + var input = context.GetOrtArgumentValue(slice, NcnnSlice.Input); + var split = slice.Slices; + var axis = slice.Axis; + var result = OrtKI.Split(input, split, axis); + return Value.FromTensors(result.Select(t => t.ToTensor()).ToArray()); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnSlice target) + { + var input = context.CheckArgumentType(target, NcnnSlice.Input); + return Visit(input, target.Slices, target.Axis); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnSlice target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnSlice target) + { + var inputType = context.GetArgumentType(target, NcnnSlice.Input); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) * 2, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnSlice target) => context.GetArgumentShape(target, NcnnSlice.Input); + + private IRType Visit(TensorType input, int[] slices, int axis) + { + var outputInfo = new List(); + var outputShape = new List>(); + for (int i = 0; i < slices.Length; i++) + { + outputShape.Add(input.Shape.ToValueList()); + outputShape[i][axis] = slices[i]; + outputInfo.Add(new TensorType(input.DType, outputShape[i].ToArray())); + } + + return new TupleType(outputInfo); + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 81657a4490..43f4be2f71 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -82,4 +82,7 @@ public static Call NcnnCrop(Expr expr, CropArgs args) => public static Call NcnnSoftplus(Expr expr) => new Call(new NcnnSoftplus(), expr); + + public static Call NcnnSlice(Expr expr, int[] slices, int axis) => + new Call(new NcnnSlice(slices, axis), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSlice.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSlice.cs new file mode 100644 index 0000000000..856678187d --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSlice.cs @@ -0,0 +1,34 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Slice expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnSlice : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnSlice), 0, "input"); + + public int[] Slices { get; } + + public int Axis { get; } + + /// + public override string DisplayProperty() + { + return $"slices:{string.Join(",", Slices.Select(x => x.ToString()))}, axis:{Axis}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSlice.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSlice.cs new file mode 100644 index 0000000000..1eb26a6ea5 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSlice.cs @@ -0,0 +1,60 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.IR.Tensors; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; +using Unsqueeze = Nncase.IR.Tensors.Unsqueeze; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerSlice : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsSplit( + IsWildcard("input"), + IsTensorConst("axis") with { TypePattern = IsScalar() }, + IsTensorConst("slices")); + + private Expr? GetReplace(Expr input, int[] slices, int axis) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + if (input.CheckedShape.Count <= 5 && axis != 0) + { + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + var newAxis = axis < 0 ? axis + input.CheckedShape.Count - 1 : axis - 1; + var slice = new Call(new Fusion("ncnn", NcnnSlice(inResO, slices, newAxis), new[] { inResO }), inRes); + var result = new List(); + for (int i = 0; i < slices.Length; i++) + { + result.Add(Unsqueeze(slice[i], new[] { 0 })); + } + + return new IR.Tuple(result.ToArray()); + } + + return null; + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index bd957f3daf..54e9263aba 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -86,6 +86,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); }); } diff --git a/tests/importer/onnx_/basic/test_split.py b/tests/importer/onnx_/basic/test_split.py index d1a125a1cf..9809619e1d 100644 --- a/tests/importer/onnx_/basic/test_split.py +++ b/tests/importer/onnx_/basic/test_split.py @@ -100,12 +100,13 @@ def _make_module(in_shape, axis, split, output_size, op_version, value_format): in_shapes = [ - [4, 8, 8] + [1, 4, 8, 8] ] axes = [ None, - -3 + -2, + 1 ] splits = [ @@ -113,14 +114,15 @@ def _make_module(in_shape, axis, split, output_size, op_version, value_format): [2, 2], [1, 1, 2], [1, 2, 1], - [2, 1, 1], - [1, 1, 1, 1] + [3, 2, 2, 1], + [2, 1, 1, 1,3], ] output_sizes = [ 2, 3, - 4 + 4, + 5 ] op_versions_and_value_formats = [ @@ -141,7 +143,7 @@ def _make_module(in_shape, axis, split, output_size, op_version, value_format): def test_split(in_shape, axis, split, output_size, op_versions_and_value_format, request): op_version, value_format = op_versions_and_value_format dim = axis if axis is not None else 0 - if (split is None and in_shape[dim] % output_size == 0) or (split is not None and len(split) == output_size): + if (split is None and in_shape[dim] % output_size == 0 ) or (split is not None and len(split) == output_size and sum(split) == in_shape[dim]): model_def = _make_module(in_shape, axis, split, output_size, op_version, value_format) runner = OnnxTestRunner(request.node.name) From 87ccd051f327c05d5724d8215ace58329542ad47 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Wed, 21 Feb 2024 15:43:54 +0800 Subject: [PATCH 071/132] fix scaler doesn't have shape. --- src/Nncase.Evaluator/TypeInference.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Nncase.Evaluator/TypeInference.cs b/src/Nncase.Evaluator/TypeInference.cs index bbe74d1739..519cc01115 100644 --- a/src/Nncase.Evaluator/TypeInference.cs +++ b/src/Nncase.Evaluator/TypeInference.cs @@ -323,7 +323,8 @@ public static IRType ReduceType(TensorType input, Expr keepDims, Expr axis) } } - return input with { Shape = new Shape(outShape.Where(x => x != 0)) }; + // TODO: please confirm the shape @zhen8838, all zero means scaler + return input with { Shape = outShape.All(x => x == 0) ? new Shape(new[] { 1 }) : new Shape(outShape.Where(x => x != 0)) }; } return input with { Shape = Shape.Unranked }; From 7aff638630ae23572bcc9745bc21f79cc5a5edf1 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Wed, 21 Feb 2024 16:47:15 +0800 Subject: [PATCH 072/132] Tile --> [ncnn op] Tile --- .../CodeGen/Ncnn/NcnnEmitter.cs | 10 +++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Evaluator/Ncnn/NcnnTile.cs | 69 +++++++++++++++++++ .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnTile.cs | 37 ++++++++++ .../Passes/Rules/Ncnn/LowerTile.cs | 53 ++++++++++++++ .../Targets/CPUTarget.cs | 1 + 8 files changed, 177 insertions(+) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnTile.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnTile.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerTile.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 2f69034bbd..693ba4f4b4 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -372,6 +372,16 @@ public void Slice(string[] name, string input, int[] slices, int axis) }); } + public void Tile(string[] name, string input, int[] repeats) + { + var repeatsData = new List { repeats.Length }; + repeatsData.AddRange(repeats); + AddLayer("Tile", name[0], new[] { input }, name, new ParamDict + { + [-2] = new ParamValue { Kind = ParamKind.ArrayOfInt, TensorValue = repeatsData.ToArray() }, + }); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index e7a8c8f48d..ff1218f7f1 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -193,6 +193,9 @@ protected override string VisitLeafCall(Call expr) names.AddRange(op.Slices.Select(i => GetNextName())); _emitter.Slice(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Slices, op.Axis); break; + case NcnnTile op: + _emitter.Tile(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Repeats); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index fedf2a537f..903a9ae826 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -39,5 +39,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnTile.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnTile.cs new file mode 100644 index 0000000000..af25c17621 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnTile.cs @@ -0,0 +1,69 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnTileEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnTile tile) + { + var input = context.GetOrtArgumentValue(tile, NcnnTile.Input); + return OrtKI.Tile(input, tile.Repeats).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnTile target) + { + var input = context.CheckArgumentType(target, NcnnTile.Input); + return Visit(input, target.Repeats); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnTile target) + { + var inputType = context.GetArgumentType(target, NcnnTile.Input); + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(inputType), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnTile target) + { + var inputType = context.GetArgumentType(target, NcnnTile.Input); + var ret = context.GetReturnType(); + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) + CostUtility.GetMemoryAccess(ret), + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnTile target) => context.GetArgumentShape(target, NcnnTile.Input); + + private IRType Visit(TensorType input, int[] repeats) + { + var outputShape = input.Shape.ToValueArray(); + for (int i = 0; i < repeats.Length; i++) + { + outputShape[i] *= repeats[i]; + } + + return new TensorType(input.DType, outputShape); + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 43f4be2f71..cf8d020ac8 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -85,4 +85,7 @@ public static Call NcnnSoftplus(Expr expr) => public static Call NcnnSlice(Expr expr, int[] slices, int axis) => new Call(new NcnnSlice(slices, axis), expr); + + public static Call NcnnTile(Expr expr, int[] repeats) => + new Call(new NcnnTile(repeats), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnTile.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnTile.cs new file mode 100644 index 0000000000..044366f006 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnTile.cs @@ -0,0 +1,37 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Tile expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnTile : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnTile), 0, "input"); + + // TODO: not support Tile with axis. + + /// + /// Gets repeats of NcnnTile. + /// + public int[] Repeats { get; } + + /// + public override string DisplayProperty() + { + return $"repeats:{string.Join(",", Repeats)}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerTile.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerTile.cs new file mode 100644 index 0000000000..72ab225b1e --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerTile.cs @@ -0,0 +1,53 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerTile : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsTile( + IsWildcard("input"), + IsTensorConst("repeats")); + + private Expr? GetReplace(Expr input, int[] repeats) + { + // TODO: split input + if (input.CheckedShape.ToList()[0] != 1) + { + return null; + } + + // TODO: confirm d dim. + if (input.CheckedShape.Count < 5 && repeats[0] == 1) + { + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + var newRepeats = repeats[1..]; + + var tile = new Call(new Fusion("ncnn", NcnnTile(inResO, newRepeats), new[] { inResO }), inRes); + return Unsqueeze(tile, new[] { 0 }); + } + + // if repeats[0] != 1, means that input can't squeeze batchSize dim. + return null; + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 54e9263aba..fc9ee3a646 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -87,6 +87,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); }); } From 6a335053ec3bb7a39890cbb598c395a96d03b2d9 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 22 Feb 2024 16:33:09 +0800 Subject: [PATCH 073/132] Transpose --> [ncnn op] Permute --- .../CodeGen/Ncnn/NcnnEmitter.cs | 8 ++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Evaluator/Ncnn/NcnnPermute.cs | 75 +++++++++++++ .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../IR/Ncnn/NcnnPermute.cs | 40 +++++++ .../Passes/Rules/Ncnn/LowerPermute.cs | 100 ++++++++++++++++++ .../Targets/CPUTarget.cs | 1 + tests/importer/onnx_/basic/test_transpose.py | 1 + 9 files changed, 232 insertions(+) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPermute.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPermute.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPermute.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 693ba4f4b4..34a738323d 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -382,6 +382,14 @@ public void Tile(string[] name, string input, int[] repeats) }); } + public void Permute(string[] name, string input, int orderType) + { + AddLayer("Permute", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = orderType }, + }); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index ff1218f7f1..bff91872a2 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -196,6 +196,9 @@ protected override string VisitLeafCall(Call expr) case NcnnTile op: _emitter.Tile(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Repeats); break; + case NcnnPermute op: + _emitter.Permute(names.ToArray(), ExprMemo[expr.Arguments[0]], op.OrderType); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 903a9ae826..689855deff 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -40,5 +40,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPermute.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPermute.cs new file mode 100644 index 0000000000..9dbd499bd7 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPermute.cs @@ -0,0 +1,75 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnPermuteEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnPermute permute) + { + var input = context.GetOrtArgumentValue(permute, NcnnPermute.Input); + return OrtKI.Transpose(input, permute.Perm.ToLongs()).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnPermute target) + { + var input = context.CheckArgumentType(target, NcnnPermute.Input); + return Visit(input, target.Perm); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnPermute target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnPermute target) + { + var inputType = context.GetArgumentType(target, NcnnPermute.Input); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) * 2, + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnPermute target) => context.GetArgumentShape(target, NcnnPermute.Input); + + private IRType Visit(TensorType input, int[] perm) + { + var outputShape = input.Shape.ToValueList(); + if (perm.Length - input.Shape.Count == 1) + { + perm = perm.Remove(0); + var realPerm = perm.Select(x => x - 1).ToArray(); + for (int i = 0; i < realPerm.Length; i++) + { + outputShape[i] = input.Shape.ToValueList()[realPerm[i]]; + } + } + + return new TensorType(input.DType, outputShape.ToArray()); + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index cf8d020ac8..610a422508 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -88,4 +88,7 @@ public static Call NcnnSlice(Expr expr, int[] slices, int axis) => public static Call NcnnTile(Expr expr, int[] repeats) => new Call(new NcnnTile(repeats), expr); + + public static Call NcnnPermute(Expr expr, int orderType, int[] perm) => + new Call(new NcnnPermute(orderType, perm), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPermute.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPermute.cs new file mode 100644 index 0000000000..3920f0b7fc --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPermute.cs @@ -0,0 +1,40 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Permute expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnPermute : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnPermute), 0, "input"); + + /// + /// Gets OrderType of NcnnPermute. + /// + public int OrderType { get; } + + /// + /// Gets perm of source transpose used to eval. + /// + public int[] Perm { get; } + + /// + public override string DisplayProperty() + { + return $"OrderType:{OrderType}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPermute.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPermute.cs new file mode 100644 index 0000000000..444f6b5777 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPermute.cs @@ -0,0 +1,100 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerPermute : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsTranspose( + IsWildcard("input"), + IsTensorConst("perm")); + + private Expr? GetReplace(Expr input, int[] perm) + { + if (input.CheckedShape[0] != 1 || input.CheckedShape.Rank > 5 || perm.Length == 2) + { + return null; + } + + int orderType = 0; + if (perm.Length == 3) + { + if (perm.Skip(1).SequenceEqual(new[] { 1, 2 })) + { + orderType = 0; + } + else if (perm.Skip(1).SequenceEqual(new[] { 2, 1 })) + { + orderType = 1; + } + else if (perm.SequenceEqual(new[] { 1, 0, 2 })) + { + orderType = 0; + } + else if (perm.SequenceEqual(new[] { 2, 0, 1 })) + { + orderType = 1; + } + else + { + return null; + } + } + else if (perm.Length == 4) + { + if (perm.Skip(1).SequenceEqual(new[] { 1, 2, 3 })) + { + orderType = 0; + } + else if (perm.Skip(1).SequenceEqual(new[] { 1, 3, 2 })) + { + orderType = 1; + } + else if (perm.Skip(1).SequenceEqual(new[] { 2, 1, 3 })) + { + orderType = 2; + } + else if (perm.Skip(1).SequenceEqual(new[] { 2, 3, 1 })) + { + orderType = 3; + } + else if (perm.Skip(1).SequenceEqual(new[] { 3, 1, 2 })) + { + orderType = 4; + } + else if (perm.Skip(1).SequenceEqual(new[] { 3, 2, 1 })) + { + orderType = 5; + } + else + { + return null; + } + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + var permute = new Call(new Fusion("ncnn", NcnnPermute(inResO, orderType, perm), inResO), inRes); + return Unsqueeze(permute, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index fc9ee3a646..48198773dd 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -88,6 +88,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); }); } diff --git a/tests/importer/onnx_/basic/test_transpose.py b/tests/importer/onnx_/basic/test_transpose.py index e22d231185..20b6fa576d 100644 --- a/tests/importer/onnx_/basic/test_transpose.py +++ b/tests/importer/onnx_/basic/test_transpose.py @@ -63,6 +63,7 @@ def _make_module(in_shape, axis, op_version): in_shapes = [ [16, 16], [3, 16, 16], + [1, 8, 16], [1, 3, 16, 16] ] From c8970983865f02f63b89a734e813fcf46bb38704 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 26 Feb 2024 15:35:58 +0800 Subject: [PATCH 074/132] MatMul --> [ncnn op] MatMul --- .../CodeGen/Ncnn/NcnnEmitter.cs | 38 +++++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 13 +- .../Evaluator/Ncnn/NcnnMatmul.cs | 146 ++++++++++++++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnMatMul.cs | 50 ++++++ .../Passes/Rules/Ncnn/LowerMatmul.cs | 61 ++++++++ .../Targets/CPUTarget.cs | 1 + 8 files changed, 312 insertions(+), 1 deletion(-) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnMatmul.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnMatMul.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerMatmul.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 34a738323d..af99e8fa3e 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -390,6 +390,44 @@ public void Permute(string[] name, string input, int orderType) }); } + public void Matmul(string name, string inputA, string inputB, int lOrR, float[] constInput, int[] constShape) + { + var inputList = new[] { inputA, inputB }; + + if (constInput != null && constShape != null) + { + if (lOrR == 1) + { + inputList[0] = name + "_memorydata"; + } + else + { + inputList[1] = name + "_memorydata"; + } + + var paramDict = new ParamDict(); + for (int i = 0; i < constShape.Length; i++) + { + int index = i switch + { + 0 => 0, + 1 => 1, + 2 when constShape.Length == 3 => 2, + 2 when constShape.Length == 4 => 11, + 3 when constShape.Length == 4 => 2, + _ => throw new NotSupportedException("Only support less than 5D"), + }; + paramDict[index] = new ParamValue { Kind = ParamKind.Int, IntValue = constShape[constShape.Length - 1 - i] }; + } + + AddLayer("MemoryData", name + "_memorydata", Array.Empty(), new[] { name + "_memorydata" }, paramDict); + + WriteFloatArray(constInput); + } + + AddLayer("MatMul", name, inputList, new[] { name }, null); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index bff91872a2..1f59cf9ab0 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -91,6 +91,7 @@ protected override string VisitLeafVar(Var expr) protected override string VisitLeafCall(Call expr) { var names = new List { GetNextName() }; + string[]? inString; switch (expr.Target) { case NcnnSoftmax op: @@ -103,7 +104,7 @@ protected override string VisitLeafCall(Call expr) _emitter.BatchNorm(names[0], ExprMemo[expr.Arguments[0]], op.Channels, op.Eps, op.SlopeData, op.MeanData, op.VarData, op.BiasData); break; case NcnnBinary op: - string[]? inString = op.LorR switch + inString = op.LorR switch { 0 => new string[] { ExprMemo[expr.Arguments[0]], ExprMemo[expr.Arguments[1]] }, 1 => new string[] { string.Empty, ExprMemo[expr.Arguments[0]] }, @@ -199,6 +200,16 @@ protected override string VisitLeafCall(Call expr) case NcnnPermute op: _emitter.Permute(names.ToArray(), ExprMemo[expr.Arguments[0]], op.OrderType); break; + case NcnnMatMul op: + inString = op.LorR switch + { + 0 => new string[] { ExprMemo[expr.Arguments[0]], ExprMemo[expr.Arguments[1]] }, + 1 => new string[] { string.Empty, ExprMemo[expr.Arguments[0]] }, + 2 => new string[] { ExprMemo[expr.Arguments[0]], string.Empty }, + _ => throw new NotImplementedException("Not found MatMul emmiter."), + }; + _emitter.Matmul(names[0], inString[0], inString[1], op.LorR, op.ConstInput, op.ConstShape); + break; default: throw new NotSupportedException(); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnMatmul.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnMatmul.cs new file mode 100644 index 0000000000..a312f8a60a --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnMatmul.cs @@ -0,0 +1,146 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.CostModel; +using Nncase.Evaluator; +using Nncase.IR; +using Nncase.IR.Ncnn; +using Nncase.Utilities; +using OrtKISharp; +using static Nncase.IR.F.Tensors; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnMatMulEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + public static IRType VisitTensorType(TensorType lhs, TensorType rhs) + { + if (lhs.Shape.IsUnranked || rhs.Shape.IsUnranked) + { + return new TensorType(lhs.DType, Shape.Unranked); + } + + // if (lhs.Shape[^1].IsUnknown || rhs.Shape[^2].IsUnknown) + // { + // return new TensorType(lhs.DType, Shape.Unranked); + // } + if (lhs.DType != rhs.DType) + { + return new InvalidType("MatMul lhs and rhs have different DType"); + } + + if (lhs.Shape[^1] != rhs.Shape[^2] && lhs.Shape[^1] != Dimension.Unknown && rhs.Shape[^2] != Dimension.Unknown) + { + return new InvalidType("MatMul lhs and rhs have not compatiable shape"); + } + + if (lhs.Shape.Count == 2 && rhs.Shape.Count == 2) + { + return new TensorType(lhs.DType, new[] { lhs.Shape[0], rhs.Shape[1] }); + } + + var lhsShape = lhs.Shape.Rank >= rhs.Shape.Rank ? lhs.Shape.ToArray() : Enumerable.Repeat((Dimension)1, rhs.Shape.Rank - lhs.Shape.Rank).Concat(lhs.Shape).ToArray(); + var rhsShape = lhs.Shape.Rank <= rhs.Shape.Rank ? rhs.Shape.ToArray() : Enumerable.Repeat((Dimension)1, lhs.Shape.Rank - rhs.Shape.Rank).Concat(rhs.Shape).ToArray(); + + var bigShape = Enumerable.Zip(lhsShape, rhsShape).SkipLast(2).Select(t => + t.First == Dimension.Unknown || t.Second == Dimension.Unknown + ? Dimension.Unknown + : System.Math.Max(t.First.FixedValue, t.Second.FixedValue)).ToArray(); + + // batch and channel + var front = bigShape; + var end = new[] { lhs.Shape[^2], rhs.Shape[^1] }; + return new TensorType(lhs.DType, front.Concat(end).ToArray()); + } + + /// + public IValue Visit(IEvaluateContext context, NcnnMatMul matMul) + { + var inputA = context.GetOrtArgumentValue(matMul, NcnnMatMul.InputA); + var inputB = context.GetOrtArgumentValue(matMul, NcnnMatMul.InputB); + + // return OrtKI. (input, dim).ToValue(); + return OrtKI.MatMul(inputA, inputB).ToValue(); + } + + public IRType Visit(NcnnMatMul target, TensorType lhs, TensorType rhs) + { + return TypeInference.BroadcastType(lhs, rhs); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnMatMul target) + { + TensorType inputA, inputB; + var dataType = target.LorR switch + { + 0 => context.CheckArgumentType(target, NcnnMatMul.InputA).DType, + 1 => context.CheckArgumentType(target, NcnnMatMul.InputB).DType, + 2 => context.CheckArgumentType(target, NcnnMatMul.InputA).DType, + _ => throw new NotSupportedException("never reach here"), + }; + + inputA = target.LorR switch + { + 1 => new TensorType(dataType, target.ConstShape), + _ => context.CheckArgumentType(target, NcnnMatMul.InputA), + }; + + inputB = target.LorR switch + { + 2 => new TensorType(inputA.DType, target.ConstShape), + _ => context.CheckArgumentType(target, NcnnMatMul.InputB), + }; + + return VisitTensorType(inputA, inputB); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnMatMul target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnMatMul target) + { + var lhs = context.GetArgumentType(target, NcnnMatMul.InputA); + var rhs = context.GetArgumentType(target, NcnnMatMul.InputB); + var outputType = context.GetReturnType(); + + var k = (UInt128)lhs.Shape[^1].FixedValue; + var m = MetricUtility.GetFLOPs(lhs) / k; + var n = MetricUtility.GetFLOPs(rhs) / k; + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(lhs) + CostUtility.GetMemoryAccess(rhs) + CostUtility.GetMemoryAccess(outputType), + [MetricFactorNames.FLOPs] = m * n * ((2 * k) - 1), + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnMatMul target) + { + var lhs = context.GetArgumentShape(target, NcnnMatMul.InputA); + var rhs = context.GetArgumentShape(target, NcnnMatMul.InputB); + return ShapeExprUtility.BroadcastShape(lhs, rhs); + } + + private Expr Visit(TensorType inputA, TensorType inputB) + { + return Cast(IR.F.ShapeExpr.MatMulShape(inputA.Shape, inputB.Shape), DataTypes.Int32); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 689855deff..b19e5599b6 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -41,5 +41,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 610a422508..b795a525b8 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -91,4 +91,7 @@ public static Call NcnnTile(Expr expr, int[] repeats) => public static Call NcnnPermute(Expr expr, int orderType, int[] perm) => new Call(new NcnnPermute(orderType, perm), expr); + + public static Call NcnnMatMul(Expr[] inExpr, int lorR, float[]? constInput, int[]? constShape) => + new Call(new NcnnMatMul(lorR, constInput, constShape), inExpr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnMatMul.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnMatMul.cs new file mode 100644 index 0000000000..b739696e63 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnMatMul.cs @@ -0,0 +1,50 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Matmul expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnMatMul : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo InputA = new(typeof(NcnnMatMul), 0, "inputA"); + + /// + /// Gets input. + /// + public static readonly ParameterInfo InputB = new(typeof(NcnnMatMul), 1, "inputB"); + + /// + /// Gets the flag of which input is const. + /// + public int LorR { get; } + + /// + /// Gets constant data. + /// + public float[] ConstInput { get; } + + /// + /// Gets shape of constant data. + /// + public int[] ConstShape { get; } + + /// + public override string DisplayProperty() + { + return $"constantShape:{string.Join(",", ConstShape ?? Array.Empty())}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerMatmul.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerMatmul.cs new file mode 100644 index 0000000000..ee453aac54 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerMatmul.cs @@ -0,0 +1,61 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Math; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerMatmul : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsMatMul( + IsWildcard("inputA") with { TypePattern = HasFixedShape() }, + IsWildcard("inputB") with { TypePattern = HasFixedShape() }); + + private Expr? GetReplace(Expr inputA, Expr inputB) + { + if (inputA is Const) + { + var constA = ((TensorConst)inputA).Value; + var constShape = inputA.CheckedShape.ToValueArray(); + + // var newB = Reshape(inputB, FixShape(inputB.CheckedShape.ToValueArray(), r)); + var newInputB = new Var(inputB.CheckedType); + return new Call(new Fusion("ncnn", NcnnMatMul(new Expr[] { newInputB }, 1, constA.ToArray(), constShape), new[] { newInputB }), inputB); + } + + if (inputB is Const) + { + // var newA = Reshape(inputA, FixShape(inputA.CheckedShape.ToValueArray(), r)); + var newInputA = new Var(inputA.CheckedType); + var constB = ((TensorConst)inputB).Value; + var constShape = inputB.CheckedShape.ToValueArray(); + return new Call(new Fusion("ncnn", NcnnMatMul(new Expr[] { newInputA }, 2, constB.ToArray(), constShape), newInputA), inputA); + } + + { + var newInputA = new Var(inputA.CheckedType); + var newInputB = new Var(inputB.CheckedType); + return new Call( + new Fusion("ncnn", NcnnMatMul(new Expr[] { newInputA, newInputB }, 0, null, null), + new[] { newInputA, newInputB }), inputA, inputB); + } + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 48198773dd..ba5311b154 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -89,6 +89,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); }); } From d9805320c0439067e46d972a889e54c91aed8b7d Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 26 Feb 2024 17:24:10 +0800 Subject: [PATCH 075/132] fix reshape --- .../Passes/Rules/Ncnn/LowerReshape.cs | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReshape.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReshape.cs index d5fa17ba02..83a6b3be32 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReshape.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReshape.cs @@ -1,6 +1,7 @@ // Copyright (c) Canaan Inc. All rights reserved. // Licensed under the Apache license. See LICENSE file in the project root for full license information. +using NetFabric.Hyperlinq; using Nncase.IR; using Nncase.PatternMatch; using static Nncase.IR.F.Ncnn; @@ -20,18 +21,28 @@ public partial class LowerReshape : RewriteRule private Expr? GetReplace(Expr input, Expr shape) { - // TODO: split input - // if shape.Length == input.Length means that batchSize need reshape. Ncnn can't support. - if (input.CheckedShape.ToList()[0] != 1 || input.CheckedShape.Count == shape.CheckedShape.Count) + if (input.CheckedShape.Count < 5) { - return null; + var inResO = new Var(input.CheckedType); + var outputShape = shape.Evaluate().AsTensor().ToArray(); + + return new Call(new Fusion("ncnn", NcnnReshape(inResO, outputShape), new[] { inResO }), input); } - var inRes = Squeeze(input, new[] { 0 }); - var inResO = new Var(inRes.CheckedType); - var outputShape = shape.Evaluate().AsTensor().ToArray(); - var reshape = new Call(new Fusion("ncnn", NcnnReshape(inResO, outputShape), new[] { inResO }), input); + return null; - return reshape; + // // TODO: split input + // // if shape.Length == input.Length means that batchSize need reshape. Ncnn can't support. + // if (input.CheckedShape.ToList()[0] != 1 || input.CheckedShape.Count == shape.CheckedShape.Count) + // { + // return null; + // } + // + // var inRes = Squeeze(input, new[] { 0 }); + // var inResO = new Var(inRes.CheckedType); + // var outputShape = shape.Evaluate().AsTensor().ToArray(); + // var reshape = new Call(new Fusion("ncnn", NcnnReshape(inResO, outputShape), new[] { inResO }), input); + // + // return reshape; } } From 9a5de03777b6a66dfd2d06374aa02e95737d4c7b Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 26 Feb 2024 17:25:02 +0800 Subject: [PATCH 076/132] update test without other op --- tests/importer/onnx_/basic/test_reshape.py | 48 ++++++++-------------- 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/tests/importer/onnx_/basic/test_reshape.py b/tests/importer/onnx_/basic/test_reshape.py index b263cb3fa6..4939e6da99 100644 --- a/tests/importer/onnx_/basic/test_reshape.py +++ b/tests/importer/onnx_/basic/test_reshape.py @@ -18,54 +18,40 @@ from onnx_test_runner import OnnxTestRunner -def _make_module(in_shape, out_channel, kernel_size): +def _make_module(in_shape): class ReshapeModule(torch.nn.Module): def __init__(self): super(ReshapeModule, self).__init__() - self.conv2d = torch.nn.Conv2d(in_shape[1], out_channel, kernel_size) self.n = in_shape[0] - self.c = out_channel - self.h = in_shape[2] - kernel_size + 1 - self.w = in_shape[3] - kernel_size + 1 + self.c = in_shape[1] + self.h = in_shape[2] + self.w = in_shape[3] def forward(self, x): - x = self.conv2d(x) - x = torch.reshape(x, (self.n, self.c * self.h * self.w)) - x = torch.reshape(x, (self.n, self.c, self.h * self.w)) - x = torch.reshape(x, (self.n, self.c * self.h, self.w)) - x = torch.reshape(x, (self.n * self.c, self.h * self.w)) - x = torch.reshape(x, (self.n * self.c, self.h, self.w)) - x = torch.reshape(x, (self.n * self.c * self.h, self.w)) - x = torch.reshape(x, (-1, self.w)) - return x + out = [] + out.append(torch.reshape(x, (self.n, self.c * self.h * self.w))) + out.append(torch.reshape(x, (self.n, self.c, self.h * self.w))) + out.append(torch.reshape(x, (self.n, self.c * self.h, self.w))) + out.append(torch.reshape(x, (self.n * self.c, self.h * self.w))) + out.append(torch.reshape(x, (self.n * self.c, self.h, self.w))) + out.append(torch.reshape(x, (self.n * self.c * self.h, self.w))) + out.append(torch.reshape(x, (-1, self.w))) + return out return ReshapeModule() in_shapes = [ [1, 4, 60, 72], - [1, 3, 224, 224] -] - -out_channels = [ - 1, - 3, - 16 -] - -kernel_sizes = [ - 1, - 3, - 5 + [1, 3, 224, 224], + [3, 4, 5, 6] ] @pytest.mark.parametrize('in_shape', in_shapes) -@pytest.mark.parametrize('out_channel', out_channels) -@pytest.mark.parametrize('kernel_size', kernel_sizes) -def test_reshape(in_shape, out_channel, kernel_size, request): - module = _make_module(in_shape, out_channel, kernel_size) +def test_reshape(in_shape, request): + module = _make_module(in_shape) runner = OnnxTestRunner(request.node.name) model_file = runner.from_torch(module, in_shape) From fb20a6c58643f8eecd5cb0d67185954f46cc1d67 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 29 Feb 2024 10:30:06 +0800 Subject: [PATCH 077/132] ConvTranspose --> [ncnn op] ConvTranspose --- .../Nncase.Modules.Ncnn/ArgsStruct/args.cs | 72 ++++++++++++++ .../CodeGen/Ncnn/NcnnEmitter.cs | 35 ++++++- .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 5 +- .../Evaluator/Ncnn/NcnnConvTranspose.cs | 94 +++++++++++++++++++ .../Ncnn/{NcnnMatmul.cs => NcnnMatMul.cs} | 0 .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 2 + .../IR/Ncnn/NcnnConvTranspose.cs | 33 +++++++ .../Passes/Rules/Ncnn/LowerConvTranspose.cs | 72 ++++++++++++++ .../Targets/CPUTarget.cs | 1 + 10 files changed, 312 insertions(+), 3 deletions(-) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConvTranspose.cs rename modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/{NcnnMatmul.cs => NcnnMatMul.cs} (100%) create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConvTranspose.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConvTranspose.cs diff --git a/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs index b8ec204623..e13a8b5d97 100644 --- a/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs +++ b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs @@ -155,3 +155,75 @@ public CropArgs(int woffset, int hoffset, int doffset, int coffset, int outw, in public int[]? Axes { get; } } + +public record ConvTransposeArgs +{ + public ConvTransposeArgs(Tensor weightData = null, float[] biasData = null, int numOutput = default, int kernelW = default, int kernelH = default, int dilationW = default, int dilationH = default, int strideW = default, int strideH = default, int padLeft = default, int padRight = default, int padTop = default, int padBottom = default, int biasTerm = default, int weightDataSize = default, int activationType = default, float[] activationParams = null, int outputPadRight = default, int outputPadBottom = default, int outputW = default, int outputH = default) + { + WeightData = weightData; + BiasData = biasData; + NumOutput = numOutput; + KernelW = kernelW; + KernelH = kernelH; + DilationW = dilationW; + DilationH = dilationH; + StrideW = strideW; + StrideH = strideH; + PadLeft = padLeft; + PadRight = padRight; + PadTop = padTop; + PadBottom = padBottom; + BiasTerm = biasTerm; + WeightDataSize = weightDataSize; + ActivationType = activationType; + ActivationParams = activationParams; + OutputPadRight = outputPadRight; + OutputPadBottom = outputPadBottom; + OutputW = outputW; + OutputH = outputH; + } + + public Tensor WeightData { get; } + + public float[] BiasData { get; } + + public int NumOutput { get; } + + public int KernelW { get; } + + public int KernelH { get; } + + public int DilationW { get; } + + public int DilationH { get; } + + public int StrideW { get; } + + public int StrideH { get; } + + public int PadLeft { get; } + + public int PadRight { get; } + + public int PadTop { get; } + + public int PadBottom { get; } + + public int BiasTerm { get; } + + public int WeightDataSize { get; } + + public int ActivationType { get; } + + public float[] ActivationParams { get; } + + public int OutputPadRight { get; } + + public int OutputPadBottom { get; } + + public int OutputW { get; } + + public int OutputH { get; } + + public override string ToString() => $"{nameof(WeightData)}: {string.Join("_", WeightData.Shape.ToValueArray())}, {nameof(NumOutput)}: {NumOutput}, {nameof(KernelW)}: {KernelW}, {nameof(KernelH)}: {KernelH}, {nameof(DilationW)}: {DilationW}, {nameof(DilationH)}: {DilationH}, {nameof(StrideW)}: {StrideW}, {nameof(StrideH)}: {StrideH}, {nameof(PadLeft)}: {PadLeft}, {nameof(PadRight)}: {PadRight}, {nameof(PadTop)}: {PadTop}, {nameof(PadBottom)}: {PadBottom}, {nameof(BiasTerm)}: {BiasTerm}, {nameof(WeightDataSize)}: {WeightDataSize}, {nameof(OutputPadRight)}: {OutputPadRight}, {nameof(OutputPadBottom)}: {OutputPadBottom}, {nameof(OutputW)}: {OutputW}, {nameof(OutputH)}: {OutputH}"; +} diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index af99e8fa3e..e940568543 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -390,7 +390,7 @@ public void Permute(string[] name, string input, int orderType) }); } - public void Matmul(string name, string inputA, string inputB, int lOrR, float[] constInput, int[] constShape) + public void Matmul(string[] name, string inputA, string inputB, int lOrR, float[] constInput, int[] constShape) { var inputList = new[] { inputA, inputB }; @@ -425,7 +425,38 @@ public void Matmul(string name, string inputA, string inputB, int lOrR, float[] WriteFloatArray(constInput); } - AddLayer("MatMul", name, inputList, new[] { name }, null); + AddLayer("MatMul", name[0], inputList, name, null); + } + + public void ConvTranspose(string[] name, string input, ConvTransposeArgs args) + { + AddLayer("Deconvolution", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = args.NumOutput }, + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = args.KernelW }, + [11] = new ParamValue { Kind = ParamKind.Int, IntValue = args.KernelH }, + [2] = new ParamValue { Kind = ParamKind.Int, IntValue = args.DilationW }, + [12] = new ParamValue { Kind = ParamKind.Int, IntValue = args.DilationH }, + [3] = new ParamValue { Kind = ParamKind.Int, IntValue = args.StrideW }, + [13] = new ParamValue { Kind = ParamKind.Int, IntValue = args.StrideH }, + [4] = new ParamValue { Kind = ParamKind.Int, IntValue = args.PadLeft }, + [14] = new ParamValue { Kind = ParamKind.Int, IntValue = args.PadTop }, + [15] = new ParamValue { Kind = ParamKind.Int, IntValue = args.PadRight }, + [16] = new ParamValue { Kind = ParamKind.Int, IntValue = args.PadBottom }, + + // [18] = new ParamValue { Kind = ParamKind.Int, IntValue = args.OutputPadRight }, + // [19] = new ParamValue { Kind = ParamKind.Int, IntValue = args.OutputPadBottom }, + // [20] = new ParamValue { Kind = ParamKind.Int, IntValue = args.OutputW }, + // [21] = new ParamValue { Kind = ParamKind.Int, IntValue = args.OutputH }, + [5] = new ParamValue { Kind = ParamKind.Int, IntValue = args.BiasTerm }, + [6] = new ParamValue { Kind = ParamKind.Int, IntValue = args.WeightDataSize }, + + // [9] = new ParamValue { Kind = ParamKind.Int, IntValue = args.ActivationType }, + // [10] = new ParamValue { Kind = ParamKind.ArrayOfFloat, TensorValue = args.ActivationParams }, + }); + WriteFloatArray(new float[] { 0 }); // quantize flag [Not exist in ncnn op.md] + WriteFloatArray(args.WeightData.ToArray()); + WriteFloatArray(args.BiasData); } private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 1f59cf9ab0..1ee4d86c81 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -208,7 +208,10 @@ protected override string VisitLeafCall(Call expr) 2 => new string[] { ExprMemo[expr.Arguments[0]], string.Empty }, _ => throw new NotImplementedException("Not found MatMul emmiter."), }; - _emitter.Matmul(names[0], inString[0], inString[1], op.LorR, op.ConstInput, op.ConstShape); + _emitter.Matmul(names.ToArray(), inString[0], inString[1], op.LorR, op.ConstInput, op.ConstShape); + break; + case NcnnConvTranspose op: + _emitter.ConvTranspose(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Args); break; default: throw new NotSupportedException(); diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConvTranspose.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConvTranspose.cs new file mode 100644 index 0000000000..6182197696 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConvTranspose.cs @@ -0,0 +1,94 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; +using static Nncase.IR.F.Tensors; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnConvTransposeEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnConvTranspose convTranspose) + { + var inputs = context.GetOrtArgumentValue(convTranspose, NcnnConvTranspose.Input); + var weights = Transpose(convTranspose.Args.WeightData, new int[] { 1, 0, 2, 3 }).Evaluate().AsTensor().ToOrtTensor(); + var bias = new Tensor(convTranspose.Args.BiasData, new[] { convTranspose.Args.NumOutput }).ToOrtTensor(); + var result = OrtKI.ConvTranspose(inputs, weights, bias, "NOTSET", new long[] { convTranspose.Args.DilationH, convTranspose.Args.DilationW }, 1, new long[] { convTranspose.Args.KernelH, convTranspose.Args.KernelW }, new long[] { convTranspose.Args.OutputPadBottom, convTranspose.Args.OutputPadRight, }, new long[] { convTranspose.Args.OutputH, convTranspose.Args.OutputW }, new long[] { convTranspose.Args.PadRight, convTranspose.Args.PadBottom, convTranspose.Args.PadRight, convTranspose.Args.PadBottom }, new long[] { convTranspose.Args.StrideH, convTranspose.Args.StrideW }); + return OrtKI.Clip(result, convTranspose.Args.ActivationParams[0], convTranspose.Args.ActivationParams[1]).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnConvTranspose target) + { + var inputs = context.CheckArgumentType(target, NcnnConvTranspose.Input); + return Visit(inputs, target); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnConvTranspose target) + { + var inputType = context.GetArgumentType(target, NcnnConvTranspose.Input); + _ = inputType.Shape.ToValueArray(); + var weightsType = new TensorType(target.Args.WeightData.ElementType, target.Args.WeightData.Shape); + var weightsShape = weightsType.Shape.ToValueArray(); + var biasType = new TensorType(target.Args.WeightData.ElementType, new[] { target.Args.BiasData.Length }); + + var macPerElement = weightsShape[1] * weightsShape[2] * weightsShape[3]; + var outputType = context.GetReturnType(); + + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(inputType) + CostUtility.GetMemoryAccess(weightsType) + CostUtility.GetMemoryAccess(biasType), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(outputType), + [CostFactorNames.CPUCycles] = CostUtility.GetCPUCycles(outputType, (uint)macPerElement * 2), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnConvTranspose target) + { + var returnType = context.GetReturnType(); + _ = returnType.Shape.ToValueArray(); + + var inputType = context.GetArgumentType(target, NcnnConvTranspose.Input); + var inputShape = inputType.Shape.ToValueArray(); + var weightType = new TensorType(target.Args.WeightData.ElementType, target.Args.WeightData.Shape); + var weightShape = weightType.Shape.ToValueArray(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) + CostUtility.GetMemoryAccess(weightType) + CostUtility.GetMemoryAccess(returnType), + [MetricFactorNames.FLOPs] = (UInt128)(inputShape[0] * weightShape[0] * weightShape[1] * inputShape[2] * inputShape[3] * weightShape[2] * weightShape[3]), + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnConvTranspose target) => context.GetArgumentShape(target, NcnnConvTranspose.Input); + + private TensorType GetTensorType(IRType input) => input switch + { + TensorType t => t, + DistributedType d => d.TensorType, + _ => throw new InvalidCastException(), + }; + + private IRType Visit(TensorType input, NcnnConvTranspose convTranspose) + { + var outputShape = input.Shape.ToList(); + outputShape[0] = convTranspose.Args.NumOutput; + outputShape[1] = convTranspose.Args.OutputH; + outputShape[2] = convTranspose.Args.OutputW; + + return new TensorType(GetTensorType(input).DType, outputShape.ToArray()); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnMatmul.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnMatMul.cs similarity index 100% rename from modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnMatmul.cs rename to modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnMatMul.cs diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index b19e5599b6..4452c3aa78 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -42,5 +42,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index b795a525b8..ed41e5c5a7 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -94,4 +94,6 @@ public static Call NcnnPermute(Expr expr, int orderType, int[] perm) => public static Call NcnnMatMul(Expr[] inExpr, int lorR, float[]? constInput, int[]? constShape) => new Call(new NcnnMatMul(lorR, constInput, constShape), inExpr); + + public static Call NcnnConvTranspose(Expr expr, ConvTransposeArgs args) => new Call(new NcnnConvTranspose(args), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConvTranspose.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConvTranspose.cs new file mode 100644 index 0000000000..d579ec07a5 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConvTranspose.cs @@ -0,0 +1,33 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.ArgsStruct; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// ConvTranspose expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnConvTranspose : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnConvTranspose), 0, "input"); + + public ConvTransposeArgs Args { get; } + + /// + public override string DisplayProperty() + { + return Args.ToString(); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConvTranspose.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConvTranspose.cs new file mode 100644 index 0000000000..ec8da3262e --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConvTranspose.cs @@ -0,0 +1,72 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.ArgsStruct; +using Nncase.Evaluator; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerConvTranspose : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsConv2DTranspose( + "conv", + conv => conv.PadMode == PadMode.Constant, + IsWildcard("input"), + IsTensorConst("weights"), + IsTensorConst("bias"), + IsTensorConst("outputShape"), + IsTensorConst("strides"), + IsTensorConst("paddings"), + IsTensorConst("outputPadding"), + IsTensorConst("dilation"), + IsTensorConst("group"), + IsTensorConst("fusedClamp")); + + private Expr? GetReplace(Expr input, Tensor weights, Tensor bias, int[] outputShape, int[] strides, int[] paddings, int[] outputPadding, int[] dilation, int group, float[] fusedClamp) + { + int[] weightShape = weights.Shape.ToValueArray(); + var (numOutput, kernelH, kernelW) = (weights.Shape[0], weights.Shape[2], weights.Shape[3]); + var (dilationH, dilationW) = (dilation[0], dilation.Length == 2 ? dilation[1] : dilation[0]); + var (strideH, strideW) = (strides[0], strides.Length == 2 ? strides[1] : strides[0]); + + var (padLeft, padRight, padTop, padBottom) = (paddings[2], paddings[3], paddings[0], paddings[1]); + int biasTerm = 1; + var weightsDataSize = weightShape[0] * weightShape[1] * weightShape[2] * weightShape[3]; + + // 1:Relu 2: leaky relu 3:clip 4: sigmoid 5: mish 6: hardswish + int activationType = 3; + + // Note: Has reordered in importer. + // reorder weights(Cin Cout H W) to (Cout Cin H W) + // var newWeights = Transpose(weights, new int[] { 1, 0, 2, 3 }).Evaluate().AsTensor(); + + // actType and actParams not used in ncnn: onnx2ncnn. + var args = new ConvTransposeArgs(weights, bias.ToArray(), numOutput.FixedValue, kernelW.FixedValue, kernelH.FixedValue, dilationW, dilationH, strideW, strideH, padLeft, padRight, padTop, padBottom, biasTerm, weightsDataSize, activationType, fusedClamp, outputPadding[1], outputPadding[0], outputShape[3], outputShape[2]); + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + var conv = new Call(new Fusion("ncnn", NcnnConvTranspose(inResO, args), new[] { inResO }), inRes); + + var outRes = Unsqueeze(conv, new[] { 0 }); + + return outRes; + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index ba5311b154..bcbeb5e8a8 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -90,6 +90,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); }); } From b3653ff6d41be1f1c7c5be54e49c167141ebb14a Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 29 Feb 2024 10:32:04 +0800 Subject: [PATCH 078/132] update ncnn version to 20240102 --- conanfile.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conanfile.py b/conanfile.py index 46e0a74fcf..2b0ba882d9 100644 --- a/conanfile.py +++ b/conanfile.py @@ -39,7 +39,7 @@ class nncaseConan(ConanFile): "vulkan_runtime": False, "openmp": True } - + def imports(self): if self.settings.os == 'Windows': self.copy("nethost.dll", "bin", "bin") @@ -48,7 +48,7 @@ def imports(self): def requirements(self): self.requires('gsl-lite/0.37.0') self.requires('hkg/0.0.1') - self.requires('ncnn/20230816') + self.requires('ncnn/20240102') if self.options.tests: self.requires('gtest/1.10.0') self.requires('ortki/0.0.2') @@ -83,7 +83,7 @@ def configure(self): if self.settings.arch not in ("x86_64",): self.options.halide = False - + if not self.options.runtime: if self.settings.os == 'Windows': self.options["nethost"].shared = True From 789116cdece8931713e40732e0aa2bac5815223f Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 29 Feb 2024 10:48:20 +0800 Subject: [PATCH 079/132] recover reduce scaler output shape --- src/Nncase.Evaluator/TypeInference.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Nncase.Evaluator/TypeInference.cs b/src/Nncase.Evaluator/TypeInference.cs index 519cc01115..bbe74d1739 100644 --- a/src/Nncase.Evaluator/TypeInference.cs +++ b/src/Nncase.Evaluator/TypeInference.cs @@ -323,8 +323,7 @@ public static IRType ReduceType(TensorType input, Expr keepDims, Expr axis) } } - // TODO: please confirm the shape @zhen8838, all zero means scaler - return input with { Shape = outShape.All(x => x == 0) ? new Shape(new[] { 1 }) : new Shape(outShape.Where(x => x != 0)) }; + return input with { Shape = new Shape(outShape.Where(x => x != 0)) }; } return input with { Shape = Shape.Unranked }; From c8118879e1037ce3dd10f778a08489cbed9f262e Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 4 Mar 2024 13:59:31 +0800 Subject: [PATCH 080/132] LayerNorm --> [ncnn op] layernorm --- .../CodeGen/Ncnn/NcnnEmitter.cs | 4 +- .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 2 +- .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 2 +- .../Passes/Rules/Ncnn/LowerLayernorm.cs | 25 ++------ tests/importer/onnx_/basic/test_layernorm.py | 64 +++++++++++++++++++ 5 files changed, 72 insertions(+), 25 deletions(-) create mode 100644 tests/importer/onnx_/basic/test_layernorm.py diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index f3a88109e0..41bc025cac 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -196,9 +196,7 @@ public void InstanceNorm(string name, string input, int channels, float eps, int WriteFloatArray(betaData); } - - public void LayerNorm(string name, string input, int affineSize, float eps, int affine, float[] gammaData, - float[] betaData) + public void LayerNorm(string name, string input, int affineSize, float eps, int affine, float[] gammaData, float[] betaData) { AddLayer("LayerNorm", name, new[] { input }, new[] { name }, new ParamDict { diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index c9c8308a3b..3396b7fdc5 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -214,7 +214,7 @@ protected override string VisitLeafCall(Call expr) _emitter.ConvTranspose(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Args); break; case NcnnLayerNorm op: - _emitter.LayerNorm(name, ExprMemo[expr.Arguments[0]], op.AffineSize, op.Eps, op.Affine, op.GammaData, op.BetaData); + _emitter.LayerNorm(names[0], ExprMemo[expr.Arguments[0]], op.AffineSize, op.Eps, op.Affine, op.GammaData, op.BetaData); break; default: throw new NotSupportedException("Not support in Ncnn ops emitter"); diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index e002da3464..ebef7a92fb 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -50,6 +50,7 @@ public static Call NcnnConv(Expr expr, float[] weightsData, float[] biasData, in public static Call NcnnInstanceNorm(Expr expr, int channels, float eps, int affine, float[] gammaData, float[] betaData) => new Call(new NcnnInstanceNorm(channels, eps, affine, gammaData, betaData), expr); + public static Call NcnnLayerNorm(Expr expr, int affineSize, float eps, int affine, float[] gammaData, float[] betaData) => new Call(new NcnnLayerNorm(affineSize, eps, affine, gammaData, betaData), expr); @@ -98,5 +99,4 @@ public static Call NcnnMatMul(Expr[] inExpr, int lorR, float[]? constInput, int[ new Call(new NcnnMatMul(lorR, constInput, constShape), inExpr); public static Call NcnnConvTranspose(Expr expr, ConvTransposeArgs args) => new Call(new NcnnConvTranspose(args), expr); - } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLayernorm.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLayernorm.cs index ce40849701..e8b480e0c8 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLayernorm.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLayernorm.cs @@ -40,30 +40,15 @@ public partial class LowerLayerNorm : RewriteRule return null; } - // var newShape = FitNcnnShape(input.CheckedShape.ToValueList(), op.Axis); // newAxis = 1; - // Console.WriteLine($"op.Axis: {op.Axis}"); - // Console.WriteLine($"oldShape: {input.CheckedShape.ToValueList().Select(x => x.ToString()).Aggregate((x, y) => x + ", " + y)}"); - // Console.WriteLine($"newShape: {newShape.Select(x => x.ToString()).Aggregate((x, y) => x + "," + y)}"); - // var inRes = Reshape(input, newShape.ToArray()); - // var inResO = new Var(inRes.CheckedType); - - // int affine = gamma.All(a => a != 1) && beta.All(a => a != 0) ? 1 : 0; - // int affineSize = newShape[2]; - - // var layerNorm = new Call(new Fusion("ncnn", NcnnLayerNorm(inResO, affineSize, op.Epsilon, affine, gamma, beta), new[] { inResO }), inRes); - - // return Reshape(layerNorm, input.CheckedShape); - - // var newShape = FitNcnnShape(input.CheckedShape.ToValueList(), op.Axis); // newAxis = 1; - // Console.WriteLine($"op.Axis: {op.Axis}"); - // Console.WriteLine($"oldShape: {input.CheckedShape.ToValueList().Select(x => x.ToString()).Aggregate((x, y) => x + ", " + y)}"); - // Console.WriteLine($"newShape: {newShape.Select(x => x.ToString()).Aggregate((x, y) => x + "," + y)}"); - var inRes = Squeeze(input, new[]{0}); + var inRes = Squeeze(input, new[] { 0 }); var inResO = new Var(inRes.CheckedType); int affine = gamma.All(a => a != 1) && beta.All(a => a != 0) ? 1 : 0; int affineSize = 1; - for(int i = 1; i < inRes.CheckedShape.Rank; i++) + + int newAxis = op.Axis > 0 ? op.Axis - 1 : 0; // BatchSize has been squeezed. + + for (int i = newAxis; i < inRes.CheckedShape.Rank; i++) { affineSize *= inRes.CheckedShape.ToValueArray()[i]; } diff --git a/tests/importer/onnx_/basic/test_layernorm.py b/tests/importer/onnx_/basic/test_layernorm.py new file mode 100644 index 0000000000..96baa77593 --- /dev/null +++ b/tests/importer/onnx_/basic/test_layernorm.py @@ -0,0 +1,64 @@ +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + +import pytest +import torch +import torch.nn as nn +import numpy as np +from onnx_test_runner import OnnxTestRunner + + +def _make_module(v_shape,axis): + class LayerNormModule(torch.nn.Module): + def __init__(self, channel_size=3): + super(LayerNormModule, self).__init__() + reduce_dim = [v_shape[i] for i in range(len(v_shape)) if i > axis] + self.scale = torch.from_numpy(np.random.rand(*reduce_dim).astype(np.float32)) + self.bias = torch.from_numpy(np.random.rand(*reduce_dim).astype(np.float32)) + # torch.layernorm init scale and bias with [1,0] because they are learnable. When the model exports to onnx, scale and bias were eliminated. + # So, random data are used to pretend parameters. + self.layernorm = nn.LayerNorm(normalized_shape=reduce_dim, elementwise_affine=True,eps=1e-03) + + def forward(self, x): + x = self.layernorm(x) + x = x *self.scale + self.bias + return x + + + return LayerNormModule() + + +lhs_shapes = [ + [1, 3, 16, 32], + [1, 3, 16] +] + +axises = [ + 1 +] + + +@pytest.mark.parametrize('lhs_shape', lhs_shapes) +@pytest.mark.parametrize('axis', axises) +def test_LayerNorm(lhs_shape, axis, request): + module = _make_module(lhs_shape, axis) + + runner = OnnxTestRunner(request.node.name) + model_file = runner.from_torch(module, lhs_shape) + runner.run(model_file) + + +if __name__ == "__main__": + pytest.main(['-vv', 'test_LayerNorm.py']) From d5a0065077d2aeb829990d2317923c236c999c66 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 4 Mar 2024 15:41:39 +0800 Subject: [PATCH 081/132] Cast --> [ncnn op] Cast --- .../CodeGen/Ncnn/NcnnEmitter.cs | 17 +++- .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 8 +- .../Evaluator/Ncnn/NcnnCast.cs | 77 ++++++++++++++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnCast.cs | 42 ++++++++++ .../Passes/Rules/Ncnn/LowerCast.cs | 78 +++++++++++++++++++ .../Targets/CPUTarget.cs | 1 + 8 files changed, 220 insertions(+), 7 deletions(-) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCast.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCast.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCast.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 41bc025cac..353d17c52a 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -196,9 +196,9 @@ public void InstanceNorm(string name, string input, int channels, float eps, int WriteFloatArray(betaData); } - public void LayerNorm(string name, string input, int affineSize, float eps, int affine, float[] gammaData, float[] betaData) + public void LayerNorm(string[] name, string input, int affineSize, float eps, int affine, float[] gammaData, float[] betaData) { - AddLayer("LayerNorm", name, new[] { input }, new[] { name }, new ParamDict + AddLayer("LayerNorm", name[0], new[] { input }, name, new ParamDict { [0] = new ParamValue { Kind = ParamKind.Int, IntValue = affineSize }, // affineSize [1] = new ParamValue { Kind = ParamKind.Float, FloatValue = eps }, // eps @@ -208,8 +208,8 @@ public void LayerNorm(string name, string input, int affineSize, float eps, int WriteFloatArray(betaData); } - public void LRN(string name, string input, float alpha, float beta, float bias, int size) => - AddLayer("LRN", name, new[] { input }, new[] { name }, new ParamDict + public void LRN(string[] name, string input, float alpha, float beta, float bias, int size) => + AddLayer("LRN", name[0], new[] { input }, name, new ParamDict { [0] = new ParamValue { Kind = ParamKind.Int, IntValue = 0 }, // region_type [1] = new ParamValue { Kind = ParamKind.Int, IntValue = size }, // size @@ -471,6 +471,15 @@ public void ConvTranspose(string[] name, string input, ConvTransposeArgs args) WriteFloatArray(args.BiasData); } + public void Cast(string[] name, string input, int fromType, int toType) + { + AddLayer("Cast", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = fromType }, + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = toType }, + }); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 3396b7fdc5..442bb4ac65 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -151,13 +151,12 @@ protected override string VisitLeafCall(Call expr) _emitter.InstanceNorm(names[0], ExprMemo[expr.Arguments[0]], op.Channels, op.Eps, op.Affine, op.GammaData, op.BetaData); break; case NcnnLRN op: - _emitter.LRN(names[0], ExprMemo[expr.Arguments[0]], op.Alpha, op.Beta, op.Bias, op.Size); + _emitter.LRN(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Alpha, op.Beta, op.Bias, op.Size); break; case NcnnLSTM op: for (int i = 1; i < op.OutputSize; i++) { var a = GetNextName(); - Console.WriteLine($"{i} = {a}"); names.Add(a); } @@ -214,7 +213,10 @@ protected override string VisitLeafCall(Call expr) _emitter.ConvTranspose(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Args); break; case NcnnLayerNorm op: - _emitter.LayerNorm(names[0], ExprMemo[expr.Arguments[0]], op.AffineSize, op.Eps, op.Affine, op.GammaData, op.BetaData); + _emitter.LayerNorm(names.ToArray(), ExprMemo[expr.Arguments[0]], op.AffineSize, op.Eps, op.Affine, op.GammaData, op.BetaData); + break; + case NcnnCast op: + _emitter.Cast(names.ToArray(), ExprMemo[expr.Arguments[0]], op.FromType, op.ToType); break; default: throw new NotSupportedException("Not support in Ncnn ops emitter"); diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCast.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCast.cs new file mode 100644 index 0000000000..9fc2765c8b --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCast.cs @@ -0,0 +1,77 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnCastEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnCast cast) + { + var input = context.GetOrtArgumentValue(cast, NcnnCast.Input); + return OrtKI.Cast(input, cast.ToType).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnCast target) + { + var input = context.CheckArgumentType(target, NcnnCast.Input); + return Visit(input, target.ToType); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnCast target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnCast target) + { + var inputType = context.GetArgumentType(target, NcnnCast.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnCast target) => context.GetArgumentShape(target, NcnnCast.Input); + + public DataType RecoverDataType(int num) + { + return num switch + { + 1 => DataTypes.Float32, + 2 => DataTypes.Float16, + 4 => DataTypes.BFloat16, + _ => throw new DataException($"not support DataTypeNum :{num}"), + }; + } + + private IRType Visit(TensorType input, int dT) + { + return new TensorType(RecoverDataType(dT), input.Shape); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 6706589c61..417779d175 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -44,5 +44,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index ebef7a92fb..901a36808d 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -99,4 +99,7 @@ public static Call NcnnMatMul(Expr[] inExpr, int lorR, float[]? constInput, int[ new Call(new NcnnMatMul(lorR, constInput, constShape), inExpr); public static Call NcnnConvTranspose(Expr expr, ConvTransposeArgs args) => new Call(new NcnnConvTranspose(args), expr); + + public static Call NcnnCast(Expr expr, int fromType, int toType) => + new Call(new NcnnCast(fromType, toType), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCast.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCast.cs new file mode 100644 index 0000000000..dd63357997 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCast.cs @@ -0,0 +1,42 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.ArgsStruct; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Cast expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnCast : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnCast), 0, "input"); + + /// + /// Gets FromType of Cast. + /// + public int FromType { get; } + + /// + /// Gets ToType of Cast. + /// + public int ToType { get; } + + /// + public override string DisplayProperty() + { + return $"FromType: {FromType}, ToType: {ToType}"; + + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCast.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCast.cs new file mode 100644 index 0000000000..c644b1e3f5 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCast.cs @@ -0,0 +1,78 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; +using Nncase.ArgsStruct; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.IR.Tensors; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerCast : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsCast( + "cast", + _ => true, + IsWildcard("input")); + + private int GetNcnnType(DataType inType) + { + if (inType == DataTypes.Float32) + { + return 1; + } + else if (inType == DataTypes.Float16) + { + return 2; + } + else if (inType == DataTypes.BFloat16) + { + return 4; + } + else + { + return -1; + } + } + + private Expr? GetReplace(Expr input, Cast cast) + { + + if (input.CheckedShape.Count > 4 || input.CheckedShape[0].FixedValue != 1) + { + Console.WriteLine("ncnn not support more than 4D or batchSize > 1"); + return null; + } + + // ncnn only support f32->f16, f32->bf16, f16->f32, bf16->f32 + int fromType = GetNcnnType(input.CheckedDataType); + int toType = GetNcnnType(cast.NewType); + + if (fromType == -1 || toType == -1) + { + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + var c = new Call(new Fusion("ncnn", NcnnCast(inResO, fromType, toType), new[] { inResO }), inRes); + return Unsqueeze(c, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 49860edc3f..4b6fbd00e3 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -92,6 +92,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); }); } From 3537c8de770ad62088fe923f3ff48876e84443c2 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Tue, 5 Mar 2024 09:36:16 +0800 Subject: [PATCH 082/132] GELU --> [ncnn op] GELU --- .../CodeGen/Ncnn/NcnnEmitter.cs | 5 ++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnGELU.cs | 68 +++++++++++++++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 2 + .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnGELU.cs | 30 ++++++++ .../Passes/Rules/Ncnn/LowerGELU.cs | 55 +++++++++++++++ .../Targets/CPUTarget.cs | 1 + tests/importer/onnx_/basic/test_gelu.py | 51 ++++++++++++++ 9 files changed, 216 insertions(+) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnGELU.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnGELU.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerGELU.cs create mode 100644 tests/importer/onnx_/basic/test_gelu.py diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 353d17c52a..48dba1563f 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -480,6 +480,11 @@ public void Cast(string[] name, string input, int fromType, int toType) }); } + public void GELU(string[] name, string input) + { + AddLayer("GELU", name[0], new[] { input }, name, null); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 442bb4ac65..4df716bc30 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -218,6 +218,9 @@ protected override string VisitLeafCall(Call expr) case NcnnCast op: _emitter.Cast(names.ToArray(), ExprMemo[expr.Arguments[0]], op.FromType, op.ToType); break; + case NcnnGELU: + _emitter.GELU(names.ToArray(), ExprMemo[expr.Arguments[0]]); + break; default: throw new NotSupportedException("Not support in Ncnn ops emitter"); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnGELU.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnGELU.cs new file mode 100644 index 0000000000..bea6c1e14d --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnGELU.cs @@ -0,0 +1,68 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.IR.NN; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnGELUEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnGELU celu) + { + var input = context.GetOrtArgumentValue(celu, NcnnGELU.Input); + var res = IR.F.Math.Mul(IR.F.NN.Erf(IR.F.Math.Div(input.ToValue().AsTensor(), 1.4142135381698608)) + 1 , input.ToValue().AsTensor()) * 0.5; + return res.Evaluate(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnGELU target) + { + var input = context.CheckArgumentType(target, NcnnGELU.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnGELU target) + { + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(ret), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnGELU target) + { + var inputType = context.GetArgumentType(target, NcnnGELU.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnGELU target) => context.GetArgumentShape(target, NcnnGELU.Input); + + private IRType Visit(TensorType input) + { + return input; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 417779d175..3abf7d003b 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -45,5 +45,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 901a36808d..61d7328301 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -102,4 +102,6 @@ public static Call NcnnMatMul(Expr[] inExpr, int lorR, float[]? constInput, int[ public static Call NcnnCast(Expr expr, int fromType, int toType) => new Call(new NcnnCast(fromType, toType), expr); + + public static Call NcnnGELU(Expr expr) => new Call(new NcnnGELU(), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnGELU.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnGELU.cs new file mode 100644 index 0000000000..bf70a88ea3 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnGELU.cs @@ -0,0 +1,30 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// GELU expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnGELU : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnGELU), 0, "input"); + + /// + public override string DisplayProperty() + { + return $""; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerGELU.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerGELU.cs new file mode 100644 index 0000000000..56428c43ea --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerGELU.cs @@ -0,0 +1,55 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DryIoc.ImTools; +using Nncase.ArgsStruct; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.IR.Tensors; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.NN; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerGELU : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsGelu( + IsWildcard("input"), + IsTensorConst("alpha")); + + private Expr? GetReplace(Expr input, float alpha) + { + + if (input.CheckedShape.Count > 4 || input.CheckedShape[0].FixedValue != 1) + { + Console.WriteLine("ncnn not support more than 4D or batchSize > 1"); + return null; + } + + // TODO: support GELU with scale. + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + if (Math.Abs(alpha - 1.0) > 1e-06) + { + return null; + } + + var c = new Call(new Fusion("ncnn", NcnnGELU(inResO), new[] { inResO }), inRes); + return Unsqueeze(c, new[] { 0 }); + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 4b6fbd00e3..fa822671ac 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -93,6 +93,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + p.Add(); }); } diff --git a/tests/importer/onnx_/basic/test_gelu.py b/tests/importer/onnx_/basic/test_gelu.py new file mode 100644 index 0000000000..66a51ec3ec --- /dev/null +++ b/tests/importer/onnx_/basic/test_gelu.py @@ -0,0 +1,51 @@ +# Copyright 2019-2021 Canaan Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=invalid-name, unused-argument, import-outside-toplevel + +import pytest +import torch +import torch.nn as nn +from onnx_test_runner import OnnxTestRunner + + +def _make_module(v_shape): + class GELUModule(torch.nn.Module): + def __init__(self,): + super(GELUModule, self).__init__() + self.gelu = nn.GELU(); + def forward(self, x): + x = self.gelu(x) + return x + + + return GELUModule() + + +lhs_shapes = [ + [1, 3, 16, 32], + [1, 3, 16] +] + + +@pytest.mark.parametrize('lhs_shape', lhs_shapes) +def test_gelu(lhs_shape, request): + module = _make_module(lhs_shape) + + runner = OnnxTestRunner(request.node.name) + model_file = runner.from_torch(module, lhs_shape) + runner.run(model_file) + + +if __name__ == "__main__": + pytest.main(['-vv', 'test_gelu.py']) From cba4030e738c8c9d484b9f29c825edb601e47f92 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Tue, 5 Mar 2024 09:58:00 +0800 Subject: [PATCH 083/132] remove glue op --- modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index fa822671ac..17553d57ef 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -64,14 +64,14 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); - // p.Add(); //0816ncnn not support + // p.Add(); p.Add(); p.Add(); p.Add(); p.Add(); p.Add(); - // p.Add(); // need ncnn later than 20230908 + // p.Add(); p.Add(); p.Add(); p.Add(); @@ -95,6 +95,12 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); }); + + passManager.AddWithName("RemoveGlueOp").Configure(p => + { + p.Add(); + p.Add(); + }); } /// From 9cfa87bb014e860764eb4364ede67d4dfb503aa8 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Tue, 5 Mar 2024 14:04:57 +0800 Subject: [PATCH 084/132] fix celu --- .../Passes/Rules/Ncnn/LowerCelu.cs | 11 +++++++---- modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs index fd784999af..54b5ba8f2b 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs @@ -29,13 +29,16 @@ public partial class LowerCelu : RewriteRule private Expr? GetReplace(Expr input, float alpha) { - if (alpha != 1.0) + if (input.CheckedShape.Count > 4 || input.CheckedShape[0].FixedValue != 1) { - return false; + Console.WriteLine("ncnn not support more than 4D or batchSize > 1"); + return null; } - var newInput = new Var(input.CheckedType); + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + var celu = new Call(new Fusion("ncnn", NcnnCelu(inResO, alpha), new[] { inResO }), inRes); + return Unsqueeze(celu, new[] { 0 }); - return new Call(new Fusion("ncnn", NcnnCelu(newInput, alpha), new[] { newInput }), input); } } diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 17553d57ef..bf1ce2cade 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -64,14 +64,14 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); - // p.Add(); + p.Add(); p.Add(); p.Add(); p.Add(); p.Add(); p.Add(); - // p.Add(); + p.Add(); p.Add(); p.Add(); p.Add(); From 1fe424393675a3ab05b9c4f335482acf238830d4 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Tue, 5 Mar 2024 16:45:00 +0800 Subject: [PATCH 085/132] dequantize --> [ncnn op] dequantize --- .../CodeGen/Ncnn/NcnnEmitter.cs | 12 ++++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnDequantize.cs | 67 +++++++++++++++++++ .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 1 + .../IR/Ncnn/NcnnDequantize.cs | 40 +++++++++++ .../Passes/Rules/Ncnn/LowerDequantize.cs | 58 ++++++++++++++++ .../Targets/CPUTarget.cs | 1 + 8 files changed, 183 insertions(+) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnDequantize.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnDequantize.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerDequantize.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 48dba1563f..2c68278055 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -485,6 +485,18 @@ public void GELU(string[] name, string input) AddLayer("GELU", name[0], new[] { input }, name, null); } + public void Dequantize(string[] name, string input, float[] scale, float[] bias) + { + AddLayer("Dequantize", name[0], new[] { input }, name, new ParamDict + { + [0] = new ParamValue { Kind = ParamKind.Int, IntValue = scale.Length }, + [1] = new ParamValue { Kind = ParamKind.Int, IntValue = bias.Length }, + }); + + WriteFloatArray(scale); + WriteFloatArray(bias); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 4df716bc30..71a0f1115e 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -221,6 +221,9 @@ protected override string VisitLeafCall(Call expr) case NcnnGELU: _emitter.GELU(names.ToArray(), ExprMemo[expr.Arguments[0]]); break; + case NcnnDequantize op: + _emitter.Dequantize(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Scale, op.Bias); + break; default: throw new NotSupportedException("Not support in Ncnn ops emitter"); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnDequantize.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnDequantize.cs new file mode 100644 index 0000000000..60a1006b36 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnDequantize.cs @@ -0,0 +1,67 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnDequantizeEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnDequantize dequantize) + { + var input = context.GetOrtArgumentValue(dequantize, NcnnDequantize.Input); + return OrtKI.DequantizeLinear(input, dequantize.Scale, dequantize.Bias, 0).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnDequantize target) + { + var input = context.CheckArgumentType(target, NcnnDequantize.Input); + return Visit(input); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnDequantize target) + { + var input = context.GetArgumentType(target, NcnnDequantize.Input); + var ret = context.GetReturnType(); + return new() + { + [CostFactorNames.MemoryLoad] = CostUtility.GetMemoryAccess(input) + + (UInt128)((target.Scale.Length + target.Bias.Length) * sizeof(float)), + [CostFactorNames.MemoryStore] = CostUtility.GetMemoryAccess(ret), + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnDequantize target) + { + var inputType = context.GetArgumentType(target, NcnnDequantize.Input); + var outputType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(inputType) + CostUtility.GetMemoryAccess(outputType), + [MetricFactorNames.FLOPs] = MetricUtility.GetFLOPs(outputType, 2), + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnDequantize target) => context.GetArgumentShape(target, NcnnDequantize.Input); + + private IRType Visit(TensorType input) + { + // ncnn only support dequantize to float32. + return new TensorType(DataTypes.Float32, input.Shape); + } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 3abf7d003b..61754c60a1 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -46,5 +46,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 61d7328301..f6fb57393e 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -104,4 +104,5 @@ public static Call NcnnCast(Expr expr, int fromType, int toType) => new Call(new NcnnCast(fromType, toType), expr); public static Call NcnnGELU(Expr expr) => new Call(new NcnnGELU(), expr); + public static Call NcnnDequantize(Expr expr, float[] scale, float[] bias) => new Call(new NcnnDequantize(scale, bias), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnDequantize.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnDequantize.cs new file mode 100644 index 0000000000..23e9a27514 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnDequantize.cs @@ -0,0 +1,40 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Dequantize expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnDequantize : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnDequantize), 0, "input"); + + /// + /// Gets scale of Ncnn Dequantize. + /// + public float[] Scale { get; } + + /// + /// Gets scale of Ncnn Dequantize. + /// + public float[] Bias { get; } + + /// + public override string DisplayProperty() + { + return $"scale_size: {Scale.Length}, bias_size: {Bias.Length}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerDequantize.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerDequantize.cs new file mode 100644 index 0000000000..1d21d29e6b --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerDequantize.cs @@ -0,0 +1,58 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.IR.Math; +using Nncase.IR.Ncnn; +using Nncase.PatternMatch; + +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; +using static Nncase.PatternMatch.F.Math; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerDequantize : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsDequantize( + "deq", + "output", + _ => true, + IsWildcard("input"), + IsTensorConst("param")); + + private Expr? GetReplace(Expr input, Dequantize deq, QuantParam[] param) + { + if (input.CheckedShape.Count > 4 || input.CheckedShape[0].FixedValue != 1 || deq.TargetType != DataTypes.Float32) + { + Console.WriteLine("ncnn not support more than 4D or batchSize > 1"); + return null; + } + + var inRes = Squeeze(input, new[] { 0 }); + var inResO = new Var(inRes.CheckedType); + + float[] scale = new float[param.Length]; + float[] bias = new float[param.Length]; + for (int i = 0; i < param.Length; i++) + { + scale[i] = param[i].Scale; + bias[i] = param[i].ZeroPoint; + } + + var dequantize = new Call(new Fusion("ncnn", NcnnDequantize(inResO, scale, bias), new[] { inResO }), inRes); + + return Unsqueeze(dequantize, new[] { 0 }); + + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index bf1ce2cade..ab1a608613 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -94,6 +94,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + // p.Add(); // ncnn dequantize int to float. }); passManager.AddWithName("RemoveGlueOp").Configure(p => From 059afa04004cc023ccf3180de5db81dc7a57b94e Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Wed, 6 Mar 2024 10:19:44 +0800 Subject: [PATCH 086/132] Squeeze --> [ncnn op] Squeeze --- .../CodeGen/Ncnn/NcnnEmitter.cs | 19 +++++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Evaluator/Ncnn/NcnnSqueeze.cs | 79 +++++++++++++++++++ .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 2 + .../IR/Ncnn/NcnnSqueeze.cs | 35 ++++++++ .../Passes/Rules/Ncnn/LowerSqueeze.cs | 35 ++++++++ .../Targets/CPUTarget.cs | 1 + 8 files changed, 175 insertions(+) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSqueeze.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSqueeze.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 2c68278055..7424ddea97 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -497,6 +497,25 @@ public void Dequantize(string[] name, string input, float[] scale, float[] bias) WriteFloatArray(bias); } + public void Squeeze(string[] name, string input, int[] dims) + { + var repeatsData = new List { dims.Length }; + repeatsData.AddRange(dims); + var args = new ParamDict(); + if (dims.Length == 0) + { + args.Add(0, new ParamValue { Kind = ParamKind.Int, IntValue = 1 }); + args.Add(1, new ParamValue { Kind = ParamKind.Int, IntValue = 1 }); + args.Add(2, new ParamValue { Kind = ParamKind.Int, IntValue = 1 }); + } + else + { + args.Add(-3, new ParamValue { Kind = ParamKind.ArrayOfInt, TensorValue = repeatsData.ToArray() }); + } + + AddLayer("Squeeze", name[0], new[] { input }, name, args); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 71a0f1115e..81b3c9e2c5 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -224,6 +224,9 @@ protected override string VisitLeafCall(Call expr) case NcnnDequantize op: _emitter.Dequantize(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Scale, op.Bias); break; + case NcnnSqueeze op: + _emitter.Squeeze(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Dims); + break; default: throw new NotSupportedException("Not support in Ncnn ops emitter"); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 61754c60a1..3537bdd3ee 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -47,5 +47,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSqueeze.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSqueeze.cs new file mode 100644 index 0000000000..a6d4c5fe3a --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSqueeze.cs @@ -0,0 +1,79 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnSqueezeEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnSqueeze squeeze) + { + var input = context.GetOrtArgumentValue(squeeze, NcnnSqueeze.Input); + var dims = squeeze.Dims; + return OrtKI.Squeeze(input, dims).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnSqueeze target) + { + var input = context.CheckArgumentType(target, NcnnSqueeze.Input); + var dims = target.Dims; + return Visit(input, dims); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnSqueeze target) + { + return new() + { + [CostFactorNames.CPUCycles] = 1, + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnSqueeze target) + { + _ = context.GetArgumentType(target, NcnnSqueeze.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = 0, + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnSqueeze target) => context.GetArgumentShape(target, NcnnSqueeze.Input); + + private IRType Visit(TensorType input, int[] dims) + { + var outputShape = input.Shape.ToValueArray(); + if (dims.Length == 0) + { + outputShape = outputShape.Where(x => x != 1).ToArray(); + } + else + { + // outputShape = outputShape.Select((value, idx) => dims.Contains(idx) ? value : (value == 1 ? 0 : value)).Where(v => v != 0).ToArray(); + outputShape = outputShape.Select((value, idx) => dims.Contains(idx) ? (value == 1 ? 0 : value) : value).Where(v => v != 0) + .ToArray(); + } + + + return new TensorType(input.DType, outputShape); + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index f6fb57393e..8e135b5d8c 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -105,4 +105,6 @@ public static Call NcnnCast(Expr expr, int fromType, int toType) => public static Call NcnnGELU(Expr expr) => new Call(new NcnnGELU(), expr); public static Call NcnnDequantize(Expr expr, float[] scale, float[] bias) => new Call(new NcnnDequantize(scale, bias), expr); + + public static Call NcnnSqueeze(Expr expr, int[] dims) => new Call(new NcnnSqueeze(dims), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSqueeze.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSqueeze.cs new file mode 100644 index 0000000000..3cb0fb8420 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnSqueeze.cs @@ -0,0 +1,35 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Squeeze expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnSqueeze : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnSqueeze), 0, "input"); + + /// + /// Gets dims of Ncnn Squeeze. + /// + public int[] Dims { get; } + + /// + public override string DisplayProperty() + { + return $"dims:{string.Join(",", Dims)}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs new file mode 100644 index 0000000000..9583611d0b --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs @@ -0,0 +1,35 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.PatternMatch; +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerSqueeze : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsSqueeze( + IsWildcard("input"), + IsTensorConst("dims")); + + private Expr? GetReplace(Expr input, int[] dims) + { + if (input.CheckedShape.Count < 5) + { + var inResO = new Var(input.CheckedType); + // var newDims = dims[0] == 0 ? dims[1..] : dims; + return new Call(new Fusion("ncnn", NcnnSqueeze(inResO, dims), new[] { inResO }), input); + } + + return null; + + + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index ab1a608613..710094d432 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -101,6 +101,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions { p.Add(); p.Add(); + p.Add(); }); } From 16b4f8f38500125aeb6b5f6a8bc72f01e5fcd50d Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Wed, 6 Mar 2024 10:49:45 +0800 Subject: [PATCH 087/132] Unsqueeze --> [ncnn op] Unsqueeze --- .../CodeGen/Ncnn/NcnnEmitter.cs | 10 +++ .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 3 + .../Evaluator/Ncnn/NcnnCast.cs | 2 +- .../Evaluator/Ncnn/NcnnGELU.cs | 4 +- .../Evaluator/Ncnn/NcnnModule.cs | 1 + .../Evaluator/Ncnn/NcnnSqueeze.cs | 1 - .../Evaluator/Ncnn/NcnnUnsqueeze.cs | 72 +++++++++++++++++++ .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 3 + .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnCast.cs | 1 - .../IR/Ncnn/NcnnUnsqueeze.cs | 35 +++++++++ .../Passes/Rules/Ncnn/LowerCast.cs | 1 - .../Passes/Rules/Ncnn/LowerCelu.cs | 1 - .../Passes/Rules/Ncnn/LowerDequantize.cs | 1 - .../Passes/Rules/Ncnn/LowerGELU.cs | 2 - .../Passes/Rules/Ncnn/LowerSqueeze.cs | 2 - .../Passes/Rules/Ncnn/LowerUnsqueeze.cs | 34 +++++++++ .../Targets/CPUTarget.cs | 6 ++ 17 files changed, 167 insertions(+), 12 deletions(-) create mode 100644 modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnUnsqueeze.cs create mode 100644 modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnUnsqueeze.cs create mode 100644 modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnsqueeze.cs diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 7424ddea97..9bf8e457c8 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -516,6 +516,16 @@ public void Squeeze(string[] name, string input, int[] dims) AddLayer("Squeeze", name[0], new[] { input }, name, args); } + public void Unsqueeze(string[] name, string input, int[] dims) + { + var repeatsData = new List { dims.Length }; + repeatsData.AddRange(dims); + AddLayer("ExpandDims", name[0], new[] { input }, name, new ParamDict + { + [-3] = new ParamValue { Kind = ParamKind.ArrayOfInt, TensorValue = repeatsData.ToArray() }, + }); + } + private void AddLayer(string type, string name, string[] bottoms, string[] tops, ParamDict? paramDict = null, int layerType = 1) { var layer = new NcnnLayer(type, name, bottoms.Length, tops.Length); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 81b3c9e2c5..ef1603f4db 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -227,6 +227,9 @@ protected override string VisitLeafCall(Call expr) case NcnnSqueeze op: _emitter.Squeeze(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Dims); break; + case NcnnUnsqueeze op: + _emitter.Unsqueeze(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Dims); + break; default: throw new NotSupportedException("Not support in Ncnn ops emitter"); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCast.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCast.cs index 9fc2765c8b..e8e2ebdbb1 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCast.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCast.cs @@ -47,7 +47,7 @@ public Cost Visit(ICostEvaluateContext context, NcnnCast target) public Metric Visit(IMetricEvaluateContext context, NcnnCast target) { - var inputType = context.GetArgumentType(target, NcnnCast.Input); + _ = context.GetArgumentType(target, NcnnCast.Input); var returnType = context.GetReturnType(); return new() diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnGELU.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnGELU.cs index bea6c1e14d..5e5b0aacdf 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnGELU.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnGELU.cs @@ -25,7 +25,7 @@ public class NcnnGELUEvaluator : IEvaluator, ITypeInferencer public IValue Visit(IEvaluateContext context, NcnnGELU celu) { var input = context.GetOrtArgumentValue(celu, NcnnGELU.Input); - var res = IR.F.Math.Mul(IR.F.NN.Erf(IR.F.Math.Div(input.ToValue().AsTensor(), 1.4142135381698608)) + 1 , input.ToValue().AsTensor()) * 0.5; + var res = IR.F.Math.Mul(IR.F.NN.Erf(IR.F.Math.Div(input.ToValue().AsTensor(), 1.4142135381698608)) + 1, input.ToValue().AsTensor()) * 0.5; return res.Evaluate(); } @@ -49,7 +49,7 @@ public Cost Visit(ICostEvaluateContext context, NcnnGELU target) public Metric Visit(IMetricEvaluateContext context, NcnnGELU target) { - var inputType = context.GetArgumentType(target, NcnnGELU.Input); + _ = context.GetArgumentType(target, NcnnGELU.Input); var returnType = context.GetReturnType(); return new() diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs index 3537bdd3ee..c31062a221 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnModule.cs @@ -48,5 +48,6 @@ public void ConfigureServices(IRegistrator registrator) registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); registrator.RegisterManyInterface(reuse: Reuse.Singleton); + registrator.RegisterManyInterface(reuse: Reuse.Singleton); } } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSqueeze.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSqueeze.cs index a6d4c5fe3a..90d9c4f953 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSqueeze.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnSqueeze.cs @@ -73,7 +73,6 @@ private IRType Visit(TensorType input, int[] dims) .ToArray(); } - return new TensorType(input.DType, outputShape); } } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnUnsqueeze.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnUnsqueeze.cs new file mode 100644 index 0000000000..a9083f71a2 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnUnsqueeze.cs @@ -0,0 +1,72 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NetFabric.Hyperlinq; +using Nncase.CostModel; +using Nncase.IR; +using Nncase.IR.Ncnn; +using OrtKISharp; + +namespace Nncase.Evaluator.Ncnn; + +/// +/// Evaluator for . +/// +public class NcnnUnsqueezeEvaluator : IEvaluator, ITypeInferencer, ICostEvaluator, IShapeEvaluator, IMetricEvaluator +{ + /// + public IValue Visit(IEvaluateContext context, NcnnUnsqueeze unsqueeze) + { + var input = context.GetOrtArgumentValue(unsqueeze, NcnnUnsqueeze.Input); + var dims = unsqueeze.Dims; + return OrtKI.Unsqueeze(input, dims).ToValue(); + } + + /// + public IRType Visit(ITypeInferenceContext context, NcnnUnsqueeze target) + { + var input = context.CheckArgumentType(target, NcnnUnsqueeze.Input); + var dims = target.Dims; + return Visit(input, dims); + } + + /// + public Cost Visit(ICostEvaluateContext context, NcnnUnsqueeze target) + { + return new() + { + [CostFactorNames.CPUCycles] = 1, + }; + } + + public Metric Visit(IMetricEvaluateContext context, NcnnUnsqueeze target) + { + _ = context.GetArgumentType(target, NcnnUnsqueeze.Input); + var returnType = context.GetReturnType(); + + return new() + { + [MetricFactorNames.OffChipMemoryTraffic] = CostUtility.GetMemoryAccess(returnType) * 2, + [MetricFactorNames.FLOPs] = 0, + [MetricFactorNames.Parallel] = 4, + }; + } + + public Expr Visit(IShapeEvaluateContext context, NcnnUnsqueeze target) => context.GetArgumentShape(target, NcnnUnsqueeze.Input); + + private IRType Visit(TensorType input, int[] dims) + { + var outputShape = input.Shape.ToValueArray().ToList(); + for (int i = dims.Length - 1; i >= 0; i--) + { + outputShape.Insert(i, 1); + } + + return new TensorType(input.DType, outputShape.ToArray()); + } +} diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 8e135b5d8c..767b5ae877 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -104,7 +104,10 @@ public static Call NcnnCast(Expr expr, int fromType, int toType) => new Call(new NcnnCast(fromType, toType), expr); public static Call NcnnGELU(Expr expr) => new Call(new NcnnGELU(), expr); + public static Call NcnnDequantize(Expr expr, float[] scale, float[] bias) => new Call(new NcnnDequantize(scale, bias), expr); public static Call NcnnSqueeze(Expr expr, int[] dims) => new Call(new NcnnSqueeze(dims), expr); + + public static Call NcnnUnsqueeze(Expr expr, int[] dims) => new Call(new NcnnUnsqueeze(dims), expr); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCast.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCast.cs index dd63357997..30ccd54df9 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCast.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnCast.cs @@ -37,6 +37,5 @@ public sealed partial class NcnnCast : Op public override string DisplayProperty() { return $"FromType: {FromType}, ToType: {ToType}"; - } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnUnsqueeze.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnUnsqueeze.cs new file mode 100644 index 0000000000..2eb1edb63c --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnUnsqueeze.cs @@ -0,0 +1,35 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Nncase.PatternMatch; + +namespace Nncase.IR.Ncnn; + +/// +/// Unsqueeze expression. +/// +[PatternFunctionalGenerator] +public sealed partial class NcnnUnsqueeze : Op +{ + /// + /// Gets input. + /// + public static readonly ParameterInfo Input = new(typeof(NcnnUnsqueeze), 0, "input"); + + /// + /// Gets dims of Ncnn Unsqueeze. + /// + public int[] Dims { get; } + + /// + public override string DisplayProperty() + { + return $"dims:{string.Join(",", Dims)}"; + } +} diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCast.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCast.cs index c644b1e3f5..0ebb65ccea 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCast.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCast.cs @@ -54,7 +54,6 @@ private int GetNcnnType(DataType inType) private Expr? GetReplace(Expr input, Cast cast) { - if (input.CheckedShape.Count > 4 || input.CheckedShape[0].FixedValue != 1) { Console.WriteLine("ncnn not support more than 4D or batchSize > 1"); diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs index 54b5ba8f2b..415a27b42b 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs @@ -39,6 +39,5 @@ public partial class LowerCelu : RewriteRule var inResO = new Var(inRes.CheckedType); var celu = new Call(new Fusion("ncnn", NcnnCelu(inResO, alpha), new[] { inResO }), inRes); return Unsqueeze(celu, new[] { 0 }); - } } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerDequantize.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerDequantize.cs index 1d21d29e6b..5e98fed5a0 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerDequantize.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerDequantize.cs @@ -53,6 +53,5 @@ public partial class LowerDequantize : RewriteRule var dequantize = new Call(new Fusion("ncnn", NcnnDequantize(inResO, scale, bias), new[] { inResO }), inRes); return Unsqueeze(dequantize, new[] { 0 }); - } } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerGELU.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerGELU.cs index 56428c43ea..6dbb3ab041 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerGELU.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerGELU.cs @@ -33,7 +33,6 @@ public partial class LowerGELU : RewriteRule private Expr? GetReplace(Expr input, float alpha) { - if (input.CheckedShape.Count > 4 || input.CheckedShape[0].FixedValue != 1) { Console.WriteLine("ncnn not support more than 4D or batchSize > 1"); @@ -41,7 +40,6 @@ public partial class LowerGELU : RewriteRule } // TODO: support GELU with scale. - var inRes = Squeeze(input, new[] { 0 }); var inResO = new Var(inRes.CheckedType); if (Math.Abs(alpha - 1.0) > 1e-06) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs index 9583611d0b..c8134b0633 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs @@ -29,7 +29,5 @@ public partial class LowerSqueeze : RewriteRule } return null; - - } } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnsqueeze.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnsqueeze.cs new file mode 100644 index 0000000000..e7dc4e96c9 --- /dev/null +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnsqueeze.cs @@ -0,0 +1,34 @@ +// Copyright (c) Canaan Inc. All rights reserved. +// Licensed under the Apache license. See LICENSE file in the project root for full license information. + +using NetFabric.Hyperlinq; +using Nncase.IR; +using Nncase.PatternMatch; +using static Nncase.IR.F.Ncnn; +using static Nncase.IR.F.Tensors; +using static Nncase.PatternMatch.F.Tensors; +using static Nncase.PatternMatch.Utility; + +namespace Nncase.Passes.Rules.Ncnn; + +[RuleGenerator] +public partial class LowerUnsqueeze : RewriteRule +{ + /// + public override Pattern Pattern { get; } = IsUnsqueeze( + IsWildcard("input"), + IsTensorConst("dims")); + + private Expr? GetReplace(Expr input, int[] dims) + { + if (input.CheckedShape.Count + dims.Length <= 5) + { + var inResO = new Var(input.CheckedType); + + // var newDims = dims[0] == 0 ? dims[1..] : dims; + return new Call(new Fusion("ncnn", NcnnUnsqueeze(inResO, dims), new[] { inResO }), input); + } + + return null; + } +} diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 710094d432..2413bbf933 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -94,6 +94,7 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); p.Add(); + // p.Add(); // ncnn dequantize int to float. }); @@ -101,7 +102,12 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions { p.Add(); p.Add(); + }); + + passManager.AddWithName("RemoveSingleSqueezeAndUnsqueeze").Configure(p => + { p.Add(); + p.Add(); }); } From d89524aeaeb2b306cc76511ee34fff168268d37b Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 18 Mar 2024 16:01:04 +0800 Subject: [PATCH 088/132] fix load rdata in ncnn runtime --- .../CodeGen/Ncnn/NcnnEmitter.cs | 12 ++++-- .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 20 +++++++-- .../CodeGen/Ncnn/NcnnLinkableFunction.cs | 3 +- .../Targets/CPUTarget.cs | 5 ++- modules/ncnn/src/runtime/runtime_function.cpp | 42 ++++++++++++++++--- src/Nncase.CodeGen/CodeGen/LinkedSection.cs | 14 +++++++ 6 files changed, 82 insertions(+), 14 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 9bf8e457c8..077e6c5852 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -17,6 +17,8 @@ namespace Nncase.CodeGen.Ncnn; internal class NcnnEmitter { + public List? RData; + private readonly NcnnModel _model; private readonly BinaryWriter _binWriter; @@ -24,6 +26,7 @@ public NcnnEmitter(BinaryWriter binWriter) { _model = new NcnnModel(); _binWriter = binWriter; + RData = new List(); } public void SaveParam(Stream paramStream) @@ -32,12 +35,14 @@ public void SaveParam(Stream paramStream) _model.Serialize(sw); } - public void SaveBin() + public void SaveBin(string dumpPath, uint id) { - using (var fileStream = File.Create(Directory.GetCurrentDirectory() + "/ncnn.bin")) + using (var fileStream = File.Create(Path.Join(dumpPath, $"/ncnn_{id}.bin"))) { _binWriter.BaseStream.Seek(0, SeekOrigin.Begin); _binWriter.BaseStream.CopyTo(fileStream); + + // _binWriter.BaseStream.SetLength(0); } } @@ -141,7 +146,7 @@ public void Conv(string name, string input, float[] weightsData, float[] biasDat [15] = new ParamValue { Kind = ParamKind.Int, IntValue = padBottom }, [5] = new ParamValue { Kind = ParamKind.Int, IntValue = biasTerm }, [6] = new ParamValue { Kind = ParamKind.Int, IntValue = weightsDataSize }, - [7] = new ParamValue { Kind = ParamKind.Int, IntValue = 1 }, // Group + [7] = new ParamValue { Kind = ParamKind.Int, IntValue = 0 }, // Group [8] = new ParamValue { Kind = ParamKind.Int, IntValue = int8Flag }, // [9] = new ParamValue { Kind = ParamKind.Int, IntValue = actType }, @@ -560,6 +565,7 @@ private void AddLayer(string type, string name, string[] bottoms, string[] tops, private void WriteFloatArray(float[] data) { + RData.AddRange(data); foreach (float value in data) { _binWriter.Write(value); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index ef1603f4db..346421f250 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -36,7 +36,7 @@ public NcnnFunctionBuilder(uint id, SectionManager sectionManager) protected override ILinkableFunction CreateLinkableFunction(uint id, BaseFunction callable, IReadOnlyList functionRefs, Stream text) { - return new NcnnLinkableFunction(id, callable, functionRefs, text, _inputs!, _outputs!); + return new NcnnLinkableFunction(id, callable, functionRefs, text, _inputs!, _outputs!, _emitter.RData!.ToArray()!); } protected override void Compile(BaseFunction callable) @@ -49,14 +49,28 @@ protected override void Compile(BaseFunction callable) protected override void WriteText() { _emitter.SaveParam(TextWriter.BaseStream); + string dumpPath = Path.Join(DumpScope.Current.Directory, "ncnn_param_dir"); + if (!Directory.Exists(dumpPath)) + { + Directory.CreateDirectory(dumpPath); + } + else if (Id == 0) + { + foreach (string filePath in Directory.GetFiles(dumpPath, "*", SearchOption.AllDirectories)) + { + File.SetAttributes(filePath, FileAttributes.Normal); // 移除所有特殊属性以便删除 + File.Delete(filePath); + } + } - using (var fileStream = File.Create(Directory.GetCurrentDirectory() + "/ncnn.param")) + // if (DumpScope.Current.IsEnabled(DumpFlags.CodeGen)) + using (var fileStream = File.Create(Path.Join(dumpPath, $"ncnn_{Id}.param"))) { TextWriter.BaseStream.Seek(0, SeekOrigin.Begin); TextWriter.BaseStream.CopyTo(fileStream); } - _emitter.SaveBin(); + _emitter.SaveBin(dumpPath, Id); } private class CodeGenVisitor : ExprVisitor diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkableFunction.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkableFunction.cs index 21a28b70d7..28fe673c69 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkableFunction.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnLinkableFunction.cs @@ -12,7 +12,7 @@ namespace Nncase.CodeGen.Ncnn; internal class NcnnLinkableFunction : ILinkableFunction { - public NcnnLinkableFunction(uint id, BaseFunction sourceFunction, IEnumerable functionRefs, Stream text, string[] inputs, string[] outputs) + public NcnnLinkableFunction(uint id, BaseFunction sourceFunction, IEnumerable functionRefs, Stream text, string[] inputs, string[] outputs, float[] rData) { Id = id; SourceFunction = sourceFunction; @@ -24,6 +24,7 @@ public NcnnLinkableFunction(uint id, BaseFunction sourceFunction, IEnumerable(); p.Add(); p.Add(); - p.Add(); + p.Add(); p.Add(); p.Add(); @@ -100,12 +100,15 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions passManager.AddWithName("RemoveGlueOp").Configure(p => { + p.Add(); p.Add(); p.Add(); + p.Add(); }); passManager.AddWithName("RemoveSingleSqueezeAndUnsqueeze").Configure(p => { + p.Add(); p.Add(); p.Add(); }); diff --git a/modules/ncnn/src/runtime/runtime_function.cpp b/modules/ncnn/src/runtime/runtime_function.cpp index f519d0499b..904c8c382f 100644 --- a/modules/ncnn/src/runtime/runtime_function.cpp +++ b/modules/ncnn/src/runtime/runtime_function.cpp @@ -21,16 +21,33 @@ #include #include -typedef std::chrono::high_resolution_clock::time_point TimeVar; +size_t rdata_offset = 0; + +typedef std::chrono::high_resolution_clock::time_point time_var; #define duration(name, a) \ std::cout \ << name << " : " \ << std::chrono::duration_cast(a).count() / \ - 1000.0 \ + 1e6 \ << " ms" << std::endl #define timeNow() std::chrono::high_resolution_clock::now() +template +void print_data(T1 *data, int size, int placeholde = 1, int changeline = 1) +{ + auto s = placeholde ? " " : ""; + for (int i = 0; i < size; i++) { + if(i%10 == 0 && changeline) + { + std::cout< ncnn_runtime_function::initialize_core( return ok(); })); + NNCASE_UNUSED stream_reader* sr = nullptr; + section_header h; + try_set(sr, context.seek_section(".rdata", h)); + // rdata_ = sr.template read(); + // std::cout << "start: " << h.body_start << "\t" << "size: " << h.body_size << std::endl; + // print_data(rdata_.data(), rdata_.size() / sizeof(float)); auto param_mem = reinterpret_cast( module().text().data() + context.header().entrypoint); - auto bin_mem = reinterpret_cast( - module().rdata().data() + context.header().entrypoint); + if(context.header().entrypoint == 0) + rdata_offset = 0; + auto bin_mem = reinterpret_cast(module().rdata().data() + + rdata_offset); + // auto bin_mem = reinterpret_cast(rdata_.data()); ::ncnn::DataReaderFromMemory paramdr(param_mem); ::ncnn::DataReaderFromMemory bindr(bin_mem); CHECK_WITH_ERR(!net_.load_param(paramdr), std::errc::invalid_argument); CHECK_WITH_ERR(!net_.load_model(bindr), std::errc::invalid_argument); - net_.opt.num_threads = omp_get_num_devices(); + + net_.opt.num_threads = 1; + + rdata_offset += h.memory_size; + return ok(); } @@ -144,7 +174,7 @@ result ncnn_runtime_function::invoke_core( // 2. Extract outputs std::vector outputs; - TimeVar t = timeNow(); + time_var t = timeNow(); for (size_t i = 0; i < output_names_.size(); i++) { ::ncnn::Mat mat; CHECK_WITH_ERR(!ex.extract(output_names_[i].c_str(), mat), diff --git a/src/Nncase.CodeGen/CodeGen/LinkedSection.cs b/src/Nncase.CodeGen/CodeGen/LinkedSection.cs index 84329d7ba2..aeaa2c9739 100644 --- a/src/Nncase.CodeGen/CodeGen/LinkedSection.cs +++ b/src/Nncase.CodeGen/CodeGen/LinkedSection.cs @@ -60,6 +60,20 @@ public static LinkedSection FromStrings(IReadOnlyCollection strings, str return new LinkedSection(ms, name, 0, 1, (ulong)ms.Length); } + public static LinkedSection FromData(IReadOnlyCollection datas, string name) + { + var ms = new MemoryStream(); + using (var bw = new BinaryWriter(ms, Encoding.UTF8, true)) + { + foreach (float s in datas) + { + bw.Write(s); + } + } + + return new LinkedSection(ms, name, 0, 1, (ulong)ms.Length); + } + public void Serialize(Stream output) { if (_content != null) From 8cc6ce61ba2675f8ea442cbf3f58d217fa2ef366 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 21 Mar 2024 14:41:39 +0800 Subject: [PATCH 089/132] fix binary --- .../Nncase.Modules.Ncnn/ArgsStruct/args.cs | 133 ++++++++++++++++++ .../Evaluator/Ncnn/NcnnBinary.cs | 13 +- .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs | 4 + .../Passes/Rules/Ncnn/LowerBinary.cs | 114 +++++++++++++-- tests/importer/onnx_/basic/test_binary.py | 6 +- 5 files changed, 250 insertions(+), 20 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs index e13a8b5d97..2881b2e66b 100644 --- a/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs +++ b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs @@ -227,3 +227,136 @@ public ConvTransposeArgs(Tensor weightData = null, float[] biasData = null, int public override string ToString() => $"{nameof(WeightData)}: {string.Join("_", WeightData.Shape.ToValueArray())}, {nameof(NumOutput)}: {NumOutput}, {nameof(KernelW)}: {KernelW}, {nameof(KernelH)}: {KernelH}, {nameof(DilationW)}: {DilationW}, {nameof(DilationH)}: {DilationH}, {nameof(StrideW)}: {StrideW}, {nameof(StrideH)}: {StrideH}, {nameof(PadLeft)}: {PadLeft}, {nameof(PadRight)}: {PadRight}, {nameof(PadTop)}: {PadTop}, {nameof(PadBottom)}: {PadBottom}, {nameof(BiasTerm)}: {BiasTerm}, {nameof(WeightDataSize)}: {WeightDataSize}, {nameof(OutputPadRight)}: {OutputPadRight}, {nameof(OutputPadBottom)}: {OutputPadBottom}, {nameof(OutputW)}: {OutputW}, {nameof(OutputH)}: {OutputH}"; } + +public record ConvArgs +{ + public ConvArgs(float[] weightData, float[] biasData, int numOutput, int kernelW, int kernelH, int dilationW, int dilationH, int strideW, int strideH, int padLeft, int padRight, int padTop, int padBottom, float padValue, int biasTerm, int weightDataSize, int int8ScaleTerm, int activationType, float[] activationParams, int dynamicWeight, int groups) + { + WeightData = weightData; + BiasData = biasData; + NumOutput = numOutput; + KernelW = kernelW; + KernelH = kernelH; + DilationW = dilationW; + DilationH = dilationH; + StrideW = strideW; + StrideH = strideH; + PadLeft = padLeft; + PadRight = padRight; + PadTop = padTop; + PadBottom = padBottom; + PadValue = padValue; + BiasTerm = biasTerm; + WeightDataSize = weightDataSize; + Int8ScaleTerm = int8ScaleTerm; + ActivationType = activationType; + ActivationParams = activationParams; + DynamicWeight = dynamicWeight; + Groups = groups; + } + + /// + /// Gets input. + /// + public float[] WeightData { get; } + + /// + /// Gets BiasData. + /// + public float[] BiasData { get; } + + /// + /// Gets NumOutput. + /// + public int NumOutput { get; } + + /// + /// Gets KernelW. + /// + public int KernelW { get; } + + /// + /// Gets KernelH. + /// + public int KernelH { get; } + + /// + /// Gets DilationW. + /// + public int DilationW { get; } + + /// + /// Gets DilationH. + /// + public int DilationH { get; } + + /// + /// Gets StrideW. + /// + public int StrideW { get; } + + /// + /// Gets StrideH. + /// + public int StrideH { get; } + + /// + /// Gets PadLeft. + /// + public int PadLeft { get; } + + /// + /// Gets PadRight. + /// + public int PadRight { get; } + + /// + /// Gets PadTop. + /// + public int PadTop { get; } + + /// + /// Gets PadBottom. + /// + public int PadBottom { get; } + + /// + /// Gets PadValue. + /// + public float PadValue { get; } + + /// + /// Gets BiasTerm. + /// + public int BiasTerm { get; } + + /// + /// Gets WeightDataSize. + /// + public int WeightDataSize { get; } + + /// + /// Gets Int8ScaleTerm. + /// + public int Int8ScaleTerm { get; } + + /// + /// Gets ActivationType. + /// + public int ActivationType { get; } + + /// + /// Gets ActivationParams. + /// + public float[] ActivationParams { get; } + + /// + /// Gets DynamicWeight. + /// + public int DynamicWeight { get; } + + /// + /// Gets Groups. + /// + public int Groups { get; } +} diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs index 9924939d48..ff165800f5 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs @@ -59,8 +59,13 @@ public IRType Visit(NcnnBinary target, TensorType lhs, TensorType rhs) /// public IRType Visit(ITypeInferenceContext context, NcnnBinary target) { - var inputA = context.CheckArgumentType(target, NcnnBinary.InputA); - return Visit(inputA); + return target.LorR switch + { + 1 => Visit(context.CheckArgumentType(target, NcnnBinary.InputA), new TensorType(context.CheckArgumentType(target, NcnnBinary.InputA).DType, target.ConstShape)), + 2 => Visit(new TensorType(context.CheckArgumentType(target, NcnnBinary.InputA).DType, target.ConstShape), context.CheckArgumentType(target, NcnnBinary.InputA)), + 0 => Visit(context.CheckArgumentType(target, NcnnBinary.InputA), context.CheckArgumentType(target, NcnnBinary.InputB)), + _ => throw new NotSupportedException("Never reach here, LorR without fourth situation."), + }; } /// @@ -125,8 +130,8 @@ private static BinaryOp MapBinaryOp(BinaryOperationType binaryOp) => // => BinaryOperationType.RATAN2, }; - private IRType Visit(TensorType input) + private IRType Visit(TensorType inputA, TensorType inputB) { - return input; + return TypeInference.BroadcastType(inputA, inputB); } } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs index 8a05bec977..de1d623113 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs @@ -70,6 +70,10 @@ public sealed partial class NcnnBinary : Op /// public override string DisplayProperty() { + if (ConstShape != null) + { + return $"BinaryOp.{OpType}, Const Shape: {String.Join(",", ConstShape)}"; + } return $"BinaryOp.{OpType}"; } } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs index 4abb3fdcfe..5145eca852 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs @@ -28,8 +28,8 @@ public partial class LowerBinary : RewriteRule public override Pattern Pattern { get; } = IsBinary( target_name: "binary", _ => true, - IsWildcard("inputA") with { TypePattern = IsFloat() & HasRank(x => x <= 3) }, - IsWildcard("inputB") with { TypePattern = IsFloat() & HasRank(x => x <= 3) }); + IsWildcard("inputA") with { TypePattern = IsFloat() }, + IsWildcard("inputB") with { TypePattern = IsFloat() }); private static BinaryOperationType? MapBinaryOp(BinaryOp binaryOp) => binaryOp switch @@ -62,11 +62,6 @@ public partial class LowerBinary : RewriteRule private int[] FixShape(int[] shape, int r) { - if (shape.Length == 1) - { - return shape; - } - var newShape = shape.ToList(); for (int i = r - shape.Length; i > 0; i--) { @@ -80,28 +75,117 @@ private int[] FixShape(int[] shape, int r) { if (MapBinaryOp(binary.BinaryOp) is BinaryOperationType op) { - var r = Math.Max(inputA.CheckedShape.Rank, inputB.CheckedShape.Rank); + int r = Math.Max(inputA.CheckedShape.Rank, inputB.CheckedShape.Rank); + if (r > 4) + { + return null; + } + + Call b; + if (inputA is Const) { + // A var constA = ((TensorConst)inputA).Value; - var constShape = FixShape(inputA.CheckedShape.ToValueArray(), r); + var aShape = FixShape(inputA.CheckedShape.ToValueArray(), r).ToList(); + + // B var newB = Reshape(inputB, FixShape(inputB.CheckedShape.ToValueArray(), r)); var newInputB = new Var(newB.CheckedType); - return new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputB }, op, 1, constA.ToArray(), constShape), new[] { newInputB }), newB); + + // Constant can not support 4D unless 0-D is 1. + if (r == 4) + { + if (newB.CheckedShape[0].FixedValue != 1 || aShape[0] != 1) + { + return null; + } + + newB = Squeeze(newB, new[] { 0 }); + newInputB = new Var(newB.CheckedType); + } + + while (aShape[0] == 1 && aShape.Count > 3 && aShape.Count > newB.CheckedShape.Count) + { + aShape.RemoveAt(0); + } + + b = new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputB }, op, 1, constA.ToArray(), aShape.ToArray()), new[] { newInputB }), newB); } else if (inputB is Const) { + // A var newA = Reshape(inputA, FixShape(inputA.CheckedShape.ToValueArray(), r)); var newInputA = new Var(newA.CheckedType); + + // B var constB = ((TensorConst)inputB).Value; - var constShape = FixShape(inputB.CheckedShape.ToValueArray(), r); - return new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputA }, op, 2, constB.ToArray(), constShape), newInputA), newA); + var bShape = FixShape(inputB.CheckedShape.ToValueArray(), r).ToList(); + + if (r == 4) + { + if (newA.CheckedShape[0].FixedValue != 1 || bShape[0] != 1) + { + return null; + } + + newA = Squeeze(newA, new[] { 0 }); + newInputA = new Var(newA.CheckedType); + } + + while (bShape[0] == 1 && bShape.Count > 3 && bShape.Count > newA.CheckedShape.Count) + { + bShape.RemoveAt(0); + } + + b = new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputA }, op, 2, constB.ToArray(), bShape.ToArray()), newInputA), newA); + } + else + { + if (r < 4) + { + var newInputA = new Var(inputA.CheckedType); + var newInputB = new Var(inputB.CheckedType); + b = new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputA, newInputB }, op, 0, null, null), new[] { newInputA, newInputB }), inputA, inputB); + } + else + { + var newA = inputA; + var newB = inputB; + var newInputA = new Var(newA.CheckedType); + var newInputB = new Var(newB.CheckedType); + if (inputA.CheckedShape[0].FixedValue != 1 && inputA.CheckedShape.Rank == r) + { + return null; + } + else + { + newA = Squeeze(inputA, new[] { 0 }); + newInputA = new Var(newA.CheckedType); + } + + if (inputB.CheckedShape[0].FixedValue != 1 && inputB.CheckedShape.Rank == r) + { + var aa = inputB.CheckedShape.AsEnumerable().Select(x => x.FixedValue).ToArray(); + return null; + } + else + { + newB = Squeeze(inputB, new[] { 0 }); + newInputB = new Var(newB.CheckedType); + } + + b = new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputA, newInputB }, op, 0, null, null), new[] { newInputA, newInputB }), newA, newB); + } + } + + if (r == 4) + { + return Unsqueeze(b, new[] { 0 }); } else { - var newInputA = new Var(inputA.CheckedType); - var newInputB = new Var(inputB.CheckedType); - return new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputA, newInputB }, op, 0, null, null), new[] { newInputA, newInputB }), inputA, inputB); + return b; } } diff --git a/tests/importer/onnx_/basic/test_binary.py b/tests/importer/onnx_/basic/test_binary.py index 6a4cecf9c0..b2d9368674 100644 --- a/tests/importer/onnx_/basic/test_binary.py +++ b/tests/importer/onnx_/basic/test_binary.py @@ -23,7 +23,10 @@ def _make_module(v_shape): class BinaryModule(torch.nn.Module): def __init__(self): super(BinaryModule, self).__init__() - self.v = torch.from_numpy(np.ones(v_shape).astype(np.float32)) + self.v = torch.from_numpy(np.random.random(v_shape).astype(np.float32)+3) + self.v2 = torch.from_numpy(np.random.random(v_shape).astype(np.float32)) + self.v3 = torch.from_numpy(np.random.random(v_shape).astype(np.float32)+1) + self.v4 = torch.from_numpy(np.random.random(v_shape).astype(np.float32)+2) def forward(self, x): outs = [] @@ -44,6 +47,7 @@ def forward(self, x): [64, 3], [3, 64, 3], [8, 3, 64, 3], + [1, 1, 64, 3], ] rhs_shapes = [ From 662020f5cb735430fbf40fb4aa638bb52650994d Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 21 Mar 2024 14:44:35 +0800 Subject: [PATCH 090/132] - [X] Split DWConv and Conv - [X] Refact conv args --- .../CodeGen/Ncnn/NcnnEmitter.cs | 71 +++++++------ .../Evaluator/Ncnn/NcnnConv.cs | 30 +++--- .../Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs | 4 +- .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs | 100 +----------------- .../Passes/Rules/Ncnn/LowerConv2d.cs | 14 ++- 5 files changed, 74 insertions(+), 145 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 077e6c5852..9c1c2f0408 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -18,7 +18,6 @@ namespace Nncase.CodeGen.Ncnn; internal class NcnnEmitter { public List? RData; - private readonly NcnnModel _model; private readonly BinaryWriter _binWriter; @@ -129,34 +128,44 @@ public void Concat(string name, string[] input, int axis) }); } - public void Conv(string name, string input, float[] weightsData, float[] biasData, int numOutput, int kernelW, int kernelH, int dilationW, int dilationH, int strideW, int strideH, int padLeft, int padRight, int padBottom, int padTop, int biasTerm, int weightsDataSize, int int8Flag, int actType, float[] actParams, float padValue, int dynamicFlag) - { - AddLayer("Convolution", name, new[] { input }, new[] { name }, new ParamDict - { - [0] = new ParamValue { Kind = ParamKind.Int, IntValue = numOutput }, - [1] = new ParamValue { Kind = ParamKind.Int, IntValue = kernelW }, - [11] = new ParamValue { Kind = ParamKind.Int, IntValue = kernelH }, - [2] = new ParamValue { Kind = ParamKind.Int, IntValue = dilationW }, - [12] = new ParamValue { Kind = ParamKind.Int, IntValue = dilationH }, - [3] = new ParamValue { Kind = ParamKind.Int, IntValue = strideW }, - [13] = new ParamValue { Kind = ParamKind.Int, IntValue = strideH }, - [4] = new ParamValue { Kind = ParamKind.Int, IntValue = padLeft }, - [14] = new ParamValue { Kind = ParamKind.Int, IntValue = padTop }, - [16] = new ParamValue { Kind = ParamKind.Int, IntValue = padRight }, - [15] = new ParamValue { Kind = ParamKind.Int, IntValue = padBottom }, - [5] = new ParamValue { Kind = ParamKind.Int, IntValue = biasTerm }, - [6] = new ParamValue { Kind = ParamKind.Int, IntValue = weightsDataSize }, - [7] = new ParamValue { Kind = ParamKind.Int, IntValue = 0 }, // Group - [8] = new ParamValue { Kind = ParamKind.Int, IntValue = int8Flag }, - - // [9] = new ParamValue { Kind = ParamKind.Int, IntValue = actType }, - // [10] = new ParamValue { Kind = ParamKind.ArrayOfFloat, TensorValue = actParams }, - [18] = new ParamValue { Kind = ParamKind.Float, FloatValue = padValue }, - [19] = new ParamValue { Kind = ParamKind.Int, IntValue = dynamicFlag }, - }); + public void Conv(string name, string input, ConvArgs args) + { + var actData = new List { args.ActivationParams.Length }; + actData.AddRange(args.ActivationParams); + + var param = new ParamDict(); + param.Add(0, new ParamValue { Kind = ParamKind.Int, IntValue = args.NumOutput }); + param.Add(1, new ParamValue { Kind = ParamKind.Int, IntValue = args.KernelW }); + param.Add(11, new ParamValue { Kind = ParamKind.Int, IntValue = args.KernelH }); + param.Add(2, new ParamValue { Kind = ParamKind.Int, IntValue = args.DilationW }); + param.Add(12, new ParamValue { Kind = ParamKind.Int, IntValue = args.DilationH }); + param.Add(3, new ParamValue { Kind = ParamKind.Int, IntValue = args.StrideW }); + param.Add(13, new ParamValue { Kind = ParamKind.Int, IntValue = args.StrideH }); + param.Add(4, new ParamValue { Kind = ParamKind.Int, IntValue = args.PadLeft }); + param.Add(15, new ParamValue { Kind = ParamKind.Int, IntValue = args.PadRight }); + param.Add(14, new ParamValue { Kind = ParamKind.Int, IntValue = args.PadTop }); + param.Add(16, new ParamValue { Kind = ParamKind.Int, IntValue = args.PadBottom }); + param.Add(18, new ParamValue { Kind = ParamKind.Float, FloatValue = args.PadValue }); + param.Add(5, new ParamValue { Kind = ParamKind.Int, IntValue = args.BiasTerm }); + param.Add(6, new ParamValue { Kind = ParamKind.Int, IntValue = args.WeightDataSize }); + param.Add(7, new ParamValue { Kind = ParamKind.Int, IntValue = args.Groups }); + param.Add(8, new ParamValue { Kind = ParamKind.Int, IntValue = args.Int8ScaleTerm }); + param.Add(9, new ParamValue { Kind = ParamKind.Int, IntValue = args.ActivationType }); + param.Add(-10, new ParamValue { Kind = ParamKind.ArrayOfIntOrFloat, TensorValue = actData.ToArray() }); + param.Add(19, new ParamValue { Kind = ParamKind.Int, IntValue = args.DynamicWeight }); + + if (args.Groups > 1) + { + AddLayer("ConvolutionDepthWise", name, new[] { input }, new[] { name }, param); + } + else + { + AddLayer("Convolution", name, new[] { input }, new[] { name }, param); + } + WriteFloatArray(new float[] { 0 }); // quantize flag [Not exist in ncnn op.md] - WriteFloatArray(weightsData); - WriteFloatArray(biasData); + WriteFloatArray(args.WeightData); + WriteFloatArray(args.BiasData); } public void Cumsum(string name, string input, int axis) @@ -326,9 +335,11 @@ public void Reshape(string[] name, string input, int[] newshape) var args = new ParamDict(); const int i = 0; - foreach (var item in newshape.Reverse()) + + // foreach (var item in newshape.Reverse()) + foreach (var (index, value) in newshape.Reverse().Select((value, index) => (index, value))) { - args.Add(i, new ParamValue { Kind = ParamKind.Int, IntValue = item }); + args.Add(index, new ParamValue { Kind = ParamKind.Int, IntValue = value }); } AddLayer("Reshape", name[0], new[] { input }, name, args); diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConv.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConv.cs index ec130db3c7..0cdd7af752 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConv.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConv.cs @@ -23,10 +23,10 @@ public class NcnnConvEvaluator : IEvaluator, ITypeInferencer public IValue Visit(IEvaluateContext context, NcnnConv conv) { var inputs = context.GetOrtArgumentValue(conv, NcnnConv.Input); - var weights = new Tensor(conv.WeightData, new[] { conv.NumOutput, conv.WeightDataSize / (conv.KernelH * conv.KernelW), conv.KernelH, conv.KernelW }).ToOrtTensor(); - var bias = new Tensor(conv.BiasData, new[] { conv.NumOutput }).ToOrtTensor(); - var result = OrtKI.Conv(inputs, weights, bias, "NOTSET", new long[] { conv.DilationH, conv.DilationW }, 1, new long[] { conv.KernelH, conv.KernelW }, new long[] { conv.PadLeft, conv.PadTop, conv.PadRight, conv.PadBottom }, new long[] { conv.StrideH, conv.StrideW }); - return OrtKI.Clip(result, conv.ActivationParams[0], conv.ActivationParams[1]).ToValue(); + var weights = new Tensor(conv.Args.WeightData, new[] { conv.Args.NumOutput, conv.Args.WeightDataSize / (conv.Args.KernelH * conv.Args.KernelW), conv.Args.KernelH, conv.Args.KernelW }).ToOrtTensor(); + var bias = new Tensor(conv.Args.BiasData, new[] { conv.Args.NumOutput }).ToOrtTensor(); + var result = OrtKI.Conv(inputs, weights, bias, "NOTSET", new long[] { conv.Args.DilationH, conv.Args.DilationW }, 1, new long[] { conv.Args.KernelH, conv.Args.KernelW }, new long[] { conv.Args.PadLeft, conv.Args.PadTop, conv.Args.PadRight, conv.Args.PadBottom }, new long[] { conv.Args.StrideH, conv.Args.StrideW }); + return OrtKI.Clip(result, conv.Args.ActivationParams[0], conv.Args.ActivationParams[1]).ToValue(); } /// @@ -40,9 +40,9 @@ public IRType Visit(ITypeInferenceContext context, NcnnConv target) public Cost Visit(ICostEvaluateContext context, NcnnConv target) { var inputType = context.GetArgumentType(target, NcnnConv.Input); - var weightsType = new TensorType(DataTypes.Float32, new[] { target.WeightDataSize }); - var biasType = new TensorType(DataTypes.Float32, new[] { target.NumOutput }); - var macPerElement = (2 * target.WeightDataSize / target.NumOutput) - 1; + var weightsType = new TensorType(DataTypes.Float32, new[] { target.Args.WeightDataSize }); + var biasType = new TensorType(DataTypes.Float32, new[] { target.Args.NumOutput }); + var macPerElement = (2 * target.Args.WeightDataSize / target.Args.NumOutput) - 1; var outputType = context.GetReturnType(); @@ -68,20 +68,20 @@ public Cost Visit(ICostEvaluateContext context, NcnnConv target) private IRType Visit(TensorType input, NcnnConv conv) { var outputShape = input.Shape.ToList(); - var kernelShape = new[] { conv.NumOutput, conv.WeightDataSize / (conv.NumOutput * conv.KernelH * conv.KernelW), conv.KernelH, conv.KernelW }; - outputShape[0] = conv.NumOutput; + var kernelShape = new[] { conv.Args.NumOutput, conv.Args.WeightDataSize / (conv.Args.NumOutput * conv.Args.KernelH * conv.Args.KernelW), conv.Args.KernelH, conv.Args.KernelW }; + outputShape[0] = conv.Args.NumOutput; outputShape[1] = IR.TypePatternUtility.GetWindowedOutputSize( - input.Shape[1].FixedValue + conv.PadLeft + conv.PadRight, + input.Shape[1].FixedValue + conv.Args.PadLeft + conv.Args.PadRight, kernelShape[2], - conv.StrideH, - conv.DilationH, + conv.Args.StrideH, + conv.Args.DilationH, false); outputShape[2] = IR.TypePatternUtility.GetWindowedOutputSize( - input.Shape[2].FixedValue + conv.PadTop + conv.PadBottom, + input.Shape[2].FixedValue + conv.Args.PadTop + conv.Args.PadBottom, kernelShape[3], - conv.StrideW, - conv.DilationW, + conv.Args.StrideW, + conv.Args.DilationW, false); return new TensorType(GetTensorType(input).DType, outputShape.ToArray()); } diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs index 767b5ae877..60d1e59d14 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/Functional.cs @@ -35,8 +35,8 @@ public static Call NcnnConcat(Expr[] expr, int axis) => new Call(new NcnnConcat(axis), new IR.Tuple(expr)); // In ncnn param file, lauout is [w, h] for kernel, dilation - public static Call NcnnConv(Expr expr, float[] weightsData, float[] biasData, int numOutput, int[] kernel, int[] dilation, int[] strides, int[] pads, int biasTerm, int weightsDataSize, int int8Flag, int actType, float[] actParams, float padValue, int dynamicFlag) => - new Call(new NcnnConv(weightsData, biasData, numOutput, kernel[1], kernel[0], dilation[1], dilation[0], strides[1], strides[0], pads[0], pads[1], pads[2], pads[3], padValue, biasTerm, weightsDataSize, int8Flag, actType, actParams, dynamicFlag), expr); + public static Call NcnnConv(Expr expr, ConvArgs args) => + new Call(new NcnnConv(args), expr); public static Call NcnnCumsum(Expr expr, int axis) => new Call(new NcnnCumsum(axis), expr); diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs index cbb104eca9..c1b317aac3 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Nncase.ArgsStruct; using Nncase.PatternMatch; namespace Nncase.IR.Ncnn; @@ -23,104 +24,9 @@ public sealed partial class NcnnConv : Op public static readonly ParameterInfo Input = new(typeof(NcnnConv), 0, "input"); /// - /// Gets input. - /// - public float[] WeightData { get; } - - /// - /// Gets BiasData. - /// - public float[] BiasData { get; } - - /// - /// Gets NumOutput. - /// - public int NumOutput { get; } - - /// - /// Gets KernelW. - /// - public int KernelW { get; } - - /// - /// Gets KernelH. - /// - public int KernelH { get; } - - /// - /// Gets DilationW. - /// - public int DilationW { get; } - - /// - /// Gets DilationH. - /// - public int DilationH { get; } - - /// - /// Gets StrideW. - /// - public int StrideW { get; } - - /// - /// Gets StrideH. - /// - public int StrideH { get; } - - /// - /// Gets PadLeft. - /// - public int PadLeft { get; } - - /// - /// Gets PadRight. - /// - public int PadRight { get; } - - /// - /// Gets PadTop. - /// - public int PadTop { get; } - - /// - /// Gets PadBottom. - /// - public int PadBottom { get; } - - /// - /// Gets PadValue. - /// - public float PadValue { get; } - - /// - /// Gets BiasTerm. - /// - public int BiasTerm { get; } - - /// - /// Gets WeightDataSize. - /// - public int WeightDataSize { get; } - - /// - /// Gets Int8ScaleTerm. - /// - public int Int8ScaleTerm { get; } - - /// - /// Gets ActivationType. - /// - public int ActivationType { get; } - - /// - /// Gets ActivationParams. - /// - public float[] ActivationParams { get; } - - /// - /// Gets DynamicWeight. + /// Gets args of NcnnConv. /// - public int DynamicWeight { get; } + public ConvArgs Args { get; } /// public override string DisplayProperty() diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs index 70cc555652..dd9ebec306 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs @@ -7,6 +7,7 @@ using System.Text; using System.Threading.Tasks; using NetFabric.Hyperlinq; +using Nncase.ArgsStruct; using Nncase.IR; using Nncase.IR.Math; using Nncase.IR.Ncnn; @@ -53,9 +54,20 @@ public partial class LowerConv : RewriteRule int activationType = 3; int dynamicWeight = 0; + var args = new ConvArgs(weights.ToArray(), bias.ToArray(), numOutput.FixedValue, + kernelW.FixedValue, kernelH.FixedValue, + dilationW, dilationH, + strideW, strideH, + padLeft, padRight, padTop, padBottom, padValue, + biasTerm, + weightsDataSize.FixedValue, + int8ScaleTerm, + activationType, fusedClamp, dynamicWeight, + groups); + var inRes = Squeeze(input, new[] { 0 }); var inResO = new Var(inRes.CheckedType); - var conv = new Call(new Fusion("ncnn", NcnnConv(inResO, weights.ToArray(), bias.ToArray(), numOutput.FixedValue, new[] { kernelH.FixedValue, kernelW.FixedValue }, new[] { dilationH, dilationW }, new[] { strideH, strideW }, new[] { padLeft, padRight, padBottom, padTop }, biasTerm, weightsDataSize.FixedValue, int8ScaleTerm, activationType, fusedClamp, padValue, dynamicWeight), new[] { inResO }), inRes); + var conv = new Call(new Fusion("ncnn", NcnnConv(inResO, args), new[] { inResO }), inRes); var outRes = Unsqueeze(conv, new[] { 0 }); From 416a8cddfa8ce1d902fa59bb3ed8ac615b6b8de1 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 21 Mar 2024 14:49:01 +0800 Subject: [PATCH 091/132] fix bin data serialize --- .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 8 +++---- src/Nncase.Core/Runtime/Ncnn/ParamDict.cs | 22 +++++++++++++++++-- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 346421f250..8f60d74e2c 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -123,7 +123,7 @@ protected override string VisitLeafCall(Call expr) 0 => new string[] { ExprMemo[expr.Arguments[0]], ExprMemo[expr.Arguments[1]] }, 1 => new string[] { string.Empty, ExprMemo[expr.Arguments[0]] }, 2 => new string[] { ExprMemo[expr.Arguments[0]], string.Empty }, - _ => throw new NotImplementedException("Not found binary emmiter."), + _ => throw new NotImplementedException("Never reach here."), }; _emitter.Binary(names[0], inString[0], inString[1], op.OpType, op.LorR, op.ConstInput, op.ConstShape); break; @@ -144,7 +144,7 @@ protected override string VisitLeafCall(Call expr) _emitter.Concat(names[0], in_.ToArray(), op.Axis); break; case NcnnConv op: - _emitter.Conv(names[0], ExprMemo[expr.Arguments[0]], op.WeightData, op.BiasData, op.NumOutput, op.KernelW, op.KernelH, op.DilationW, op.DilationH, op.StrideW, op.StrideH, op.PadLeft, op.PadTop, op.PadRight, op.PadBottom, op.BiasTerm, op.WeightDataSize, op.Int8ScaleTerm, op.ActivationType, op.ActivationParams, op.PadValue, op.DynamicWeight); + _emitter.Conv(names[0], ExprMemo[expr.Arguments[0]], op.Args); break; case NcnnCumsum op: _emitter.Cumsum(names[0], ExprMemo[expr.Arguments[0]], op.Axis); @@ -219,7 +219,7 @@ protected override string VisitLeafCall(Call expr) 0 => new string[] { ExprMemo[expr.Arguments[0]], ExprMemo[expr.Arguments[1]] }, 1 => new string[] { string.Empty, ExprMemo[expr.Arguments[0]] }, 2 => new string[] { ExprMemo[expr.Arguments[0]], string.Empty }, - _ => throw new NotImplementedException("Not found MatMul emmiter."), + _ => throw new NotImplementedException("Never reach here."), }; _emitter.Matmul(names.ToArray(), inString[0], inString[1], op.LorR, op.ConstInput, op.ConstShape); break; @@ -245,7 +245,7 @@ protected override string VisitLeafCall(Call expr) _emitter.Unsqueeze(names.ToArray(), ExprMemo[expr.Arguments[0]], op.Dims); break; default: - throw new NotSupportedException("Not support in Ncnn ops emitter"); + throw new NotSupportedException($"Not support {nameof(expr.Target)} in Ncnn ops emitter"); } // serialize outputs to string. diff --git a/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs b/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs index ee3ae1aa42..10c06d3089 100644 --- a/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs +++ b/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs @@ -136,7 +136,8 @@ public void Serialize(TextWriter writer) var paramValue = field.Value; var isArray = paramValue.Kind is ParamKind.ArrayOfFloat or ParamKind.ArrayOfIntOrFloat or ParamKind.ArrayOfInt; - var isFloat = paramValue.Kind is ParamKind.ArrayOfFloat or ParamKind.Float; + var isFloat = paramValue.Kind is ParamKind.ArrayOfFloat or ParamKind.ArrayOfIntOrFloat or ParamKind.Float; + var mixIntAndFloat = paramValue.Kind is ParamKind.ArrayOfIntOrFloat; if (isArray) { @@ -149,7 +150,24 @@ public void Serialize(TextWriter writer) { if (isFloat) { - writer.Write(string.Join(',', paramValue.TensorValue!.Cast().Select(x => x.ToString("e")))); + var sourceData = paramValue.TensorValue!.ToArray(); + + if (mixIntAndFloat) + { + int size = (int)sourceData[0]; + writer.Write($"{size},"); + } + + // Console.WriteLine($"{String.Join(",", sourceData[1..])}"); + writer.Write(string.Join(',', sourceData[1..].Select(x => + { + return x switch + { + _ when float.IsPositiveInfinity(x) => "3.402823e+38", + _ when float.IsNegativeInfinity(x) => "-3.402823e+38", + _ => x.ToString("e"), + }; + }))); } else { From a21fc8f7d0ddb6c58944d155cac2084ae171699c Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 21 Mar 2024 17:14:07 +0800 Subject: [PATCH 092/132] fix reshape --- .../Evaluator/Ncnn/NcnnReshape.cs | 1 + .../Passes/Rules/Ncnn/LowerReshape.cs | 55 +++++++++++++------ 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReshape.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReshape.cs index 4083749b91..77dec2298b 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReshape.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReshape.cs @@ -68,6 +68,7 @@ private IRType Visit(TensorType input, int[] newShape) if (newShape[i] == -1) { negAxis = i; + break; } } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReshape.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReshape.cs index 83a6b3be32..fc13bc6154 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReshape.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReshape.cs @@ -1,6 +1,7 @@ // Copyright (c) Canaan Inc. All rights reserved. // Licensed under the Apache license. See LICENSE file in the project root for full license information. +using DryIoc.ImTools; using NetFabric.Hyperlinq; using Nncase.IR; using Nncase.PatternMatch; @@ -21,28 +22,46 @@ public partial class LowerReshape : RewriteRule private Expr? GetReplace(Expr input, Expr shape) { - if (input.CheckedShape.Count < 5) + if (input.CheckedShape.Rank < 5) { - var inResO = new Var(input.CheckedType); - var outputShape = shape.Evaluate().AsTensor().ToArray(); + Call r; + var newInput = input; + var newInputVar = new Var(input.CheckedType); - return new Call(new Fusion("ncnn", NcnnReshape(inResO, outputShape), new[] { inResO }), input); + var outputShape = shape.Evaluate().AsTensor().ToArray().ToList(); + + int needSqueeze = 0; + if (newInput.CheckedShape[0].FixedValue != 1 && newInput.CheckedShape.Rank == 4) + { + return null; + } + else if (newInput.CheckedShape[0].FixedValue == 1 && newInput.CheckedShape.Rank == 4) + { + newInput = Squeeze(newInput, new[] { 0 }); + newInputVar = new Var(newInput.CheckedType); + needSqueeze += 1; + } + + if (outputShape.Count > 4 || (outputShape.Count == 4 && outputShape[0] != 1)) + { + return null; + } + else if (outputShape.Count == 4 && outputShape[0] == 1 && needSqueeze == 1) + { + outputShape.RemoveAt(0); // Avoid reshape input to 4D with multi batchsize. + needSqueeze += 2; + } // left 3D shape. + + r = new Call(new Fusion("ncnn", NcnnReshape(newInputVar, outputShape.ToArray()), new[] { newInputVar }), newInput); + + if (needSqueeze == 3) + { + return Unsqueeze(r, new[] { 0 }); + } + + return r; } return null; - - // // TODO: split input - // // if shape.Length == input.Length means that batchSize need reshape. Ncnn can't support. - // if (input.CheckedShape.ToList()[0] != 1 || input.CheckedShape.Count == shape.CheckedShape.Count) - // { - // return null; - // } - // - // var inRes = Squeeze(input, new[] { 0 }); - // var inResO = new Var(inRes.CheckedType); - // var outputShape = shape.Evaluate().AsTensor().ToArray(); - // var reshape = new Call(new Fusion("ncnn", NcnnReshape(inResO, outputShape), new[] { inResO }), input); - // - // return reshape; } } From ac822172f560c2c99cfa30fb034d9296091dfafd Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 22 Mar 2024 10:59:23 +0800 Subject: [PATCH 093/132] fix --- .../CodeGen/Ncnn/NcnnEmitter.cs | 23 +++++++------------ .../Passes/Rules/Ncnn/LowerSqueeze.cs | 21 +++++++++++++++-- .../Passes/Rules/Ncnn/LowerUnsqueeze.cs | 4 +--- .../Targets/CPUTarget.cs | 8 ++++--- 4 files changed, 33 insertions(+), 23 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 9c1c2f0408..a408ac78a1 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -327,19 +327,12 @@ public void Reduction(string[] name, string input, ReductionArgs reductionArgs) public void Reshape(string[] name, string input, int[] newshape) { - if (newshape.Length == 4) - { - newshape[2] *= newshape[3]; - newshape.RemoveAt(3); - } - var args = new ParamDict(); - const int i = 0; - - // foreach (var item in newshape.Reverse()) - foreach (var (index, value) in newshape.Reverse().Select((value, index) => (index, value))) + int i = 0; + foreach (int item in newshape.Reverse()) { - args.Add(index, new ParamValue { Kind = ParamKind.Int, IntValue = value }); + args.Add(i, new ParamValue { Kind = ParamKind.Int, IntValue = item }); + i += 1; } AddLayer("Reshape", name[0], new[] { input }, name, args); @@ -359,16 +352,16 @@ public void Crop(string[] name, string input, CropArgs cropArgs) var args = new ParamDict(); // TODO: if need to fit torch crop, add other args into paramDict. - if (cropArgs.Axes.Length > 0) + if (cropArgs.Axes!.Length > 0) { var startData = new List { cropArgs.Axes.Length }; var endData = new List { cropArgs.Axes.Length }; var axisData = new List { cropArgs.Axes.Length }; for (int i = 0; i < cropArgs.Axes.Length; i++) { - startData.Add(cropArgs.Starts[i]); - endData.Add(cropArgs.Ends[i]); - axisData.Add(cropArgs.Axes[i]); + startData.Add(cropArgs.Starts![i]); + endData.Add(cropArgs.Ends![i]); + axisData.Add(cropArgs.Axes![i]); } args.Add(-9, new ParamValue { Kind = ParamKind.ArrayOfInt, TensorValue = startData.ToArray() }); diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs index c8134b0633..e8cf987d4b 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs @@ -24,8 +24,25 @@ public partial class LowerSqueeze : RewriteRule if (input.CheckedShape.Count < 5) { var inResO = new Var(input.CheckedType); - // var newDims = dims[0] == 0 ? dims[1..] : dims; - return new Call(new Fusion("ncnn", NcnnSqueeze(inResO, dims), new[] { inResO }), input); + + var newDims = new List(); + foreach (int item in dims) + { + if (item > 4 || item < -3) + { + return null; + } + else + { + newDims.Add(item); + } + if(input.CheckedShape[item] != 1) + { + return null; + } + } + + return new Call(new Fusion("ncnn", NcnnSqueeze(inResO, newDims.ToArray()), new[] { inResO }), input); } return null; diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnsqueeze.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnsqueeze.cs index e7dc4e96c9..c71ff61685 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnsqueeze.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnsqueeze.cs @@ -21,11 +21,9 @@ public partial class LowerUnsqueeze : RewriteRule private Expr? GetReplace(Expr input, int[] dims) { - if (input.CheckedShape.Count + dims.Length <= 5) + if (input.CheckedShape.Count + dims.Length < 5) { var inResO = new Var(input.CheckedType); - - // var newDims = dims[0] == 0 ? dims[1..] : dims; return new Call(new Fusion("ncnn", NcnnUnsqueeze(inResO, dims), new[] { inResO }), input); } diff --git a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs index 9d12f7f169..3a326a04c9 100644 --- a/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs +++ b/modules/Nncase.Modules.StackVM/Targets/CPUTarget.cs @@ -95,6 +95,10 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions p.Add(); p.Add(); + p.Add(); + p.Add(); + p.Add(); + // p.Add(); // ncnn dequantize int to float. }); @@ -102,13 +106,11 @@ public void RegisterTargetDependentPass(IPassManager passManager, CompileOptions { p.Add(); p.Add(); - p.Add(); - p.Add(); + p.Add(); }); passManager.AddWithName("RemoveSingleSqueezeAndUnsqueeze").Configure(p => { - p.Add(); p.Add(); p.Add(); }); From 47b7f82d1d12efd779b1a5131777fed47185cf4c Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 22 Mar 2024 14:05:19 +0800 Subject: [PATCH 094/132] fix warning --- .../Nncase.Modules.Ncnn/ArgsStruct/args.cs | 12 ++++------ .../CodeGen/Ncnn/NcnnEmitter.cs | 11 ++++++--- .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 4 ++-- .../Evaluator/Ncnn/NcnnBinary.cs | 4 ++-- .../Evaluator/Ncnn/NcnnCrop.cs | 6 ++--- .../Evaluator/Ncnn/NcnnPadding.cs | 12 ++++------ .../Evaluator/Ncnn/NcnnPooling.cs | 3 +-- .../Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs | 13 +++++----- .../Passes/Rules/Ncnn/LowerConv2d.cs | 24 ++++++++++++++----- .../Passes/Rules/Ncnn/LowerCrop.cs | 3 +-- .../Passes/Rules/Ncnn/LowerMatmul.cs | 3 +-- .../Passes/Rules/Ncnn/LowerSqueeze.cs | 3 ++- 12 files changed, 55 insertions(+), 43 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs index 2881b2e66b..9826158fcf 100644 --- a/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs +++ b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs @@ -8,9 +8,7 @@ namespace Nncase.ArgsStruct; /// public record PoolingArgs { - public PoolingArgs(int poolingType, int kernelW, int kernelH, int strideW, int strideH, int padLeft, int padRight, - int padTop, int padBottom, bool globalPooling, int padMode, bool avgPoolCountIncludePad, bool adaptivePooling, - int outW, int outH, bool ceilMode) + public PoolingArgs(int poolingType, int kernelW, int kernelH, int strideW, int strideH, int padLeft, int padRight, int padTop, int padBottom, bool globalPooling, int padMode, bool avgPoolCountIncludePad, bool adaptivePooling, int outW, int outH, bool ceilMode) { PoolingType = poolingType; KernelH = kernelH; @@ -158,10 +156,10 @@ public CropArgs(int woffset, int hoffset, int doffset, int coffset, int outw, in public record ConvTransposeArgs { - public ConvTransposeArgs(Tensor weightData = null, float[] biasData = null, int numOutput = default, int kernelW = default, int kernelH = default, int dilationW = default, int dilationH = default, int strideW = default, int strideH = default, int padLeft = default, int padRight = default, int padTop = default, int padBottom = default, int biasTerm = default, int weightDataSize = default, int activationType = default, float[] activationParams = null, int outputPadRight = default, int outputPadBottom = default, int outputW = default, int outputH = default) + public ConvTransposeArgs(Tensor? weightData = null, float[]? biasData = null, int numOutput = default, int kernelW = default, int kernelH = default, int dilationW = default, int dilationH = default, int strideW = default, int strideH = default, int padLeft = default, int padRight = default, int padTop = default, int padBottom = default, int biasTerm = default, int weightDataSize = default, int activationType = default, float[]? activationParams = null, int outputPadRight = default, int outputPadBottom = default, int outputW = default, int outputH = default) { - WeightData = weightData; - BiasData = biasData; + WeightData = weightData!; + BiasData = biasData!; NumOutput = numOutput; KernelW = kernelW; KernelH = kernelH; @@ -176,7 +174,7 @@ public ConvTransposeArgs(Tensor weightData = null, float[] biasData = null, int BiasTerm = biasTerm; WeightDataSize = weightDataSize; ActivationType = activationType; - ActivationParams = activationParams; + ActivationParams = activationParams!; OutputPadRight = outputPadRight; OutputPadBottom = outputPadBottom; OutputW = outputW; diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index a408ac78a1..e3d714f27e 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -17,15 +17,20 @@ namespace Nncase.CodeGen.Ncnn; internal class NcnnEmitter { - public List? RData; private readonly NcnnModel _model; private readonly BinaryWriter _binWriter; + private readonly List? _rData; public NcnnEmitter(BinaryWriter binWriter) { _model = new NcnnModel(); _binWriter = binWriter; - RData = new List(); + _rData = new List(); + } + + public List? GetRData() + { + return _rData; } public void SaveParam(Stream paramStream) @@ -569,7 +574,7 @@ private void AddLayer(string type, string name, string[] bottoms, string[] tops, private void WriteFloatArray(float[] data) { - RData.AddRange(data); + _rData!.AddRange(data); foreach (float value in data) { _binWriter.Write(value); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index 8f60d74e2c..c33d4f93d5 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -36,7 +36,7 @@ public NcnnFunctionBuilder(uint id, SectionManager sectionManager) protected override ILinkableFunction CreateLinkableFunction(uint id, BaseFunction callable, IReadOnlyList functionRefs, Stream text) { - return new NcnnLinkableFunction(id, callable, functionRefs, text, _inputs!, _outputs!, _emitter.RData!.ToArray()!); + return new NcnnLinkableFunction(id, callable, functionRefs, text, _inputs!, _outputs!, _emitter.GetRData()!.ToArray()!); } protected override void Compile(BaseFunction callable) @@ -125,7 +125,7 @@ protected override string VisitLeafCall(Call expr) 2 => new string[] { ExprMemo[expr.Arguments[0]], string.Empty }, _ => throw new NotImplementedException("Never reach here."), }; - _emitter.Binary(names[0], inString[0], inString[1], op.OpType, op.LorR, op.ConstInput, op.ConstShape); + _emitter.Binary(names[0], inString[0], inString[1], op.OpType, op.LorR, op.ConstInput!, op.ConstShape!); break; case NcnnCelu op: _emitter.Celu(names[0], ExprMemo[expr.Arguments[0]], op.Alpha); diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs index ff165800f5..a3cc410e3c 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnBinary.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using DryIoc; using NetFabric.Hyperlinq; using Nncase.CostModel; using Nncase.Evaluator; @@ -110,8 +111,7 @@ private static BinaryOp MapBinaryOp(BinaryOperationType binaryOp) => BinaryOperationType.MAX => BinaryOp.Max, BinaryOperationType.MIN => BinaryOp.Min, BinaryOperationType.POW => BinaryOp.Pow, - - // _ => null, + _ => throw new ArgumentException("Invalid binary op type"), // unsupported Binary ops // BinaryOp.Mod => diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCrop.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCrop.cs index 627fb08151..02956bc523 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCrop.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnCrop.cs @@ -63,10 +63,10 @@ public Metric Visit(IMetricEvaluateContext context, NcnnCrop target) private IRType Visit(TensorType input, CropArgs args) { var outputShape = input.Shape.ToArray(); - for (int i = 0; i < args.Axes.Length; i++) + for (int i = 0; i < args.Axes!.Length; i++) { - var tStart = args.Starts[i] >= 0 ? args.Starts[i] : args.Starts[i] + outputShape[args.Axes[i]].FixedValue; - var tEnd = args.Ends[i] >= 0 ? args.Ends[i] : args.Ends[i] + outputShape[args.Axes[i]].FixedValue; + int tStart = args.Starts![i] >= 0 ? args.Starts[i] : args.Starts[i] + outputShape[args.Axes[i]].FixedValue; + int tEnd = args.Ends![i] >= 0 ? args.Ends[i] : args.Ends[i] + outputShape[args.Axes[i]].FixedValue; outputShape[args.Axes[i] < 0 ? args.Axes[i] + outputShape.Length : args.Axes[i]] = System.Math.Abs(tEnd - tStart); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs index 99a2502b59..c82cd7d312 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs @@ -26,13 +26,11 @@ public IValue Visit(IEvaluateContext context, NcnnPadding padding) // ncnn not support N var pads = new Tensor(new int[] { 0, 0, padding.Front, padding.Top, padding.Left, padding.Behind, padding.Bottom, padding.Right }); - return OrtKI.Pad(input, pads.ToOrtTensor(), padding.Value, padding.Type switch - { - 0 => "Constant", - 1 => "Reflect", - 2 => "Edge", - _ => "Symmetric", - }).ToValue(); + return OrtKI.Pad( + input, + pads.ToOrtTensor(), + padding.Value, + padding.Type switch { 0 => "Constant", 1 => "Reflect", 2 => "Edge", _ => "Symmetric", }).ToValue(); } /// diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPooling.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPooling.cs index 5cebbca4a2..bd806a09f2 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPooling.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPooling.cs @@ -86,8 +86,7 @@ private IRType Visit(NcnnPooling pooling, TensorType input) var padData = new long[] { pooling.Args.PadTop, pooling.Args.PadBottom, pooling.Args.PadLeft, pooling.Args.PadRight }; _ = pooling.Args.AvgPoolCountIncludePad ? 1 : 0; var newInput = new TensorType(input.DType, input.Shape.InsertAndClone(0, 1)); - var output_ = TypeInference.ReduceWindow2DType(newInput, kernelSize, stride, - Tensor.From(padData, new[] { 2, 2, }), pooling.Args.CeilMode); + var output_ = TypeInference.ReduceWindow2DType(newInput, kernelSize, stride, Tensor.From(padData, new[] { 2, 2, }), pooling.Args.CeilMode); if (output_ is TensorType t) { var newShape = t.Shape.ToArray(); diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs index de1d623113..1ada289725 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnBinary.cs @@ -22,11 +22,11 @@ public enum BinaryOperationType POW = 6, // Not support below - RSUB = 7, - RDIV = 8, - RPOW = 9, - ATAN2 = 10, - RATAN2 = 11, + // RSUB = 7, + // RDIV = 8, + // RPOW = 9, + // ATAN2 = 10, + // RATAN2 = 11, } /// @@ -72,8 +72,9 @@ public override string DisplayProperty() { if (ConstShape != null) { - return $"BinaryOp.{OpType}, Const Shape: {String.Join(",", ConstShape)}"; + return $"BinaryOp.{OpType}, Const Shape: {string.Join(",", ConstShape!)}"; } + return $"BinaryOp.{OpType}"; } } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs index dd9ebec306..38e11c3056 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs @@ -54,15 +54,27 @@ public partial class LowerConv : RewriteRule int activationType = 3; int dynamicWeight = 0; - var args = new ConvArgs(weights.ToArray(), bias.ToArray(), numOutput.FixedValue, - kernelW.FixedValue, kernelH.FixedValue, - dilationW, dilationH, - strideW, strideH, - padLeft, padRight, padTop, padBottom, padValue, + var args = new ConvArgs( + weights.ToArray(), + bias.ToArray(), + numOutput.FixedValue, + kernelW.FixedValue, + kernelH.FixedValue, + dilationW, + dilationH, + strideW, + strideH, + padLeft, + padRight, + padTop, + padBottom, + padValue, biasTerm, weightsDataSize.FixedValue, int8ScaleTerm, - activationType, fusedClamp, dynamicWeight, + activationType, + fusedClamp, + dynamicWeight, groups); var inRes = Squeeze(input, new[] { 0 }); diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCrop.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCrop.cs index cbd6479dd9..bead6f8148 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCrop.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCrop.cs @@ -58,9 +58,8 @@ public partial class LowerCrop : RewriteRule tStart.RemoveAt(i); tEnds.RemoveAt(i); tAxes.RemoveAt(i); + break; } - - break; } for (int i = 0; i < tAxes.Count; i++) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerMatmul.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerMatmul.cs index ee453aac54..2b413b8fe4 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerMatmul.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerMatmul.cs @@ -54,8 +54,7 @@ public partial class LowerMatmul : RewriteRule var newInputA = new Var(inputA.CheckedType); var newInputB = new Var(inputB.CheckedType); return new Call( - new Fusion("ncnn", NcnnMatMul(new Expr[] { newInputA, newInputB }, 0, null, null), - new[] { newInputA, newInputB }), inputA, inputB); + new Fusion("ncnn", NcnnMatMul(new Expr[] { newInputA, newInputB }, 0, null, null), new[] { newInputA, newInputB }), inputA, inputB); } } } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs index e8cf987d4b..bd0c79d6fa 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs @@ -36,7 +36,8 @@ public partial class LowerSqueeze : RewriteRule { newDims.Add(item); } - if(input.CheckedShape[item] != 1) + + if (input.CheckedShape[item] != 1) { return null; } From 6e8cb7cc9a53268f03543e32d5b8b00ed3a312bd Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 22 Mar 2024 16:41:01 +0800 Subject: [PATCH 095/132] fix binary 4D --- .../CodeGen/Ncnn/NcnnEmitter.cs | 12 +++++++++++- .../Passes/Rules/Ncnn/LowerBinary.cs | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index e3d714f27e..0d7487f04a 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -333,10 +333,20 @@ public void Reduction(string[] name, string input, ReductionArgs reductionArgs) public void Reshape(string[] name, string input, int[] newshape) { var args = new ParamDict(); + List index; + if (newshape.Length < 4) + { + index = Enumerable.Range(0, newshape.Length).ToList(); + } + else + { + index = new List { 0, 1, 11, 2 }; + } + int i = 0; foreach (int item in newshape.Reverse()) { - args.Add(i, new ParamValue { Kind = ParamKind.Int, IntValue = item }); + args.Add(index[i], new ParamValue { Kind = ParamKind.Int, IntValue = item }); i += 1; } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs index 5145eca852..4a78e2baa4 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs @@ -71,6 +71,16 @@ private int[] FixShape(int[] shape, int r) return newShape.ToArray(); } + private bool HasGap(List aShape, List bShape) + { + if ((aShape.Count == 1 && bShape.Count != 1 && aShape[0] != bShape[^1]) || (bShape.Count == 1 && aShape.Count != 1 && bShape[0] != aShape[^1])) + { + return true; + } + + return false; + } + private Expr? GetReplace(Binary binary, Expr inputA, Expr inputB) { if (MapBinaryOp(binary.BinaryOp) is BinaryOperationType op) @@ -81,6 +91,13 @@ private int[] FixShape(int[] shape, int r) return null; } + // ncnn [1, 3, 1, 1], [3] --> [1,3,1,3] + // onnx [1, 3, 1, 1], [3] --> [1,3,1,1] + if (HasGap(inputA.CheckedShape.ToValueArray().ToList(), inputB.CheckedShape.ToValueArray().ToList())) + { + return null; + } + Call b; if (inputA is Const) From 417ef36e63b0953d3a92aa9e402190f99fe4adf2 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 22 Mar 2024 17:35:21 +0800 Subject: [PATCH 096/132] fix reduction --- .../Evaluator/Ncnn/NcnnReduction.cs | 25 ++++++++++++------- .../Passes/Rules/Ncnn/LowerReduction.cs | 3 +-- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReduction.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReduction.cs index f02b078cc9..505a403f35 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReduction.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnReduction.cs @@ -80,18 +80,25 @@ public Metric Visit(IMetricEvaluateContext context, NcnnReduction target) private IRType Visit(NcnnReduction reduction, TensorType input) { - var newInput = new TensorType(input.DType, input.Shape.InsertAndClone(0, 1)); - var axis = reduction.Args.Axes; - - var newAxis = axis.Select(x => x > 0 ? x + 1 : x).ToArray(); + var inputShape = input.Shape.ToList(); + var axis = reduction.Args.Axes.ToList(); + var keepDims = reduction.Args.Keepdims; - var output_ = TypeInference.ReduceType(newInput, reduction.Args.Keepdims, newAxis); - if (output_ is TensorType t) + while (axis.Count > 0) { - var newShape = t.Shape.ToArray(); - return new TensorType(t.DType, newShape[1..]); + var (maxValue, maxIndex) = (axis.Max(), axis.IndexOf(axis.Max())); + if (keepDims == 1) + { + inputShape[(int)maxValue] = 1; + } + else + { + inputShape.RemoveAt((int)maxValue); + } + + axis.RemoveAt(maxIndex); } - return output_; + return new TensorType(input.DType, inputShape.ToArray()); } } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReduction.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReduction.cs index ef09adf357..62bc5259a3 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReduction.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReduction.cs @@ -78,10 +78,9 @@ public virtual int GetOpT() newAxis[i] = newAxis[i] > 0 ? newAxis[i] - 1 : newAxis[i]; } - var args = new ReductionArgs(reductionType, axis.Length > 0 ? 0 : 1, 0, newAxis, keepDims ? 1 : 0); - var inRes = Squeeze(input, new[] { 0 }); var inResO = new Var(inRes.CheckedType); + var args = new ReductionArgs(reductionType, newAxis.Length == inRes.CheckedShape.Rank ? 1 : 0, 0, newAxis, keepDims ? 1 : 0); var pool = new Call(new Fusion("ncnn", NcnnReduction(inResO, args), new[] { inResO }), inRes); return Unsqueeze(pool, new[] { 0 }); From 3fb7fadaf88da6a4dec9efd99a3ef8177d4e8e53 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 25 Mar 2024 09:57:40 +0800 Subject: [PATCH 097/132] fix scaler binary and two input binary --- .../Passes/Rules/Ncnn/LowerBinary.cs | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs index 4a78e2baa4..c2622bccae 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs @@ -85,6 +85,7 @@ private bool HasGap(List aShape, List bShape) { if (MapBinaryOp(binary.BinaryOp) is BinaryOperationType op) { + bool needUnSqueeze = false; int r = Math.Max(inputA.CheckedShape.Rank, inputB.CheckedShape.Rank); if (r > 4) { @@ -93,7 +94,7 @@ private bool HasGap(List aShape, List bShape) // ncnn [1, 3, 1, 1], [3] --> [1,3,1,3] // onnx [1, 3, 1, 1], [3] --> [1,3,1,1] - if (HasGap(inputA.CheckedShape.ToValueArray().ToList(), inputB.CheckedShape.ToValueArray().ToList())) + if (inputA.CheckedShape.Rank != 0 && inputB.CheckedShape.Rank != 0 && HasGap(inputA.CheckedShape.ToValueArray().ToList(), inputB.CheckedShape.ToValueArray().ToList())) { return null; } @@ -120,6 +121,7 @@ private bool HasGap(List aShape, List bShape) newB = Squeeze(newB, new[] { 0 }); newInputB = new Var(newB.CheckedType); + needUnSqueeze = true; } while (aShape[0] == 1 && aShape.Count > 3 && aShape.Count > newB.CheckedShape.Count) @@ -148,6 +150,7 @@ private bool HasGap(List aShape, List bShape) newA = Squeeze(newA, new[] { 0 }); newInputA = new Var(newA.CheckedType); + needUnSqueeze = true; } while (bShape[0] == 1 && bShape.Count > 3 && bShape.Count > newA.CheckedShape.Count) @@ -159,44 +162,47 @@ private bool HasGap(List aShape, List bShape) } else { + var inA = Reshape(inputA, FixShape(inputA.CheckedShape.ToValueArray(), r)); + var inB = Reshape(inputB, FixShape(inputB.CheckedShape.ToValueArray(), r)); + var newInputA = new Var(inA.CheckedType); + var newInputB = new Var(inB.CheckedType); if (r < 4) { - var newInputA = new Var(inputA.CheckedType); - var newInputB = new Var(inputB.CheckedType); - b = new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputA, newInputB }, op, 0, null, null), new[] { newInputA, newInputB }), inputA, inputB); + b = new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputA, newInputB }, op, 0, null, null), new[] { newInputA, newInputB }), inA, inB); } else { - var newA = inputA; - var newB = inputB; - var newInputA = new Var(newA.CheckedType); - var newInputB = new Var(newB.CheckedType); - if (inputA.CheckedShape[0].FixedValue != 1 && inputA.CheckedShape.Rank == r) + var newA = inA; + var newB = inB; + newInputA = new Var(newA.CheckedType); + newInputB = new Var(newB.CheckedType); + if (inA.CheckedShape[0].FixedValue != 1 && inA.CheckedShape.Rank == r) { return null; } - else + else if (inA.CheckedShape[0].FixedValue == 1) { - newA = Squeeze(inputA, new[] { 0 }); + newA = Squeeze(inA, new[] { 0 }); newInputA = new Var(newA.CheckedType); + needUnSqueeze = true; } - if (inputB.CheckedShape[0].FixedValue != 1 && inputB.CheckedShape.Rank == r) + if (inB.CheckedShape[0].FixedValue != 1 && inB.CheckedShape.Rank == r) { - var aa = inputB.CheckedShape.AsEnumerable().Select(x => x.FixedValue).ToArray(); return null; } - else + else if (inB.CheckedShape[0].FixedValue == 1) { - newB = Squeeze(inputB, new[] { 0 }); + newB = Squeeze(inB, new[] { 0 }); newInputB = new Var(newB.CheckedType); + needUnSqueeze = true; } b = new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputA, newInputB }, op, 0, null, null), new[] { newInputA, newInputB }), newA, newB); } } - if (r == 4) + if (needUnSqueeze) { return Unsqueeze(b, new[] { 0 }); } From 0b2e0a734669b0aec6917cb7123e56adcf4932e4 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 25 Mar 2024 14:03:48 +0800 Subject: [PATCH 098/132] add more test for ncnn --- tests/importer/onnx_/basic/test_cast.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/importer/onnx_/basic/test_cast.py b/tests/importer/onnx_/basic/test_cast.py index 3fc91c3303..65feac003f 100644 --- a/tests/importer/onnx_/basic/test_cast.py +++ b/tests/importer/onnx_/basic/test_cast.py @@ -18,7 +18,7 @@ from onnx import helper from onnx import AttributeProto, TensorProto, GraphProto from onnx_test_runner import OnnxTestRunner - +import numpy as np def _make_module(in_shape, in_type, out_type, op_version): attributes_dict = {} @@ -52,6 +52,9 @@ def _make_module(in_shape, in_type, out_type, op_version): ([8, 3, 12, 3], TensorProto.FLOAT, TensorProto.FLOAT16), ([8, 3, 12, 3], TensorProto.FLOAT, TensorProto.UINT8), ([8, 3, 12, 3], TensorProto.FLOAT, TensorProto.INT32), + ([1, 3, 12, 3], TensorProto.FLOAT16, TensorProto.FLOAT), + ([1, 3, 12, 3], TensorProto.FLOAT, TensorProto.FLOAT16), + ] op_versions = [ From e9a98db674b543c837a0018bec3c57bf7bc6f635 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 25 Mar 2024 14:10:52 +0800 Subject: [PATCH 099/132] remove log and fix ncnn output shape --- modules/ncnn/src/runtime/runtime_function.cpp | 38 ++----------------- 1 file changed, 4 insertions(+), 34 deletions(-) diff --git a/modules/ncnn/src/runtime/runtime_function.cpp b/modules/ncnn/src/runtime/runtime_function.cpp index 904c8c382f..d009034f6e 100644 --- a/modules/ncnn/src/runtime/runtime_function.cpp +++ b/modules/ncnn/src/runtime/runtime_function.cpp @@ -23,38 +23,13 @@ size_t rdata_offset = 0; -typedef std::chrono::high_resolution_clock::time_point time_var; - -#define duration(name, a) \ - std::cout \ - << name << " : " \ - << std::chrono::duration_cast(a).count() / \ - 1e6 \ - << " ms" << std::endl -#define timeNow() std::chrono::high_resolution_clock::now() - -template -void print_data(T1 *data, int size, int placeholde = 1, int changeline = 1) -{ - auto s = placeholde ? " " : ""; - for (int i = 0; i < size; i++) { - if(i%10 == 0 && changeline) - { - std::cout< ncnn_runtime_function::initialize_core( NNCASE_UNUSED stream_reader* sr = nullptr; section_header h; try_set(sr, context.seek_section(".rdata", h)); - // rdata_ = sr.template read(); - // std::cout << "start: " << h.body_start << "\t" << "size: " << h.body_size << std::endl; - // print_data(rdata_.data(), rdata_.size() / sizeof(float)); auto param_mem = reinterpret_cast( module().text().data() + context.header().entrypoint); if(context.header().entrypoint == 0) rdata_offset = 0; auto bin_mem = reinterpret_cast(module().rdata().data() + rdata_offset); + // auto bin_mem = reinterpret_cast(rdata_.data()); ::ncnn::DataReaderFromMemory paramdr(param_mem); ::ncnn::DataReaderFromMemory bindr(bin_mem); @@ -102,7 +75,7 @@ result ncnn_runtime_function::initialize_core( CHECK_WITH_ERR(!net_.load_param(paramdr), std::errc::invalid_argument); CHECK_WITH_ERR(!net_.load_model(bindr), std::errc::invalid_argument); - net_.opt.num_threads = 1; + net_.opt.num_threads = omp_get_thread_num(); rdata_offset += h.memory_size; @@ -174,7 +147,6 @@ result ncnn_runtime_function::invoke_core( // 2. Extract outputs std::vector outputs; - time_var t = timeNow(); for (size_t i = 0; i < output_names_.size(); i++) { ::ncnn::Mat mat; CHECK_WITH_ERR(!ex.extract(output_names_[i].c_str(), mat), @@ -195,8 +167,7 @@ result ncnn_runtime_function::invoke_core( shape = {(size_t)mat.c, (size_t)mat.h, (size_t)mat.w}; break; case 4: - shape = {(size_t)mat.c, (size_t)mat.d, (size_t)mat.h, (size_t)mat.w, - 1}; + shape = {(size_t)mat.c, (size_t)mat.d, (size_t)mat.h, (size_t)mat.w}; break; default: return err(std::errc::invalid_argument); @@ -236,7 +207,6 @@ result ncnn_runtime_function::invoke_core( tensor t(std::in_place, dt, shape, get_default_strides(shape), buf); outputs.emplace_back(t); } - duration("ncnn run", timeNow() - t); auto ret_val = output_names_.size() == 1 ? outputs[0] : tuple(std::in_place, std::move(outputs)); From 582508e1d9d24a1da05995fe50afdbb41900c3dd Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 25 Mar 2024 14:50:14 +0800 Subject: [PATCH 100/132] fix get max thread for ncnn::opt.num_thread --- CMakeLists.txt | 4 ++++ modules/ncnn/src/runtime/runtime_function.cpp | 9 +++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7570b7e1b4..fc0f5da33f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,6 +59,10 @@ if(ENABLE_DUMP_MEM) add_definitions(-DDUMP_MEM) endif() +if(ENABLE_OPENMP) + add_definitions(-DENABLE_OPENMP) +endif() + if(NOT CMAKE_TOOLCHAIN_FILE) include(toolchains/native.toolchain.cmake) endif() diff --git a/modules/ncnn/src/runtime/runtime_function.cpp b/modules/ncnn/src/runtime/runtime_function.cpp index d009034f6e..a38fabc16e 100644 --- a/modules/ncnn/src/runtime/runtime_function.cpp +++ b/modules/ncnn/src/runtime/runtime_function.cpp @@ -19,7 +19,13 @@ #include #include #include + +#ifdef ENABLE_OPENMP #include +#define OMP_MAX_THREAD omp_get_max_threads() +#else +#define OMP_MAX_THREAD 1 +#endif // ENABLE_OPENMP size_t rdata_offset = 0; @@ -75,8 +81,7 @@ result ncnn_runtime_function::initialize_core( CHECK_WITH_ERR(!net_.load_param(paramdr), std::errc::invalid_argument); CHECK_WITH_ERR(!net_.load_model(bindr), std::errc::invalid_argument); - net_.opt.num_threads = omp_get_thread_num(); - + net_.opt.num_threads = OMP_MAX_THREAD; rdata_offset += h.memory_size; From 68758404ba32c2f1221339c7cfafef071513ce43 Mon Sep 17 00:00:00 2001 From: curioyang Date: Mon, 25 Mar 2024 08:33:02 +0000 Subject: [PATCH 101/132] Apply code-format changes --- modules/Nncase.Modules.Ncnn/packages.lock.json | 9 ++++++++- modules/ncnn/src/runtime/runtime_function.cpp | 10 +++++----- tests/importer/onnx_/basic/test_binary.py | 6 +++--- tests/importer/onnx_/basic/test_cast.py | 1 + tests/importer/onnx_/basic/test_gelu.py | 4 ++-- tests/importer/onnx_/basic/test_layernorm.py | 8 ++++---- tests/importer/onnx_/basic/test_slice.py | 2 +- tests/importer/onnx_/basic/test_split.py | 4 ++-- 8 files changed, 26 insertions(+), 18 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/packages.lock.json b/modules/Nncase.Modules.Ncnn/packages.lock.json index 4830d0bbf6..8820c05237 100644 --- a/modules/Nncase.Modules.Ncnn/packages.lock.json +++ b/modules/Nncase.Modules.Ncnn/packages.lock.json @@ -121,7 +121,8 @@ "dependencies": { "Extension.Mathematics": "[1.2.12, )", "Nncase.Core": "[1.0.0, )", - "Nncase.IO": "[1.0.0, )" + "Nncase.IO": "[1.0.0, )", + "Razor.Templating.Core": "[1.9.0, )" } }, "nncase.core": { @@ -266,6 +267,12 @@ "libortki": "0.0.2" } }, + "Razor.Templating.Core": { + "type": "CentralTransitive", + "requested": "[1.9.0, )", + "resolved": "1.9.0", + "contentHash": "eHNqkpmNcPr5rvP/8/FFkddnvzVMH0BSyrq03H0VLZK2r1GUe3RgIgsoIXnImHMIrBzUS8gOwV65MfRPdYRi6g==" + }, "Singulink.Collections.Weak": { "type": "CentralTransitive", "requested": "[1.0.2, )", diff --git a/modules/ncnn/src/runtime/runtime_function.cpp b/modules/ncnn/src/runtime/runtime_function.cpp index a38fabc16e..60eb323d48 100644 --- a/modules/ncnn/src/runtime/runtime_function.cpp +++ b/modules/ncnn/src/runtime/runtime_function.cpp @@ -35,7 +35,7 @@ using namespace nncase::runtime::ncnn; namespace { class DataReaderFromEmpty : public ::ncnn::DataReader { -public: + public: virtual int scan(const char *format, void *p) const { return 0; } virtual size_t read(void *buf, size_t size) const { memset(buf, 0, size); @@ -64,12 +64,12 @@ result ncnn_runtime_function::initialize_core( return ok(); })); - NNCASE_UNUSED stream_reader* sr = nullptr; + NNCASE_UNUSED stream_reader *sr = nullptr; section_header h; try_set(sr, context.seek_section(".rdata", h)); auto param_mem = reinterpret_cast( module().text().data() + context.header().entrypoint); - if(context.header().entrypoint == 0) + if (context.header().entrypoint == 0) rdata_offset = 0; auto bin_mem = reinterpret_cast(module().rdata().data() + rdata_offset); @@ -84,7 +84,6 @@ result ncnn_runtime_function::initialize_core( net_.opt.num_threads = OMP_MAX_THREAD; rdata_offset += h.memory_size; - return ok(); } @@ -172,7 +171,8 @@ result ncnn_runtime_function::invoke_core( shape = {(size_t)mat.c, (size_t)mat.h, (size_t)mat.w}; break; case 4: - shape = {(size_t)mat.c, (size_t)mat.d, (size_t)mat.h, (size_t)mat.w}; + shape = {(size_t)mat.c, (size_t)mat.d, (size_t)mat.h, + (size_t)mat.w}; break; default: return err(std::errc::invalid_argument); diff --git a/tests/importer/onnx_/basic/test_binary.py b/tests/importer/onnx_/basic/test_binary.py index b2d9368674..95c2d93a1e 100644 --- a/tests/importer/onnx_/basic/test_binary.py +++ b/tests/importer/onnx_/basic/test_binary.py @@ -23,10 +23,10 @@ def _make_module(v_shape): class BinaryModule(torch.nn.Module): def __init__(self): super(BinaryModule, self).__init__() - self.v = torch.from_numpy(np.random.random(v_shape).astype(np.float32)+3) + self.v = torch.from_numpy(np.random.random(v_shape).astype(np.float32) + 3) self.v2 = torch.from_numpy(np.random.random(v_shape).astype(np.float32)) - self.v3 = torch.from_numpy(np.random.random(v_shape).astype(np.float32)+1) - self.v4 = torch.from_numpy(np.random.random(v_shape).astype(np.float32)+2) + self.v3 = torch.from_numpy(np.random.random(v_shape).astype(np.float32) + 1) + self.v4 = torch.from_numpy(np.random.random(v_shape).astype(np.float32) + 2) def forward(self, x): outs = [] diff --git a/tests/importer/onnx_/basic/test_cast.py b/tests/importer/onnx_/basic/test_cast.py index 65feac003f..5d15111823 100644 --- a/tests/importer/onnx_/basic/test_cast.py +++ b/tests/importer/onnx_/basic/test_cast.py @@ -20,6 +20,7 @@ from onnx_test_runner import OnnxTestRunner import numpy as np + def _make_module(in_shape, in_type, out_type, op_version): attributes_dict = {} diff --git a/tests/importer/onnx_/basic/test_gelu.py b/tests/importer/onnx_/basic/test_gelu.py index 66a51ec3ec..54003d7347 100644 --- a/tests/importer/onnx_/basic/test_gelu.py +++ b/tests/importer/onnx_/basic/test_gelu.py @@ -23,12 +23,12 @@ def _make_module(v_shape): class GELUModule(torch.nn.Module): def __init__(self,): super(GELUModule, self).__init__() - self.gelu = nn.GELU(); + self.gelu = nn.GELU() + def forward(self, x): x = self.gelu(x) return x - return GELUModule() diff --git a/tests/importer/onnx_/basic/test_layernorm.py b/tests/importer/onnx_/basic/test_layernorm.py index 96baa77593..e540017cb9 100644 --- a/tests/importer/onnx_/basic/test_layernorm.py +++ b/tests/importer/onnx_/basic/test_layernorm.py @@ -20,7 +20,7 @@ from onnx_test_runner import OnnxTestRunner -def _make_module(v_shape,axis): +def _make_module(v_shape, axis): class LayerNormModule(torch.nn.Module): def __init__(self, channel_size=3): super(LayerNormModule, self).__init__() @@ -29,14 +29,14 @@ def __init__(self, channel_size=3): self.bias = torch.from_numpy(np.random.rand(*reduce_dim).astype(np.float32)) # torch.layernorm init scale and bias with [1,0] because they are learnable. When the model exports to onnx, scale and bias were eliminated. # So, random data are used to pretend parameters. - self.layernorm = nn.LayerNorm(normalized_shape=reduce_dim, elementwise_affine=True,eps=1e-03) + self.layernorm = nn.LayerNorm(normalized_shape=reduce_dim, + elementwise_affine=True, eps=1e-03) def forward(self, x): x = self.layernorm(x) - x = x *self.scale + self.bias + x = x * self.scale + self.bias return x - return LayerNormModule() diff --git a/tests/importer/onnx_/basic/test_slice.py b/tests/importer/onnx_/basic/test_slice.py index 9b1ddc883a..65f345c051 100644 --- a/tests/importer/onnx_/basic/test_slice.py +++ b/tests/importer/onnx_/basic/test_slice.py @@ -165,7 +165,7 @@ def _make_module(in_shape, start, end, axes, step, outshape, op_version, value_f [[1, 20, 10, 5], [1], [1000], [2], [1], [1, 20, 9, 5]], [[1, 20, 10, 5], [0], [-1], [1], [1], [1, 19, 10, 5]], [[1, 20, 10, 5], [0, 0, 3], [20, 10, 4], [-3, -2, -1], None, [1, 20, 10, 1]], - [[1, 20, 10, 5], [ 20, 10, 4], [ 0, 0, 1], [1, 2, 3], [-1, -3, -2], [1, 20, 4, 2]], + [[1, 20, 10, 5], [20, 10, 4], [0, 0, 1], [1, 2, 3], [-1, -3, -2], [1, 20, 4, 2]], [[1, 20, 10, 5], [-1], [-9223372036854775807], [1], [-1], [1, 20, 10, 5]] ] diff --git a/tests/importer/onnx_/basic/test_split.py b/tests/importer/onnx_/basic/test_split.py index 9809619e1d..abe8e44442 100644 --- a/tests/importer/onnx_/basic/test_split.py +++ b/tests/importer/onnx_/basic/test_split.py @@ -115,7 +115,7 @@ def _make_module(in_shape, axis, split, output_size, op_version, value_format): [1, 1, 2], [1, 2, 1], [3, 2, 2, 1], - [2, 1, 1, 1,3], + [2, 1, 1, 1, 3], ] output_sizes = [ @@ -143,7 +143,7 @@ def _make_module(in_shape, axis, split, output_size, op_version, value_format): def test_split(in_shape, axis, split, output_size, op_versions_and_value_format, request): op_version, value_format = op_versions_and_value_format dim = axis if axis is not None else 0 - if (split is None and in_shape[dim] % output_size == 0 ) or (split is not None and len(split) == output_size and sum(split) == in_shape[dim]): + if (split is None and in_shape[dim] % output_size == 0) or (split is not None and len(split) == output_size and sum(split) == in_shape[dim]): model_def = _make_module(in_shape, axis, split, output_size, op_version, value_format) runner = OnnxTestRunner(request.node.name) From 6c5d0b536d13174973815744879388a7b6b720c3 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 25 Mar 2024 16:37:06 +0800 Subject: [PATCH 102/132] enable riscv64-unknown-linux-gnu-13.2.0 in Conan setting --- .github/workflows/runtime-build.yml | 1 + modules/Nncase.Modules.Ncnn/packages.lock.json | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/runtime-build.yml b/.github/workflows/runtime-build.yml index 40701adaf9..15dd0a3165 100644 --- a/.github/workflows/runtime-build.yml +++ b/.github/workflows/runtime-build.yml @@ -126,6 +126,7 @@ jobs: conan profile update settings.compiler.libcxx=libstdc++11 default conan config init sed -i 's/xtensalx7]/xtensalx7, ${{matrix.config.arch}}]/g' ~/.conan/settings.yml + sed -i 's/'12.1', '12.2']/'12.1', '12.2', '13.2']/g' ~/.conan/settings.yml - name: Configure CMake run: | diff --git a/modules/Nncase.Modules.Ncnn/packages.lock.json b/modules/Nncase.Modules.Ncnn/packages.lock.json index 4830d0bbf6..8820c05237 100644 --- a/modules/Nncase.Modules.Ncnn/packages.lock.json +++ b/modules/Nncase.Modules.Ncnn/packages.lock.json @@ -121,7 +121,8 @@ "dependencies": { "Extension.Mathematics": "[1.2.12, )", "Nncase.Core": "[1.0.0, )", - "Nncase.IO": "[1.0.0, )" + "Nncase.IO": "[1.0.0, )", + "Razor.Templating.Core": "[1.9.0, )" } }, "nncase.core": { @@ -266,6 +267,12 @@ "libortki": "0.0.2" } }, + "Razor.Templating.Core": { + "type": "CentralTransitive", + "requested": "[1.9.0, )", + "resolved": "1.9.0", + "contentHash": "eHNqkpmNcPr5rvP/8/FFkddnvzVMH0BSyrq03H0VLZK2r1GUe3RgIgsoIXnImHMIrBzUS8gOwV65MfRPdYRi6g==" + }, "Singulink.Collections.Weak": { "type": "CentralTransitive", "requested": "[1.0.2, )", From 0382455a9c5d948a2fbfc0a34809dfa732e63691 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 25 Mar 2024 16:44:36 +0800 Subject: [PATCH 103/132] remove unused code --- tests/importer/onnx_/basic/test_binary.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/importer/onnx_/basic/test_binary.py b/tests/importer/onnx_/basic/test_binary.py index 95c2d93a1e..16dca0f08b 100644 --- a/tests/importer/onnx_/basic/test_binary.py +++ b/tests/importer/onnx_/basic/test_binary.py @@ -23,10 +23,7 @@ def _make_module(v_shape): class BinaryModule(torch.nn.Module): def __init__(self): super(BinaryModule, self).__init__() - self.v = torch.from_numpy(np.random.random(v_shape).astype(np.float32) + 3) - self.v2 = torch.from_numpy(np.random.random(v_shape).astype(np.float32)) - self.v3 = torch.from_numpy(np.random.random(v_shape).astype(np.float32) + 1) - self.v4 = torch.from_numpy(np.random.random(v_shape).astype(np.float32) + 2) + self.v = torch.from_numpy(np.random.random(v_shape).astype(np.float32)) def forward(self, x): outs = [] From 3a316ac593aa75f1932f3fcce03866d90b4e06e6 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 25 Mar 2024 17:06:44 +0800 Subject: [PATCH 104/132] add more vision in conan setting --- .github/workflows/runtime-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runtime-build.yml b/.github/workflows/runtime-build.yml index 15dd0a3165..ff81588dfc 100644 --- a/.github/workflows/runtime-build.yml +++ b/.github/workflows/runtime-build.yml @@ -126,7 +126,7 @@ jobs: conan profile update settings.compiler.libcxx=libstdc++11 default conan config init sed -i 's/xtensalx7]/xtensalx7, ${{matrix.config.arch}}]/g' ~/.conan/settings.yml - sed -i 's/'12.1', '12.2']/'12.1', '12.2', '13.2']/g' ~/.conan/settings.yml + sed -i 's/'12.1', '12.2']/'12.1', '12.2', '13', '13.1', '13.2']/g' ~/.conan/settings.yml - name: Configure CMake run: | From 006e3e4b3908aae5786e823cb5505b2446bd6cbc Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 25 Mar 2024 17:10:14 +0800 Subject: [PATCH 105/132] add more vision in conan setting --- .github/workflows/runtime-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/runtime-build.yml b/.github/workflows/runtime-build.yml index ff81588dfc..af8534838c 100644 --- a/.github/workflows/runtime-build.yml +++ b/.github/workflows/runtime-build.yml @@ -49,6 +49,7 @@ jobs: conan profile update settings.compiler.version=10 default echo "CC=gcc-10" >> $GITHUB_ENV echo "CXX=g++-10" >> $GITHUB_ENV + sed -i 's/'12.1', '12.2']/'12.1', '12.2', '13', '13.1', '13.2']/g' ~/.conan/settings.yml if: runner.os == 'Linux' - name: Configure CMake From 9481ca4cb424cafc6119977affd5175ef01d06de Mon Sep 17 00:00:00 2001 From: Curio Yang <39184746+curioyang@users.noreply.github.com> Date: Mon, 25 Mar 2024 17:23:45 +0800 Subject: [PATCH 106/132] Update runtime-build.yml --- .github/workflows/runtime-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/runtime-build.yml b/.github/workflows/runtime-build.yml index af8534838c..ee7cdb17bd 100644 --- a/.github/workflows/runtime-build.yml +++ b/.github/workflows/runtime-build.yml @@ -49,7 +49,7 @@ jobs: conan profile update settings.compiler.version=10 default echo "CC=gcc-10" >> $GITHUB_ENV echo "CXX=g++-10" >> $GITHUB_ENV - sed -i 's/'12.1', '12.2']/'12.1', '12.2', '13', '13.1', '13.2']/g' ~/.conan/settings.yml + sed -i 's/"12.1", "12.2"]/"12.1", "12.2", "13", "13.1", "13.2"]/g' ~/.conan/settings.yml if: runner.os == 'Linux' - name: Configure CMake @@ -127,7 +127,7 @@ jobs: conan profile update settings.compiler.libcxx=libstdc++11 default conan config init sed -i 's/xtensalx7]/xtensalx7, ${{matrix.config.arch}}]/g' ~/.conan/settings.yml - sed -i 's/'12.1', '12.2']/'12.1', '12.2', '13', '13.1', '13.2']/g' ~/.conan/settings.yml + sed -i 's/"12.1", "12.2"]/"12.1", "12.2", "13", "13.1", "13.2"]/g' ~/.conan/settings.yml - name: Configure CMake run: | From d22db45ae2327a1302c47fd17419bf4c93e72b0a Mon Sep 17 00:00:00 2001 From: Curio Yang <39184746+curioyang@users.noreply.github.com> Date: Mon, 25 Mar 2024 17:27:59 +0800 Subject: [PATCH 107/132] Update runtime-build.yml --- .github/workflows/runtime-build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/runtime-build.yml b/.github/workflows/runtime-build.yml index ee7cdb17bd..05ad36ca5e 100644 --- a/.github/workflows/runtime-build.yml +++ b/.github/workflows/runtime-build.yml @@ -49,7 +49,6 @@ jobs: conan profile update settings.compiler.version=10 default echo "CC=gcc-10" >> $GITHUB_ENV echo "CXX=g++-10" >> $GITHUB_ENV - sed -i 's/"12.1", "12.2"]/"12.1", "12.2", "13", "13.1", "13.2"]/g' ~/.conan/settings.yml if: runner.os == 'Linux' - name: Configure CMake From 5f062e56cf03a5d61ac565ebb030a26b5b97bffd Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Tue, 26 Mar 2024 10:31:24 +0800 Subject: [PATCH 108/132] update layernorm test --- tests/importer/onnx_/basic/test_layernorm.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/importer/onnx_/basic/test_layernorm.py b/tests/importer/onnx_/basic/test_layernorm.py index e540017cb9..4026480508 100644 --- a/tests/importer/onnx_/basic/test_layernorm.py +++ b/tests/importer/onnx_/basic/test_layernorm.py @@ -22,20 +22,24 @@ def _make_module(v_shape, axis): class LayerNormModule(torch.nn.Module): - def __init__(self, channel_size=3): + def __init__(self) -> None: super(LayerNormModule, self).__init__() reduce_dim = [v_shape[i] for i in range(len(v_shape)) if i > axis] self.scale = torch.from_numpy(np.random.rand(*reduce_dim).astype(np.float32)) self.bias = torch.from_numpy(np.random.rand(*reduce_dim).astype(np.float32)) - # torch.layernorm init scale and bias with [1,0] because they are learnable. When the model exports to onnx, scale and bias were eliminated. - # So, random data are used to pretend parameters. - self.layernorm = nn.LayerNorm(normalized_shape=reduce_dim, - elementwise_affine=True, eps=1e-03) + self.axis = [ i for i in range(len(v_shape)) if i > axis] def forward(self, x): - x = self.layernorm(x) - x = x * self.scale + self.bias - return x + reduce_mean = torch.mean(x, self.axis, keepdim=True) + x_sub = x - reduce_mean + x_pow = torch.pow(x_sub, 2) + x_pow_mean = torch.mean(x_pow, self.axis, keepdim=True) + x_add = x_pow_mean + 1e-06 + x_sqrt = torch.sqrt(x_add) + x_div = x_sub / x_sqrt + x_mul = x_div * self.scale + x_add_bias = x_mul + self.bias + return x_add_bias return LayerNormModule() From 561d65b13daab13675d896e24414d1d8788e71c5 Mon Sep 17 00:00:00 2001 From: curioyang Date: Tue, 26 Mar 2024 02:34:15 +0000 Subject: [PATCH 109/132] Apply code-format changes --- tests/importer/onnx_/basic/test_layernorm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/importer/onnx_/basic/test_layernorm.py b/tests/importer/onnx_/basic/test_layernorm.py index 4026480508..d07555f724 100644 --- a/tests/importer/onnx_/basic/test_layernorm.py +++ b/tests/importer/onnx_/basic/test_layernorm.py @@ -27,7 +27,7 @@ def __init__(self) -> None: reduce_dim = [v_shape[i] for i in range(len(v_shape)) if i > axis] self.scale = torch.from_numpy(np.random.rand(*reduce_dim).astype(np.float32)) self.bias = torch.from_numpy(np.random.rand(*reduce_dim).astype(np.float32)) - self.axis = [ i for i in range(len(v_shape)) if i > axis] + self.axis = [i for i in range(len(v_shape)) if i > axis] def forward(self, x): reduce_mean = torch.mean(x, self.axis, keepdim=True) From c92d0231a5f01f21ede28a538575070771079310 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Tue, 26 Mar 2024 16:32:56 +0800 Subject: [PATCH 110/132] fix --- modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs | 6 ++++-- .../CodeGen/Ncnn/NcnnEmitter.cs | 16 ++++++++++------ .../Evaluator/Ncnn/NcnnConv.cs | 4 ++-- modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs | 2 +- .../Passes/Rules/Ncnn/LowerConv2d.cs | 5 +++++ .../Passes/Rules/Ncnn/LowerConvTranspose.cs | 7 ++++++- 6 files changed, 28 insertions(+), 12 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs index 9826158fcf..9c55e05146 100644 --- a/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs +++ b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs @@ -156,7 +156,7 @@ public CropArgs(int woffset, int hoffset, int doffset, int coffset, int outw, in public record ConvTransposeArgs { - public ConvTransposeArgs(Tensor? weightData = null, float[]? biasData = null, int numOutput = default, int kernelW = default, int kernelH = default, int dilationW = default, int dilationH = default, int strideW = default, int strideH = default, int padLeft = default, int padRight = default, int padTop = default, int padBottom = default, int biasTerm = default, int weightDataSize = default, int activationType = default, float[]? activationParams = null, int outputPadRight = default, int outputPadBottom = default, int outputW = default, int outputH = default) + public ConvTransposeArgs(float[]? weightData = null, float[]? biasData = null, int numOutput = default, int kernelW = default, int kernelH = default, int dilationW = default, int dilationH = default, int strideW = default, int strideH = default, int padLeft = default, int padRight = default, int padTop = default, int padBottom = default, int biasTerm = default, int weightDataSize = default, int activationType = default, float[]? activationParams = null, int outputPadRight = default, int outputPadBottom = default, int outputW = default, int outputH = default) { WeightData = weightData!; BiasData = biasData!; @@ -181,7 +181,7 @@ public ConvTransposeArgs(Tensor? weightData = null, float[]? biasData = null, in OutputH = outputH; } - public Tensor WeightData { get; } + public float[] WeightData { get; } public float[] BiasData { get; } @@ -357,4 +357,6 @@ public ConvArgs(float[] weightData, float[] biasData, int numOutput, int kernelW /// Gets Groups. /// public int Groups { get; } + + public override string ToString() => $"{nameof(WeightData)}: {WeightData}, {nameof(BiasData)}: {BiasData}, {nameof(NumOutput)}: {NumOutput}, {nameof(KernelW)}: {KernelW}, {nameof(KernelH)}: {KernelH}, {nameof(DilationW)}: {DilationW}, {nameof(DilationH)}: {DilationH}, {nameof(StrideW)}: {StrideW}, {nameof(StrideH)}: {StrideH}, {nameof(PadLeft)}: {PadLeft}, {nameof(PadRight)}: {PadRight}, {nameof(PadTop)}: {PadTop}, {nameof(PadBottom)}: {PadBottom}, {nameof(PadValue)}: {PadValue}, {nameof(BiasTerm)}: {BiasTerm}, {nameof(WeightDataSize)}: {WeightDataSize}, {nameof(Int8ScaleTerm)}: {Int8ScaleTerm}, {nameof(ActivationType)}: {ActivationType}, {nameof(ActivationParams)}: {ActivationParams}, {nameof(DynamicWeight)}: {DynamicWeight}, {nameof(Groups)}: {Groups}"; } diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 0d7487f04a..6e253f618f 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -466,6 +466,9 @@ public void Matmul(string[] name, string inputA, string inputB, int lOrR, float[ public void ConvTranspose(string[] name, string input, ConvTransposeArgs args) { + var actData = new List { args.ActivationParams.Length }; + actData.AddRange(args.ActivationParams); + AddLayer("Deconvolution", name[0], new[] { input }, name, new ParamDict { [0] = new ParamValue { Kind = ParamKind.Int, IntValue = args.NumOutput }, @@ -480,18 +483,19 @@ public void ConvTranspose(string[] name, string input, ConvTransposeArgs args) [15] = new ParamValue { Kind = ParamKind.Int, IntValue = args.PadRight }, [16] = new ParamValue { Kind = ParamKind.Int, IntValue = args.PadBottom }, - // [18] = new ParamValue { Kind = ParamKind.Int, IntValue = args.OutputPadRight }, - // [19] = new ParamValue { Kind = ParamKind.Int, IntValue = args.OutputPadBottom }, - // [20] = new ParamValue { Kind = ParamKind.Int, IntValue = args.OutputW }, - // [21] = new ParamValue { Kind = ParamKind.Int, IntValue = args.OutputH }, [5] = new ParamValue { Kind = ParamKind.Int, IntValue = args.BiasTerm }, [6] = new ParamValue { Kind = ParamKind.Int, IntValue = args.WeightDataSize }, // [9] = new ParamValue { Kind = ParamKind.Int, IntValue = args.ActivationType }, - // [10] = new ParamValue { Kind = ParamKind.ArrayOfFloat, TensorValue = args.ActivationParams }, + // [-10] = new ParamValue { Kind = ParamKind.ArrayOfIntOrFloat, TensorValue = actData.ToArray() }, + // + // [18] = new ParamValue { Kind = ParamKind.Int, IntValue = args.OutputPadRight }, + // [19] = new ParamValue { Kind = ParamKind.Int, IntValue = args.OutputPadBottom }, + // [20] = new ParamValue { Kind = ParamKind.Int, IntValue = args.OutputW }, + // [21] = new ParamValue { Kind = ParamKind.Int, IntValue = args.OutputH }, }); WriteFloatArray(new float[] { 0 }); // quantize flag [Not exist in ncnn op.md] - WriteFloatArray(args.WeightData.ToArray()); + WriteFloatArray(args.WeightData); WriteFloatArray(args.BiasData); } diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConv.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConv.cs index 0cdd7af752..362fdfc601 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConv.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnConv.cs @@ -72,13 +72,13 @@ private IRType Visit(TensorType input, NcnnConv conv) outputShape[0] = conv.Args.NumOutput; outputShape[1] = IR.TypePatternUtility.GetWindowedOutputSize( - input.Shape[1].FixedValue + conv.Args.PadLeft + conv.Args.PadRight, + input.Shape[1].FixedValue + conv.Args.PadTop + conv.Args.PadBottom, kernelShape[2], conv.Args.StrideH, conv.Args.DilationH, false); outputShape[2] = IR.TypePatternUtility.GetWindowedOutputSize( - input.Shape[2].FixedValue + conv.Args.PadTop + conv.Args.PadBottom, + input.Shape[2].FixedValue + conv.Args.PadLeft + conv.Args.PadRight, kernelShape[3], conv.Args.StrideW, conv.Args.DilationW, diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs index c1b317aac3..f8faf9351e 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnConv.cs @@ -31,6 +31,6 @@ public sealed partial class NcnnConv : Op /// public override string DisplayProperty() { - return $""; + return Args.ToString(); } } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs index 38e11c3056..8c13e9e979 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs @@ -39,6 +39,11 @@ public partial class LowerConv : RewriteRule private Expr? GetReplace(Expr input, Tensor weights, Tensor bias, int[] strides, int[] paddings, int[] dilation, int groups, float[] fusedClamp) { + if (input.CheckedShape.Rank != 4 || input.CheckedShape[0].FixedValue != 1) + { + return null; + } + var (numOutput, kernelH, kernelW) = (weights.Shape[0], weights.Shape[2], weights.Shape[3]); var (dilationH, dilationW) = (dilation[0], dilation.Length == 2 ? dilation[1] : dilation[0]); var (strideH, strideW) = (strides[0], strides.Length == 2 ? strides[1] : strides[0]); diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConvTranspose.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConvTranspose.cs index ec8da3262e..cb5b2a3a09 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConvTranspose.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConvTranspose.cs @@ -42,6 +42,11 @@ public partial class LowerConvTranspose : RewriteRule private Expr? GetReplace(Expr input, Tensor weights, Tensor bias, int[] outputShape, int[] strides, int[] paddings, int[] outputPadding, int[] dilation, int group, float[] fusedClamp) { + if (input.CheckedShape.Rank != 4 || input.CheckedShape[0].FixedValue != 1) + { + return null; + } + int[] weightShape = weights.Shape.ToValueArray(); var (numOutput, kernelH, kernelW) = (weights.Shape[0], weights.Shape[2], weights.Shape[3]); var (dilationH, dilationW) = (dilation[0], dilation.Length == 2 ? dilation[1] : dilation[0]); @@ -59,7 +64,7 @@ public partial class LowerConvTranspose : RewriteRule // var newWeights = Transpose(weights, new int[] { 1, 0, 2, 3 }).Evaluate().AsTensor(); // actType and actParams not used in ncnn: onnx2ncnn. - var args = new ConvTransposeArgs(weights, bias.ToArray(), numOutput.FixedValue, kernelW.FixedValue, kernelH.FixedValue, dilationW, dilationH, strideW, strideH, padLeft, padRight, padTop, padBottom, biasTerm, weightsDataSize, activationType, fusedClamp, outputPadding[1], outputPadding[0], outputShape[3], outputShape[2]); + var args = new ConvTransposeArgs(weights.ToArray(), bias.ToArray(), numOutput.FixedValue, kernelW.FixedValue, kernelH.FixedValue, dilationW, dilationH, strideW, strideH, padLeft, padRight, padTop, padBottom, biasTerm, weightsDataSize, activationType, fusedClamp, outputPadding[1], outputPadding[0], outputShape[3], outputShape[2]); var inRes = Squeeze(input, new[] { 0 }); var inResO = new Var(inRes.CheckedType); From 1d3d1cfcc86d809169b56eb9681d89f3a630c56b Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Tue, 26 Mar 2024 16:36:20 +0800 Subject: [PATCH 111/132] fix --- modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs | 4 ++-- modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs | 2 +- .../Passes/Rules/Ncnn/LowerConvTranspose.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs index 9c55e05146..0bbbaf459d 100644 --- a/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs +++ b/modules/Nncase.Modules.Ncnn/ArgsStruct/args.cs @@ -156,7 +156,7 @@ public CropArgs(int woffset, int hoffset, int doffset, int coffset, int outw, in public record ConvTransposeArgs { - public ConvTransposeArgs(float[]? weightData = null, float[]? biasData = null, int numOutput = default, int kernelW = default, int kernelH = default, int dilationW = default, int dilationH = default, int strideW = default, int strideH = default, int padLeft = default, int padRight = default, int padTop = default, int padBottom = default, int biasTerm = default, int weightDataSize = default, int activationType = default, float[]? activationParams = null, int outputPadRight = default, int outputPadBottom = default, int outputW = default, int outputH = default) + public ConvTransposeArgs(Tensor? weightData = null, float[]? biasData = null, int numOutput = default, int kernelW = default, int kernelH = default, int dilationW = default, int dilationH = default, int strideW = default, int strideH = default, int padLeft = default, int padRight = default, int padTop = default, int padBottom = default, int biasTerm = default, int weightDataSize = default, int activationType = default, float[]? activationParams = null, int outputPadRight = default, int outputPadBottom = default, int outputW = default, int outputH = default) { WeightData = weightData!; BiasData = biasData!; @@ -181,7 +181,7 @@ public ConvTransposeArgs(float[]? weightData = null, float[]? biasData = null, i OutputH = outputH; } - public float[] WeightData { get; } + public Tensor WeightData { get; } public float[] BiasData { get; } diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 6e253f618f..fb27f8d978 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -495,7 +495,7 @@ public void ConvTranspose(string[] name, string input, ConvTransposeArgs args) // [21] = new ParamValue { Kind = ParamKind.Int, IntValue = args.OutputH }, }); WriteFloatArray(new float[] { 0 }); // quantize flag [Not exist in ncnn op.md] - WriteFloatArray(args.WeightData); + WriteFloatArray(args.WeightData.ToArray()); WriteFloatArray(args.BiasData); } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConvTranspose.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConvTranspose.cs index cb5b2a3a09..0761161ef6 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConvTranspose.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConvTranspose.cs @@ -64,7 +64,7 @@ public partial class LowerConvTranspose : RewriteRule // var newWeights = Transpose(weights, new int[] { 1, 0, 2, 3 }).Evaluate().AsTensor(); // actType and actParams not used in ncnn: onnx2ncnn. - var args = new ConvTransposeArgs(weights.ToArray(), bias.ToArray(), numOutput.FixedValue, kernelW.FixedValue, kernelH.FixedValue, dilationW, dilationH, strideW, strideH, padLeft, padRight, padTop, padBottom, biasTerm, weightsDataSize, activationType, fusedClamp, outputPadding[1], outputPadding[0], outputShape[3], outputShape[2]); + var args = new ConvTransposeArgs(weights, bias.ToArray(), numOutput.FixedValue, kernelW.FixedValue, kernelH.FixedValue, dilationW, dilationH, strideW, strideH, padLeft, padRight, padTop, padBottom, biasTerm, weightsDataSize, activationType, fusedClamp, outputPadding[1], outputPadding[0], outputShape[3], outputShape[2]); var inRes = Squeeze(input, new[] { 0 }); var inResO = new Var(inRes.CheckedType); From 41b7299f27034337dd3e93c646e35f1e79d547a9 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 28 Mar 2024 13:08:48 +0800 Subject: [PATCH 112/132] fix --- src/Nncase.Core/Runtime/Ncnn/ParamDict.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs b/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs index 10c06d3089..8088eeae21 100644 --- a/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs +++ b/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs @@ -133,7 +133,6 @@ public void Serialize(TextWriter writer) foreach (var field in _values) { var id = field.Key; - var paramValue = field.Value; var isArray = paramValue.Kind is ParamKind.ArrayOfFloat or ParamKind.ArrayOfIntOrFloat or ParamKind.ArrayOfInt; var isFloat = paramValue.Kind is ParamKind.ArrayOfFloat or ParamKind.ArrayOfIntOrFloat or ParamKind.Float; @@ -190,6 +189,8 @@ _ when float.IsNegativeInfinity(x) => "-3.402823e+38", { writer.Write(' '); } + + writer.Write('\n'); } } } From a150bb46b9f1a2026dfc950141c679e0f28af840 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Thu, 28 Mar 2024 14:19:33 +0800 Subject: [PATCH 113/132] update tset ncnn version --- requirements.test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.test.txt b/requirements.test.txt index 2e5eabae04..3968b128c1 100644 --- a/requirements.test.txt +++ b/requirements.test.txt @@ -6,7 +6,7 @@ onnx==1.12.0 onnx-simplifier==0.3.6 onnxoptimizer==0.2.6 onnxruntime==1.12.0 -ncnn==1.0.20230816 +ncnn==1.0.20240102 numpy==1.21.0 torch==1.9.0 torchvision==0.10.0 From c63ed29984ca28e87fa29fa12df57f91c44da7ae Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 29 Mar 2024 18:31:11 +0800 Subject: [PATCH 114/132] fix padding bug --- modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs | 4 ++-- .../Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs index c82cd7d312..5f5bcfc131 100644 --- a/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs +++ b/modules/Nncase.Modules.Ncnn/Evaluator/Ncnn/NcnnPadding.cs @@ -38,8 +38,8 @@ public IRType Visit(ITypeInferenceContext context, NcnnPadding target) { var input = context.CheckArgumentType(target, NcnnPadding.Input); var c = target.Front + target.Behind; - var h = target.Left + target.Right; - var w = target.Top + target.Bottom; + var h = target.Top + target.Bottom; + var w = target.Left + target.Right; return Visit(input, c, h, w); } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs index ae4804f621..cf84988d17 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs @@ -27,7 +27,7 @@ public partial class LowerPooling : RewriteRule public override Pattern Pattern { get; } = IsReduceWindow2D( "pdp", _ => true, - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() }, IsWildcard("initValue"), IsWildcard("filter"), IsWildcard("stride"), @@ -58,10 +58,10 @@ public partial class LowerPooling : RewriteRule var (strideW, strideH) = (stride_[1], stride_[0]); var padding_ = padding.Evaluate().AsTensor().ToArray(); - var (padLeft, padTop, padRight, padBottom) = (padding_[2], padding_[3], padding_[0], padding_[1]); + var (padLeft, padRight, padTop, padBottom) = (padding_[2], padding_[3], padding_[0], padding_[1]); // Globalpool has been converted to pool, reflected in the size of the kernel. - var args = new PoolingArgs(poolingType, kernelW, kernelH, strideW, strideH, padLeft, padRight, padTop, padBottom, false, ceilMode ? 0 : 1, countIncludePad, false, 0, 0, ceilMode); + var args = new PoolingArgs(poolingType, kernelW, kernelH, strideW, strideH, padLeft, padRight, padTop, padBottom, false, countIncludePad ? 1 : 0, countIncludePad, false, 0, 0, ceilMode); var inRes = Squeeze(input, new[] { 0 }); var inResO = new Var(inRes.CheckedType); From 52b083a23a1b326434b88e221ed7e38fc0eb37db Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 29 Mar 2024 18:32:39 +0800 Subject: [PATCH 115/132] fix pattern with fixesshape --- .../Passes/Rules/Ncnn/LowerBinary.cs | 4 ++-- .../Passes/Rules/Ncnn/LowerCast.cs | 2 +- .../Passes/Rules/Ncnn/LowerCelu.cs | 2 +- .../Passes/Rules/Ncnn/LowerClamp.cs | 2 +- .../Passes/Rules/Ncnn/LowerConcat.cs | 2 +- .../Passes/Rules/Ncnn/LowerConv2d.cs | 2 +- .../Passes/Rules/Ncnn/LowerConvTranspose.cs | 2 +- .../Passes/Rules/Ncnn/LowerCrop.cs | 2 +- .../Passes/Rules/Ncnn/LowerCumsum.cs | 2 +- .../Passes/Rules/Ncnn/LowerDequantize.cs | 2 +- .../Passes/Rules/Ncnn/LowerElu.cs | 2 +- .../Passes/Rules/Ncnn/LowerErf.cs | 2 +- .../Passes/Rules/Ncnn/LowerGELU.cs | 2 +- .../Passes/Rules/Ncnn/LowerHardSigmoid.cs | 2 +- .../Passes/Rules/Ncnn/LowerHardSwish.cs | 2 +- .../Passes/Rules/Ncnn/LowerInstanceNorm.cs | 2 +- .../Passes/Rules/Ncnn/LowerLSTM.cs | 2 +- .../Passes/Rules/Ncnn/LowerLayernorm.cs | 2 +- .../Passes/Rules/Ncnn/LowerLrn.cs | 2 +- .../Passes/Rules/Ncnn/LowerPReLU.cs | 2 +- .../Passes/Rules/Ncnn/LowerPadding.cs | 2 +- .../Passes/Rules/Ncnn/LowerPermute.cs | 2 +- .../Passes/Rules/Ncnn/LowerReduction.cs | 12 ++++++------ .../Passes/Rules/Ncnn/LowerReshape.cs | 8 +++++++- .../Passes/Rules/Ncnn/LowerSELU.cs | 2 +- .../Passes/Rules/Ncnn/LowerSigmoid.cs | 2 +- .../Passes/Rules/Ncnn/LowerSlice.cs | 2 +- .../Passes/Rules/Ncnn/LowerSoftmax.cs | 2 +- .../Passes/Rules/Ncnn/LowerSoftplus.cs | 2 +- .../Passes/Rules/Ncnn/LowerSqueeze.cs | 3 ++- .../Passes/Rules/Ncnn/LowerTile.cs | 2 +- .../Passes/Rules/Ncnn/LowerUnary.cs | 2 +- .../Passes/Rules/Ncnn/LowerUnsqueeze.cs | 3 ++- 33 files changed, 47 insertions(+), 39 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs index c2622bccae..96ae1c982c 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs @@ -28,8 +28,8 @@ public partial class LowerBinary : RewriteRule public override Pattern Pattern { get; } = IsBinary( target_name: "binary", _ => true, - IsWildcard("inputA") with { TypePattern = IsFloat() }, - IsWildcard("inputB") with { TypePattern = IsFloat() }); + IsWildcard("inputA") with { TypePattern = IsFloat() & HasFixedShape() }, + IsWildcard("inputB") with { TypePattern = IsFloat() & HasFixedShape() }); private static BinaryOperationType? MapBinaryOp(BinaryOp binaryOp) => binaryOp switch diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCast.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCast.cs index 0ebb65ccea..55f9f2c903 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCast.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCast.cs @@ -30,7 +30,7 @@ public partial class LowerCast : RewriteRule public override Pattern Pattern { get; } = IsCast( "cast", _ => true, - IsWildcard("input")); + IsWildcard("input") with { TypePattern = HasFixedShape() }); private int GetNcnnType(DataType inType) { diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs index 415a27b42b..d9b94c1936 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCelu.cs @@ -24,7 +24,7 @@ public partial class LowerCelu : RewriteRule { /// public override Pattern Pattern { get; } = IsCelu( - IsWildcard("input") with { TypePattern = IsFloat() }, + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }, IsTensorConst("alpha") with { TypePattern = IsFloatScalar() }); private Expr? GetReplace(Expr input, float alpha) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerClamp.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerClamp.cs index 5cbdf3b3bc..5a7c72d413 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerClamp.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerClamp.cs @@ -23,7 +23,7 @@ public partial class LowerClamp : RewriteRule { /// public override Pattern Pattern { get; } = IsClamp( - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("min", t => t.Value.ElementType == DataTypes.Float32), IsTensorConst("max", t => t.Value.ElementType == DataTypes.Float32)); diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConcat.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConcat.cs index 2b971f4e64..69544b21f6 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConcat.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConcat.cs @@ -25,7 +25,7 @@ public partial class LowerConcat : RewriteRule public override Pattern Pattern { get; } = IsConcat( "concat", _ => true, - IsTuple("tuple", IsVArgsRepeat("tupleInputs", () => IsWildcard()))); + IsTuple("tuple", IsVArgsRepeat("tupleInputs", () => IsWildcard() with { TypePattern = HasFixedShape() }))); // squeeze concat to 3D, get outputShape，set new axis as 1 private static (List> NewShape, List OldOutputShape) GetFixedShapeAndOldOutputShape(IReadOnlyList tupleInputs, int oldAxis) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs index 8c13e9e979..87a879bacd 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConv2d.cs @@ -28,7 +28,7 @@ public partial class LowerConv : RewriteRule public override Pattern Pattern { get; } = IsConv2D( "conv", conv => conv.PadMode == PadMode.Constant, - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("weights"), IsTensorConst("bias"), IsTensorConst("strides"), diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConvTranspose.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConvTranspose.cs index 0761161ef6..189dc404b8 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConvTranspose.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerConvTranspose.cs @@ -29,7 +29,7 @@ public partial class LowerConvTranspose : RewriteRule public override Pattern Pattern { get; } = IsConv2DTranspose( "conv", conv => conv.PadMode == PadMode.Constant, - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("weights"), IsTensorConst("bias"), IsTensorConst("outputShape"), diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCrop.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCrop.cs index bead6f8148..7d5844fa0b 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCrop.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCrop.cs @@ -27,7 +27,7 @@ public partial class LowerCrop : RewriteRule { /// public override Pattern Pattern { get; } = IsSlice( - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("starts"), IsTensorConst("ends"), IsTensorConst("axes"), diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCumsum.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCumsum.cs index 0af7b4c1be..5c45a9ed16 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCumsum.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerCumsum.cs @@ -25,7 +25,7 @@ public partial class LowerCumsum : RewriteRule { /// public override Pattern Pattern { get; } = IsCumSum( - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("axis"), IsTensorConst("exclusive"), IsTensorConst("reverse")); diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerDequantize.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerDequantize.cs index 5e98fed5a0..4652892db7 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerDequantize.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerDequantize.cs @@ -28,7 +28,7 @@ public partial class LowerDequantize : RewriteRule "deq", "output", _ => true, - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("param")); private Expr? GetReplace(Expr input, Dequantize deq, QuantParam[] param) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerElu.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerElu.cs index 1fe0110338..329c4f34b2 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerElu.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerElu.cs @@ -24,7 +24,7 @@ public partial class LowerElu : RewriteRule { /// public override Pattern Pattern { get; } = IsElu( - IsWildcard("input") with { TypePattern = IsFloat() }, + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }, IsTensorConst("alpha") with { TypePattern = IsFloatScalar() }); private Expr? GetReplace(Expr input, float alpha) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerErf.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerErf.cs index 4f15d9b6a9..8c265d6433 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerErf.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerErf.cs @@ -24,7 +24,7 @@ public partial class LowerErf : RewriteRule { /// public override Pattern Pattern { get; } = IsErf( - IsWildcard("input") with { TypePattern = IsFloat() }); + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }); private Expr? GetReplace(Expr input) { diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerGELU.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerGELU.cs index 6dbb3ab041..509b231cde 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerGELU.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerGELU.cs @@ -28,7 +28,7 @@ public partial class LowerGELU : RewriteRule { /// public override Pattern Pattern { get; } = IsGelu( - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("alpha")); private Expr? GetReplace(Expr input, float alpha) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSigmoid.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSigmoid.cs index cf5895600c..97de87a795 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSigmoid.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSigmoid.cs @@ -24,7 +24,7 @@ public partial class LowerHardSigmoid : RewriteRule { /// public override Pattern Pattern { get; } = IsHardSigmoid( - IsWildcard("input") with { TypePattern = IsFloat() }, + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }, IsTensorConst("alpha") with { TypePattern = IsScalar() }, IsTensorConst("beta") with { TypePattern = IsScalar() }); diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSwish.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSwish.cs index a307282a3b..6b281b6359 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSwish.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerHardSwish.cs @@ -24,7 +24,7 @@ public partial class LowerHardSwish : RewriteRule { /// public override Pattern Pattern { get; } = IsHardSwish( - IsWildcard("input")); + IsWildcard("input") with { TypePattern = HasFixedShape() }); private Expr? GetReplace(Expr input) { diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerInstanceNorm.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerInstanceNorm.cs index 7a9ef57da6..a4e5a845d8 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerInstanceNorm.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerInstanceNorm.cs @@ -24,7 +24,7 @@ public partial class LowerInstanceNorm : RewriteRule { /// public override Pattern Pattern { get; } = IsInstanceNormalization( - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("gamma"), IsTensorConst("beta"), IsTensorConst("epsilon") with { TypePattern = IsFloatScalar() }); diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLSTM.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLSTM.cs index 588bcae200..02125b4d97 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLSTM.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLSTM.cs @@ -31,7 +31,7 @@ public partial class LowerLSTM : RewriteRule target_name: "lstm", call_name: "call", _ => true, - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("w"), IsTensorConst("r"), IsTensorConst("b"), diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLayernorm.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLayernorm.cs index e8b480e0c8..de53fd37e7 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLayernorm.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLayernorm.cs @@ -28,7 +28,7 @@ public partial class LowerLayerNorm : RewriteRule public override Pattern Pattern { get; } = IsLayerNorm( "op", _ => true, - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("gamma"), IsTensorConst("beta")); diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLrn.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLrn.cs index e735f9bef0..1981f6f94b 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLrn.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerLrn.cs @@ -24,7 +24,7 @@ public partial class LowerLRN : RewriteRule { /// public override Pattern Pattern { get; } = IsLRN( - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("alpha") with { TypePattern = IsScalar() }, IsTensorConst("beta") with { TypePattern = IsScalar() }, IsTensorConst("bias") with { TypePattern = IsScalar() }, diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPReLU.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPReLU.cs index afafc54dae..9fabba82ab 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPReLU.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPReLU.cs @@ -25,7 +25,7 @@ public partial class LowerPReLU : RewriteRule { /// public override Pattern Pattern { get; } = IsPRelu( - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("slope")); private Expr? GetReplace(Expr input, Tensor slope) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPadding.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPadding.cs index 920239c2c9..360812ee64 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPadding.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPadding.cs @@ -28,7 +28,7 @@ public partial class LowerPadding : RewriteRule "pad", "padCall", x => true, - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("pads"), IsTensorConst("padValue")); diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPermute.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPermute.cs index 444f6b5777..f42d0de4c9 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPermute.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPermute.cs @@ -25,7 +25,7 @@ public partial class LowerPermute : RewriteRule { /// public override Pattern Pattern { get; } = IsTranspose( - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("perm")); private Expr? GetReplace(Expr input, int[] perm) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReduction.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReduction.cs index 62bc5259a3..eb3fe7f03b 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReduction.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReduction.cs @@ -29,7 +29,7 @@ public partial class LowerReduction : RewriteRule "reduce", "reduceCall", _ => true, - IsWildcard("input") with { TypePattern = IsFloat() }, + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }, IsTensorConst("axis"), IsTensorConst("initValue"), IsTensorConst("keepDims")); @@ -99,7 +99,7 @@ public partial class LowerReductionSumSquare : LowerReduction IsUnary( "square", UnaryOp.Square, - IsWildcard("input") with { TypePattern = IsFloat() }), + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }), IsTensorConst("axis"), IsTensorConst("initValue"), IsTensorConst("keepDims")); @@ -122,7 +122,7 @@ public partial class LowerReductionL1 : LowerReduction IsUnary( "abs", UnaryOp.Abs, - IsWildcard("input") with { TypePattern = IsFloat() }), + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }), IsTensorConst("axis"), IsTensorConst("initValue"), IsTensorConst("keepDims")); @@ -148,7 +148,7 @@ public class LowerReductionL2 : LowerReduction IsUnary( "square", UnaryOp.Square, - IsWildcard("input") with { TypePattern = IsFloat() }), + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }), IsTensorConst("axis"), IsTensorConst("initValue"), IsTensorConst("keepDims"))); @@ -171,7 +171,7 @@ public partial class LowerReductionLogSum : LowerReduction "reduce", "reduceCall", ReduceOp.Sum, - IsWildcard("input") with { TypePattern = IsFloat() }, + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }, IsTensorConst("axis"), IsTensorConst("initValue"), IsTensorConst("keepDims"))); @@ -197,7 +197,7 @@ public partial class LowerReductionLogSumExp : LowerReduction IsUnary( "exp", UnaryOp.Exp, - IsWildcard("input") with { TypePattern = IsFloat() }), + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }), IsTensorConst("axis"), IsTensorConst("initValue"), IsTensorConst("keepDims"))); diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReshape.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReshape.cs index fc13bc6154..eb13c91cb1 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReshape.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReshape.cs @@ -7,6 +7,7 @@ using Nncase.PatternMatch; using static Nncase.IR.F.Ncnn; using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; using static Nncase.PatternMatch.F.Tensors; using static Nncase.PatternMatch.Utility; @@ -17,7 +18,7 @@ public partial class LowerReshape : RewriteRule { /// public override Pattern Pattern { get; } = IsReshape( - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() & !IsScalar() }, IsTensorConst("shape")); private Expr? GetReplace(Expr input, Expr shape) @@ -52,6 +53,11 @@ public partial class LowerReshape : RewriteRule needSqueeze += 2; } // left 3D shape. + if (outputShape.Count == 0) + { + return null; + } + r = new Call(new Fusion("ncnn", NcnnReshape(newInputVar, outputShape.ToArray()), new[] { newInputVar }), newInput); if (needSqueeze == 3) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSELU.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSELU.cs index e54e5b01c9..7e1148fef8 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSELU.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSELU.cs @@ -24,7 +24,7 @@ public partial class LowerSELU : RewriteRule { /// public override Pattern Pattern { get; } = IsSelu( - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("alpha"), IsTensorConst("gamma")); diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSigmoid.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSigmoid.cs index 79fe8a7169..52f5a1e81a 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSigmoid.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSigmoid.cs @@ -24,7 +24,7 @@ public partial class LowerSigmoid : RewriteRule { /// public override Pattern Pattern { get; } = IsSigmoid( - IsWildcard("input")); + IsWildcard("input") with { TypePattern = HasFixedShape() }); private Expr? GetReplace(Expr input) { diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSlice.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSlice.cs index 1eb26a6ea5..21eea2e09a 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSlice.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSlice.cs @@ -28,7 +28,7 @@ public partial class LowerSlice : RewriteRule { /// public override Pattern Pattern { get; } = IsSplit( - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("axis") with { TypePattern = IsScalar() }, IsTensorConst("slices")); diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftmax.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftmax.cs index 9dcdd24f74..2732e0cb23 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftmax.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftmax.cs @@ -24,7 +24,7 @@ public partial class LowerSoftmax : RewriteRule { /// public override Pattern Pattern { get; } = IsSoftmax( - IsWildcard("input") with { TypePattern = IsFloat() }, + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }, IsTensorConst("axis")); // squeeze softmax to 3D，set axis to 1 diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftplus.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftplus.cs index c3ff475253..db25ebd1b7 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftplus.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSoftplus.cs @@ -24,7 +24,7 @@ public partial class LowerSoftplus : RewriteRule { /// public override Pattern Pattern { get; } = IsSoftplus( - IsWildcard("input")); + IsWildcard("input") with { TypePattern = HasFixedShape() }); private Expr? GetReplace(Expr input) { diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs index bd0c79d6fa..f8990d52e1 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSqueeze.cs @@ -6,6 +6,7 @@ using Nncase.PatternMatch; using static Nncase.IR.F.Ncnn; using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; using static Nncase.PatternMatch.F.Tensors; using static Nncase.PatternMatch.Utility; @@ -16,7 +17,7 @@ public partial class LowerSqueeze : RewriteRule { /// public override Pattern Pattern { get; } = IsSqueeze( - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() & !IsScalar() }, IsTensorConst("dims")); private Expr? GetReplace(Expr input, int[] dims) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerTile.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerTile.cs index 72ab225b1e..4074972494 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerTile.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerTile.cs @@ -25,7 +25,7 @@ public partial class LowerTile : RewriteRule { /// public override Pattern Pattern { get; } = IsTile( - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() }, IsTensorConst("repeats")); private Expr? GetReplace(Expr input, int[] repeats) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnary.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnary.cs index ef4f3fbcb9..9b91e1d9d7 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnary.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnary.cs @@ -26,7 +26,7 @@ public partial class LowerUnary : RewriteRule public override Pattern Pattern { get; } = IsUnary( target_name: "unary", _ => true, - IsWildcard("input") with { TypePattern = IsFloat() }); + IsWildcard("input") with { TypePattern = IsFloat() & HasFixedShape() }); private static UnaryOperationType? MapUnaryOp(UnaryOp unaryOp) => unaryOp switch diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnsqueeze.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnsqueeze.cs index c71ff61685..c494de8fb2 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnsqueeze.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerUnsqueeze.cs @@ -6,6 +6,7 @@ using Nncase.PatternMatch; using static Nncase.IR.F.Ncnn; using static Nncase.IR.F.Tensors; +using static Nncase.IR.TypePatternUtility; using static Nncase.PatternMatch.F.Tensors; using static Nncase.PatternMatch.Utility; @@ -16,7 +17,7 @@ public partial class LowerUnsqueeze : RewriteRule { /// public override Pattern Pattern { get; } = IsUnsqueeze( - IsWildcard("input"), + IsWildcard("input") with { TypePattern = HasFixedShape() & !IsScalar() }, IsTensorConst("dims")); private Expr? GetReplace(Expr input, int[] dims) From bd90248db62790e316eb9761aff9462ade0d2aea Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 29 Mar 2024 18:33:09 +0800 Subject: [PATCH 116/132] add paddings NHWC to NCHW --- src/Nncase.Core/IR/Tensors/Functional.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/Nncase.Core/IR/Tensors/Functional.cs b/src/Nncase.Core/IR/Tensors/Functional.cs index d20f69cbf5..5460377afe 100644 --- a/src/Nncase.Core/IR/Tensors/Functional.cs +++ b/src/Nncase.Core/IR/Tensors/Functional.cs @@ -187,4 +187,19 @@ public static Call TopK(Expr x, Expr k, Expr axis, Expr largest, Expr sorted) => public static Call IndexOf(Expr input, Expr value) => new Call(new IndexOf(), input, value); public static Call Trilu(Expr input, Expr k, Expr upper) => new Call(new Trilu(), input, k, upper); + + public static Call PaddingNHWCToNCHW(Expr paddings) + { + var strides = new int[1] { 1 }; + var newPaddings = Concat( + new IR.Tuple(new[] + { + Slice(paddings, new int[] { 0 }, new int[] { 1 }, new int[] { 0 }, strides), + Slice(paddings, new int[] { 3 }, new int[] { 4 }, new int[] { 0 }, strides), + Slice(paddings, new int[] { 1 }, new int[] { 2 }, new int[] { 0 }, strides), + Slice(paddings, new int[] { 2 }, new int[] { 3 }, new int[] { 0 }, strides), + }), + 0); + return newPaddings; + } } From 9d68d7400f4db75ef0ba3aef967a24c7137f2e53 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 29 Mar 2024 18:33:50 +0800 Subject: [PATCH 117/132] fix fold pad to reducewindows2d --- src/Nncase.Passes/Rules/Neutral/FoldPad.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Nncase.Passes/Rules/Neutral/FoldPad.cs b/src/Nncase.Passes/Rules/Neutral/FoldPad.cs index 2ea62feff0..ef0a5506af 100644 --- a/src/Nncase.Passes/Rules/Neutral/FoldPad.cs +++ b/src/Nncase.Passes/Rules/Neutral/FoldPad.cs @@ -132,7 +132,7 @@ public sealed partial class FoldReduceWindow2DPads : IRewriteRule IsWildcard("ceilMode"), IsWildcard("countIncludePad")); - private Expr? GetReplace(ReduceWindow2D pdp, Expr input, Expr initValue, Expr filter, Expr stride, Tensor padding, Expr dilation, Expr ceilMode, Expr countIncludePad, Tensor ext_pad, float ext_pad_init) + private Expr? GetReplace(ReduceWindow2D pdp, Expr input, Expr initValue, Expr filter, Expr stride, Tensor padding, Expr dilation, Expr ceilMode, bool countIncludePad, Tensor ext_pad, float ext_pad_init) { if (!(ext_pad[0, 0] == 0 && ext_pad[0, 1] == 0 && ext_pad[1, 0] == 0 && ext_pad[1, 1] == 0)) @@ -140,6 +140,11 @@ public sealed partial class FoldReduceWindow2DPads : IRewriteRule return null; } + if (countIncludePad || pdp.ReduceOp != ReduceOp.Max) + { + return null; + } + var new_pad = padding.Clone(); new_pad[0, 0] += ext_pad[2, 0]; new_pad[0, 1] += ext_pad[2, 1]; From 23bcda5bcf7b600369da3f4859a71e36b0a4209b Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 29 Mar 2024 18:34:15 +0800 Subject: [PATCH 118/132] fix param format --- src/Nncase.Core/Runtime/Ncnn/ParamDict.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs b/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs index 8088eeae21..e84d91d3fc 100644 --- a/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs +++ b/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs @@ -189,8 +189,8 @@ _ when float.IsNegativeInfinity(x) => "-3.402823e+38", { writer.Write(' '); } - - writer.Write('\n'); } + + writer.Write('\n'); } } From 1571e149b5e0766cab4563f73f9c43335e1b9511 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 29 Mar 2024 18:35:13 +0800 Subject: [PATCH 119/132] fix tflite pad transpose input\output\paddings --- src/Nncase.Importer/TFLite/Pad.cs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Nncase.Importer/TFLite/Pad.cs b/src/Nncase.Importer/TFLite/Pad.cs index 7cf10f8998..9e140c5638 100644 --- a/src/Nncase.Importer/TFLite/Pad.cs +++ b/src/Nncase.Importer/TFLite/Pad.cs @@ -18,6 +18,8 @@ public partial class TFLiteImporter private Expr VisitPad(in tflite.Operator op) { var (input, paddings) = GetInputExprs(op, 0, 1); + input = F.Tensors.NHWCToNCHW(input); + paddings = F.Tensors.PaddingNHWCToNCHW(paddings); var padValue = GetInputTensor(op, 0).Type switch { TensorType.FLOAT32 => 0.0f, @@ -26,20 +28,23 @@ private Expr VisitPad(in tflite.Operator op) _ => throw new NotSupportedException("Unsupported Constant Pad Value"), }; - return F.NN.Pad(input, paddings, PadMode.Constant, padValue); + return F.Tensors.NCHWToNHWC(F.NN.Pad(input, paddings, PadMode.Constant, padValue)); } private Expr VisitPadV2(in tflite.Operator op) { var (input, paddings) = GetInputExprs(op, 0, 1); + input = F.Tensors.NHWCToNCHW(input); + paddings = F.Tensors.PaddingNHWCToNCHW(paddings); var padValue = GetInputExprs(op, 2); - return F.NN.Pad(input, paddings, PadMode.Constant, padValue); + return F.Tensors.NCHWToNHWC(F.NN.Pad(input, paddings, PadMode.Constant, padValue)); } private Expr VisitMirrorPad(in tflite.Operator op) { var (input, paddings) = GetInputExprs(op, 0, 1); - + input = F.Tensors.NHWCToNCHW(input); + paddings = F.Tensors.PaddingNHWCToNCHW(paddings); var padMode = op.BuiltinOptionsAsMirrorPadOptions().Mode switch { tflite.MirrorPadMode.REFLECT => PadMode.Reflect, @@ -47,7 +52,7 @@ private Expr VisitMirrorPad(in tflite.Operator op) _ => throw new NotSupportedException("Unsupported Mirror Pad Mode"), }; - return F.NN.Pad(input, paddings, padMode, 0.0f); + return F.Tensors.NCHWToNHWC(F.NN.Pad(input, paddings, padMode, 0.0f)); } } } From 2f45751eedbfe747fdb4781f45d11dbdc81da94e Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 29 Mar 2024 18:36:01 +0800 Subject: [PATCH 120/132] fix tflite reducewindows2d with real countIncludePad --- src/Nncase.Importer/TFLite/ReduceWindow2D.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Nncase.Importer/TFLite/ReduceWindow2D.cs b/src/Nncase.Importer/TFLite/ReduceWindow2D.cs index 1fabda70ed..75f099169c 100644 --- a/src/Nncase.Importer/TFLite/ReduceWindow2D.cs +++ b/src/Nncase.Importer/TFLite/ReduceWindow2D.cs @@ -25,9 +25,10 @@ private Expr VisitReduceWindow2D(in tflite.Operator op, ReduceOp reduceOp, float var filter = Tensor.From(new[] { filterH, filterW }, new[] { 2 }); var stride = Tensor.From(new[] { strideH, strideW }, new[] { 2 }); var padding = Util.ConcatPadding(padH, padW); + var countIncludePad = option.Padding == tflite.Padding.SAME ? false : true; return F.Tensors.NCHWToNHWC( F.NN.ReduceWindow2D( - reduceOp, input, initValue, filter, stride, padding, Tensor.From(new long[] { 1, 1 }), false, false)); + reduceOp, input, initValue, filter, stride, padding, Tensor.From(new long[] { 1, 1 }), false, countIncludePad)); } } } From ed2ff1344aa67bfa6da3a95286fb18806c59e314 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 1 Apr 2024 10:23:39 +0800 Subject: [PATCH 121/132] fix pool pad mode --- modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs index cf84988d17..8418318697 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPooling.cs @@ -61,7 +61,7 @@ public partial class LowerPooling : RewriteRule var (padLeft, padRight, padTop, padBottom) = (padding_[2], padding_[3], padding_[0], padding_[1]); // Globalpool has been converted to pool, reflected in the size of the kernel. - var args = new PoolingArgs(poolingType, kernelW, kernelH, strideW, strideH, padLeft, padRight, padTop, padBottom, false, countIncludePad ? 1 : 0, countIncludePad, false, 0, 0, ceilMode); + var args = new PoolingArgs(poolingType, kernelW, kernelH, strideW, strideH, padLeft, padRight, padTop, padBottom, false, 1, countIncludePad, false, 0, 0, ceilMode); var inRes = Squeeze(input, new[] { 0 }); var inResO = new Var(inRes.CheckedType); From a31b38051be8037e4b1d8a04af0ad155e1737ddb Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 1 Apr 2024 12:27:19 +0800 Subject: [PATCH 122/132] fix pad with tp --- src/Nncase.Core/IR/Tensors/Functional.cs | 55 ++++++++++++++++-------- src/Nncase.Importer/TFLite/Pad.cs | 13 +++--- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/src/Nncase.Core/IR/Tensors/Functional.cs b/src/Nncase.Core/IR/Tensors/Functional.cs index 5460377afe..d8df282669 100644 --- a/src/Nncase.Core/IR/Tensors/Functional.cs +++ b/src/Nncase.Core/IR/Tensors/Functional.cs @@ -35,26 +35,31 @@ public static Expr NHWCToNCHW(Expr input) } else { - throw new InvalidOperationException(); + return input; } return Transpose(input, perm); } - public static Expr NCHWToNHWC(Expr input) + public static Expr NCHWToNHWC(Expr input, int rank = 0) { + if (rank == 0) + { + rank = input.CheckedShape.Rank; + } + int[] perm; - if (input.CheckedShape.Rank == 4) + if (rank == 4) { perm = new[] { 0, 2, 3, 1 }; } - else if (input.CheckedShape.Rank == 3) + else if (rank == 3) { perm = new[] { 0, 2, 1 }; } else { - throw new InvalidOperationException(); + return input; } return Transpose(input, perm); @@ -188,18 +193,34 @@ public static Call TopK(Expr x, Expr k, Expr axis, Expr largest, Expr sorted) => public static Call Trilu(Expr input, Expr k, Expr upper) => new Call(new Trilu(), input, k, upper); - public static Call PaddingNHWCToNCHW(Expr paddings) + public static Expr PaddingNHWCToNCHW(Expr paddings, int rank) { - var strides = new int[1] { 1 }; - var newPaddings = Concat( - new IR.Tuple(new[] - { - Slice(paddings, new int[] { 0 }, new int[] { 1 }, new int[] { 0 }, strides), - Slice(paddings, new int[] { 3 }, new int[] { 4 }, new int[] { 0 }, strides), - Slice(paddings, new int[] { 1 }, new int[] { 2 }, new int[] { 0 }, strides), - Slice(paddings, new int[] { 2 }, new int[] { 3 }, new int[] { 0 }, strides), - }), - 0); - return newPaddings; + switch (rank) + { + case 1: + case 2: + return paddings; + case 3: + return Concat( + new IR.Tuple( + new[] { + Slice(paddings, new int[] { 0 }, new int[] { 1 }, new int[] { 0 }, new int[] { 1 }), + Slice(paddings, new int[] { 2 }, new int[] { 3 }, new int[] { 0 }, new int[] { 1 }), + Slice(paddings, new int[] { 1 }, new int[] { 2 }, new int[] { 0 }, new int[] { 1 }), + }), + 0); + case 4: + return Concat( + new IR.Tuple( + new[] { + Slice(paddings, new int[] { 0 }, new int[] { 1 }, new int[] { 0 }, new int[] { 1 }), + Slice(paddings, new int[] { 3 }, new int[] { 4 }, new int[] { 0 }, new int[] { 1 }), + Slice(paddings, new int[] { 1 }, new int[] { 2 }, new int[] { 0 }, new int[] { 1 }), + Slice(paddings, new int[] { 2 }, new int[] { 3 }, new int[] { 0 }, new int[] { 1 }), + }), + 0); + default: + return paddings; + } } } diff --git a/src/Nncase.Importer/TFLite/Pad.cs b/src/Nncase.Importer/TFLite/Pad.cs index 9e140c5638..6ed91debe2 100644 --- a/src/Nncase.Importer/TFLite/Pad.cs +++ b/src/Nncase.Importer/TFLite/Pad.cs @@ -4,6 +4,7 @@ using System; using System.Diagnostics; using System.Linq; +using NetFabric.Hyperlinq; using Nncase.IR; using Nncase.IR.Tensors; using tflite; @@ -19,7 +20,7 @@ private Expr VisitPad(in tflite.Operator op) { var (input, paddings) = GetInputExprs(op, 0, 1); input = F.Tensors.NHWCToNCHW(input); - paddings = F.Tensors.PaddingNHWCToNCHW(paddings); + paddings = F.Tensors.PaddingNHWCToNCHW(paddings, input.CheckedShape.Rank); var padValue = GetInputTensor(op, 0).Type switch { TensorType.FLOAT32 => 0.0f, @@ -28,23 +29,23 @@ private Expr VisitPad(in tflite.Operator op) _ => throw new NotSupportedException("Unsupported Constant Pad Value"), }; - return F.Tensors.NCHWToNHWC(F.NN.Pad(input, paddings, PadMode.Constant, padValue)); + return F.Tensors.NCHWToNHWC(F.NN.Pad(input, paddings, PadMode.Constant, padValue), input.CheckedShape.Rank); } private Expr VisitPadV2(in tflite.Operator op) { var (input, paddings) = GetInputExprs(op, 0, 1); input = F.Tensors.NHWCToNCHW(input); - paddings = F.Tensors.PaddingNHWCToNCHW(paddings); + paddings = F.Tensors.PaddingNHWCToNCHW(paddings, input.CheckedShape.Rank); var padValue = GetInputExprs(op, 2); - return F.Tensors.NCHWToNHWC(F.NN.Pad(input, paddings, PadMode.Constant, padValue)); + return F.Tensors.NCHWToNHWC(F.NN.Pad(input, paddings, PadMode.Constant, padValue), input.CheckedShape.Rank); } private Expr VisitMirrorPad(in tflite.Operator op) { var (input, paddings) = GetInputExprs(op, 0, 1); input = F.Tensors.NHWCToNCHW(input); - paddings = F.Tensors.PaddingNHWCToNCHW(paddings); + paddings = F.Tensors.PaddingNHWCToNCHW(paddings, input.CheckedShape.Rank); var padMode = op.BuiltinOptionsAsMirrorPadOptions().Mode switch { tflite.MirrorPadMode.REFLECT => PadMode.Reflect, @@ -52,7 +53,7 @@ private Expr VisitMirrorPad(in tflite.Operator op) _ => throw new NotSupportedException("Unsupported Mirror Pad Mode"), }; - return F.Tensors.NCHWToNHWC(F.NN.Pad(input, paddings, padMode, 0.0f)); + return F.Tensors.NCHWToNHWC(F.NN.Pad(input, paddings, padMode, 0.0f), input.CheckedShape.Rank); } } } From 643ab65e2badd2fef2f9a9c68d05b1c3f5cb703b Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 1 Apr 2024 12:29:02 +0800 Subject: [PATCH 123/132] fix DeConv test case with limitation of stride and kernel size --- tests/importer/tflite_/basic/test_conv2d_transpose.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/importer/tflite_/basic/test_conv2d_transpose.py b/tests/importer/tflite_/basic/test_conv2d_transpose.py index 0b2a8b8c9a..61a8ef0e04 100644 --- a/tests/importer/tflite_/basic/test_conv2d_transpose.py +++ b/tests/importer/tflite_/basic/test_conv2d_transpose.py @@ -91,13 +91,14 @@ def __call__(self, x): @pytest.mark.parametrize('dilations', dilations) @pytest.mark.parametrize('bias', biases) def test_conv2d_transpose(n, i_channels, i_size, k_size, o_channels, strides, padding, dilations, bias, request): - module = _make_module(n, i_channels, i_size, k_size, o_channels, + if k_size[0] >= strides[0] and k_size[1] >= strides[1]: + module = _make_module(n, i_channels, i_size, k_size, o_channels, strides, padding, dilations, bias) - #runner = TfliteTestRunner(request.node.name, ['k510']) - runner = TfliteTestRunner(request.node.name) - model_file = runner.from_tensorflow(module) - runner.run(model_file) + #runner = TfliteTestRunner(request.node.name, ['k510']) + runner = TfliteTestRunner(request.node.name) + model_file = runner.from_tensorflow(module) + runner.run(model_file) if __name__ == "__main__": From dbd361c94b930ee5419bc7e409aa323ec1eab0fd Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Mon, 1 Apr 2024 14:56:04 +0800 Subject: [PATCH 124/132] fix less than 4D --- .../Passes/Rules/Ncnn/LowerReduction.cs | 52 +++++++++++++------ .../Passes/Rules/Ncnn/LowerSigmoid.cs | 28 +++++++--- 2 files changed, 56 insertions(+), 24 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReduction.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReduction.cs index eb3fe7f03b..42f088b17c 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReduction.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerReduction.cs @@ -47,6 +47,7 @@ public virtual int GetOpT() return null; } + // Support other reduction ops combined with several ops. var otherOpType = GetOpT(); var reductionType = otherOpType == -1 ? reduce.ReduceOp switch @@ -56,34 +57,53 @@ public virtual int GetOpT() ReduceOp.Max => 4, ReduceOp.Min => 5, ReduceOp.Prod => 6, - _ => throw new NotImplementedException($"{reduce.ReduceOp} not support in ncnn!"), + _ => -1, } : otherOpType; - - var newAxis = axis; - if (axis.Length == input.CheckedShape.Rank) + if (reductionType == -1) { - // 排除batch维度 - newAxis = newAxis.Remove(0); - newAxis = newAxis.Remove(-input.CheckedShape.Rank); + return null; } - for (int i = 0; i < newAxis.Length; i++) + var newAxis = axis; + + var newInput = input; + var newInputVar = new Var(newInput.CheckedType); + if (input.CheckedShape.Rank == 4) { - if (newAxis[i] == 0 || newAxis[i] > 4 || newAxis[i] < -3) + if (axis.Length == input.CheckedShape.Rank) + { + // 排除batch维度 + newAxis = newAxis.Remove(0); + newAxis = newAxis.Remove(-input.CheckedShape.Rank); + } + + for (int i = 0; i < newAxis.Length; i++) { - return null; + if (newAxis[i] == 0 || newAxis[i] > 4 || newAxis[i] < -3) + { + return null; + } + + newAxis[i] = newAxis[i] > 0 ? newAxis[i] - 1 : newAxis[i]; } - newAxis[i] = newAxis[i] > 0 ? newAxis[i] - 1 : newAxis[i]; + newInput = Squeeze(input, new[] { 0 }); + newInputVar = new Var(newInput.CheckedType); } - var inRes = Squeeze(input, new[] { 0 }); - var inResO = new Var(inRes.CheckedType); - var args = new ReductionArgs(reductionType, newAxis.Length == inRes.CheckedShape.Rank ? 1 : 0, 0, newAxis, keepDims ? 1 : 0); + var args = new ReductionArgs(reductionType, newAxis.Length == newInput.CheckedShape.Rank ? 1 : 0, 0, newAxis, keepDims ? 1 : 0); - var pool = new Call(new Fusion("ncnn", NcnnReduction(inResO, args), new[] { inResO }), inRes); - return Unsqueeze(pool, new[] { 0 }); + var pool = new Call(new Fusion("ncnn", NcnnReduction(newInputVar, args), new[] { newInputVar }), newInput); + + if (input.CheckedShape.Rank == 4) + { + return Unsqueeze(pool, new[] { 0 }); + } + else + { + return pool; + } } } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSigmoid.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSigmoid.cs index 52f5a1e81a..b7ae02fc2f 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSigmoid.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerSigmoid.cs @@ -28,17 +28,29 @@ public partial class LowerSigmoid : RewriteRule private Expr? GetReplace(Expr input) { - if (input.CheckedShape.Count > 3) + if (input.CheckedShape.Rank > 4 || (input.CheckedShape.Rank == 4 && input.CheckedShape[0].FixedValue != 1)) { - var inRes = Squeeze(input, new[] { 0 }); - var inResO = new Var(inRes.CheckedType); + return null; + } + + var newInput = input; + var newInputVar = new Var(newInput.CheckedType); - var sigmoid = new Call(new Fusion("ncnn", NcnnSigmoid(inResO), new[] { inResO }), inRes); - return Unsqueeze(sigmoid, new[] { 0 }); + if (input.CheckedShape.Count == 4 && input.CheckedShape[0].FixedValue == 1) + { + newInput = Squeeze(input, new[] { 0 }); + newInputVar = new Var(newInput.CheckedType); } - // if input has shape [1] - var newInput = new Var(input.CheckedType); - return new Call(new Fusion("ncnn", NcnnSigmoid(newInput), new[] { newInput }), input); + var sigmoid = new Call(new Fusion("ncnn", NcnnSigmoid(newInputVar), new[] { newInputVar }), newInput); + + if (input.CheckedShape.Count == 4 && input.CheckedShape[0].FixedValue == 1) + { + return Unsqueeze(sigmoid, new int[] { 0 }); + } + else + { + return sigmoid; + } } } From 54a7efca0d4d561d9fe21735a6cc2327cb79a606 Mon Sep 17 00:00:00 2001 From: curioyang Date: Mon, 1 Apr 2024 06:58:30 +0000 Subject: [PATCH 125/132] Apply code-format changes --- tests/importer/tflite_/basic/test_conv2d_transpose.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/importer/tflite_/basic/test_conv2d_transpose.py b/tests/importer/tflite_/basic/test_conv2d_transpose.py index 61a8ef0e04..a8eb0cf2eb 100644 --- a/tests/importer/tflite_/basic/test_conv2d_transpose.py +++ b/tests/importer/tflite_/basic/test_conv2d_transpose.py @@ -93,7 +93,7 @@ def __call__(self, x): def test_conv2d_transpose(n, i_channels, i_size, k_size, o_channels, strides, padding, dilations, bias, request): if k_size[0] >= strides[0] and k_size[1] >= strides[1]: module = _make_module(n, i_channels, i_size, k_size, o_channels, - strides, padding, dilations, bias) + strides, padding, dilations, bias) #runner = TfliteTestRunner(request.node.name, ['k510']) runner = TfliteTestRunner(request.node.name) From 8e4f7c3e2d4d0a4c4f3e228d78e30b7fcf5e90e7 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Tue, 2 Apr 2024 10:21:33 +0800 Subject: [PATCH 126/132] fix Prelu bug --- modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPReLU.cs | 2 +- modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPReLU.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPReLU.cs b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPReLU.cs index e1cde8c8ca..4c06407f2c 100644 --- a/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPReLU.cs +++ b/modules/Nncase.Modules.Ncnn/IR/Ncnn/NcnnPReLU.cs @@ -30,6 +30,6 @@ public sealed partial class NcnnPReLU : Op /// public override string DisplayProperty() { - return $""; + return $"{string.Join(", ", Slope)}"; } } diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPReLU.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPReLU.cs index 9fabba82ab..5d68910827 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPReLU.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerPReLU.cs @@ -31,7 +31,7 @@ public partial class LowerPReLU : RewriteRule private Expr? GetReplace(Expr input, Tensor slope) { // TODO: split input - if (input.CheckedShape.ToList()[0] != 1) + if (input.CheckedShape.ToList()[0] != 1 || input.CheckedShape.Rank > 4) { return null; } @@ -39,7 +39,7 @@ public partial class LowerPReLU : RewriteRule var inRes = Squeeze(input, new[] { 0 }); var inResO = new Var(inRes.CheckedType); - var pReLU = new Call(new Fusion("ncnn", NcnnPReLU(inResO, slope.ToArray()), new[] { inResO }), input); + var pReLU = new Call(new Fusion("ncnn", NcnnPReLU(inResO, slope.ToArray()), new[] { inResO }), inRes); return Unsqueeze(pReLU, new[] { 0 }); } } From 782f4c0f8a74a7ac998d045e91386822cc2ec32b Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Tue, 2 Apr 2024 16:28:54 +0800 Subject: [PATCH 127/132] fix string bug --- modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs | 6 +++--- src/Nncase.Core/Runtime/Ncnn/ParamDict.cs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index fb27f8d978..28549bb046 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -434,11 +434,11 @@ public void Matmul(string[] name, string inputA, string inputB, int lOrR, float[ { if (lOrR == 1) { - inputList[0] = name + "_memorydata"; + inputList[0] = name[0] + "_memorydata"; } else { - inputList[1] = name + "_memorydata"; + inputList[1] = name[0] + "_memorydata"; } var paramDict = new ParamDict(); @@ -456,7 +456,7 @@ public void Matmul(string[] name, string inputA, string inputB, int lOrR, float[ paramDict[index] = new ParamValue { Kind = ParamKind.Int, IntValue = constShape[constShape.Length - 1 - i] }; } - AddLayer("MemoryData", name + "_memorydata", Array.Empty(), new[] { name + "_memorydata" }, paramDict); + AddLayer("MemoryData", name[0] + "_memorydata", Array.Empty(), new[] { name[0] + "_memorydata" }, paramDict); WriteFloatArray(constInput); } diff --git a/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs b/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs index e84d91d3fc..57efc2ad46 100644 --- a/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs +++ b/src/Nncase.Core/Runtime/Ncnn/ParamDict.cs @@ -191,6 +191,6 @@ _ when float.IsNegativeInfinity(x) => "-3.402823e+38", } } - writer.Write('\n'); + writer.Write(' '); } } From 30f17534be507c22733e7745c59130d469abfd70 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 31 May 2024 11:59:04 +0800 Subject: [PATCH 128/132] fix bin data --- modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index 28549bb046..b0b5713423 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -46,7 +46,7 @@ public void SaveBin(string dumpPath, uint id) _binWriter.BaseStream.Seek(0, SeekOrigin.Begin); _binWriter.BaseStream.CopyTo(fileStream); - // _binWriter.BaseStream.SetLength(0); + _binWriter.BaseStream.SetLength(0); } } From dac19f2aa8ac1d2dbbe2062e6a035ced05bdc8ee Mon Sep 17 00:00:00 2001 From: curioyang Date: Fri, 31 May 2024 04:47:03 +0000 Subject: [PATCH 129/132] Apply code-format changes --- modules/Nncase.Modules.CPU/packages.lock.json | 8 ++ .../Nncase.Modules.Ncnn/packages.lock.json | 112 ++++++++++-------- src/Nncase.Cli/packages.lock.json | 7 ++ src/Nncase.Compiler/packages.lock.json | 7 ++ src/Nncase.Studio/packages.lock.json | 7 ++ .../packages.lock.json | 7 ++ src/Nncase.Tests/packages.lock.json | 7 ++ 7 files changed, 104 insertions(+), 51 deletions(-) diff --git a/modules/Nncase.Modules.CPU/packages.lock.json b/modules/Nncase.Modules.CPU/packages.lock.json index dded23c243..08721aae0d 100644 --- a/modules/Nncase.Modules.CPU/packages.lock.json +++ b/modules/Nncase.Modules.CPU/packages.lock.json @@ -190,10 +190,18 @@ "nncase.io": { "type": "Project" }, + "nncase.modules.ncnn": { + "type": "Project", + "dependencies": { + "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Passes": "[1.0.0, )" + } + }, "nncase.modules.stackvm": { "type": "Project", "dependencies": { "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Modules.Ncnn": "[1.0.0, )", "Nncase.Passes": "[1.0.0, )" } }, diff --git a/modules/Nncase.Modules.Ncnn/packages.lock.json b/modules/Nncase.Modules.Ncnn/packages.lock.json index 8820c05237..e5ee906aff 100644 --- a/modules/Nncase.Modules.Ncnn/packages.lock.json +++ b/modules/Nncase.Modules.Ncnn/packages.lock.json @@ -69,33 +69,40 @@ }, "Microsoft.Extensions.Configuration.Abstractions": { "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "qWzV9o+ZRWq+pGm+1dF+R7qTgTYoXvbyowRoBxQJGfqTpqDun2eteerjRQhq5PQ/14S+lqto3Ft4gYaRyl4rdQ==", + "resolved": "8.0.0", + "contentHash": "3lE/iLSutpgX1CC0NOW70FJoGARRHbyKmG7dc0klnUZ9Dd9hS6N/POPWhKhMLCEuNN5nXEY5agmlFtH562vqhQ==", "dependencies": { - "Microsoft.Extensions.Primitives": "6.0.0" + "Microsoft.Extensions.Primitives": "8.0.0" } }, "Microsoft.Extensions.DependencyInjection.Abstractions": { "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "xlzi2IYREJH3/m6+lUrQlujzX8wDitm4QGnUu6kUXTQAWPuZY8i+ticFJbzfqaetLA6KR/rO6Ew/HuYD+bxifg==" + "resolved": "8.0.1", + "contentHash": "fGLiCRLMYd00JYpClraLjJTNKLmMJPnqxMaiRzEBIIvevlzxz33mXy39Lkd48hu1G+N21S7QpaO5ZzKsI6FRuA==" }, - "Microsoft.Extensions.FileProviders.Abstractions": { + "Microsoft.Extensions.Diagnostics.Abstractions": { "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "0pd4/fho0gC12rQswaGQxbU34jOS1TPS8lZPpkFCH68ppQjHNHYle9iRuHeev1LhrJ94YPvzcRd8UmIuFk23Qw==", + "resolved": "8.0.0", + "contentHash": "JHYCQG7HmugNYUhOl368g+NMxYE/N/AiclCYRNlgCY9eVyiBkOHMwK4x60RYMxv9EL3+rmj1mqHvdCiPpC+D4Q==", "dependencies": { - "Microsoft.Extensions.Primitives": "6.0.0" + "Microsoft.Extensions.DependencyInjection.Abstractions": "8.0.0", + "Microsoft.Extensions.Options": "8.0.0", + "System.Diagnostics.DiagnosticSource": "8.0.0" } }, - "Microsoft.Extensions.Primitives": { + "Microsoft.Extensions.FileProviders.Abstractions": { "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "9+PnzmQFfEFNR9J2aDTfJGGupShHjOuGw4VUv+JB044biSHrnmCIMD+mJHmb2H7YryrfBEXDurxQ47gJZdCKNQ==", + "resolved": "8.0.0", + "contentHash": "ZbaMlhJlpisjuWbvXr4LdAst/1XxH3vZ6A0BsgTphZ2L4PGuxRLz7Jr/S7mkAAnOn78Vu0fKhEgNF5JO3zfjqQ==", "dependencies": { - "System.Runtime.CompilerServices.Unsafe": "6.0.0" + "Microsoft.Extensions.Primitives": "8.0.0" } }, + "Microsoft.Extensions.Primitives": { + "type": "Transitive", + "resolved": "8.0.0", + "contentHash": "bXJEZrW9ny8vjMF1JV253WeLhpEVzFo1lyaZu1vQ4ZxWUlVvknZ/+ftFgVheLubb4eZPSwwxBeqS1JkCOjxd8g==" + }, "NetFabric.Hyperlinq.Abstractions": { "type": "Transitive", "resolved": "1.3.0", @@ -111,32 +118,36 @@ "resolved": "4.5.1", "contentHash": "Rw7ijyl1qqRS0YQD/WycNst8hUUMgrMH4FCn1nNm27M4VxchZ1js3fVjQaANHO5f3sN4isvP4a+Met9Y4YomAg==" }, + "System.Diagnostics.DiagnosticSource": { + "type": "Transitive", + "resolved": "8.0.0", + "contentHash": "c9xLpVz6PL9lp/djOWtk5KPDZq3cSYpmXoJQY524EOtuFl5z9ZtsotpsyrDW40U1DRnQSYvcPKEUV0X//u6gkQ==" + }, "System.Runtime.CompilerServices.Unsafe": { "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg==" + "resolved": "5.0.0", + "contentHash": "ZD9TMpsmYJLrxbbmdvhwt9YEgG5WntEnZ/d1eH8JBX9LBp+Ju8BSBhUGbZMNVHHomWo2KVImJhTDl2hIgw/6MA==" }, "nncase.codegen": { "type": "Project", "dependencies": { "Extension.Mathematics": "[1.2.12, )", "Nncase.Core": "[1.0.0, )", - "Nncase.IO": "[1.0.0, )", - "Razor.Templating.Core": "[1.9.0, )" + "Nncase.IO": "[1.0.0, )" } }, "nncase.core": { "type": "Project", "dependencies": { + "CommunityToolkit.HighPerformance": "[8.2.2, )", "DryIoc.dll": "[5.3.1, )", "GiGraph.Dot": "[2.0.0, )", - "Microsoft.Extensions.Hosting.Abstractions": "[6.0.0, )", - "Microsoft.Extensions.Logging.Abstractions": "[6.0.0, )", - "Microsoft.Extensions.Options": "[6.0.0, )", - "Microsoft.Toolkit.HighPerformance": "[7.1.1, )", + "Microsoft.Extensions.Hosting.Abstractions": "[8.0.0, )", + "Microsoft.Extensions.Logging.Abstractions": "[8.0.1, )", + "Microsoft.Extensions.Options": "[8.0.2, )", "NetFabric.Hyperlinq": "[3.0.0-beta48, )", "System.CommandLine": "[2.0.0-beta4.22272.1, )", - "System.Reactive": "[5.0.0, )" + "System.Reactive": "[6.0.0, )" } }, "nncase.egraph": { @@ -176,6 +187,12 @@ "Nncase.Graph": "[1.0.0, )" } }, + "CommunityToolkit.HighPerformance": { + "type": "CentralTransitive", + "requested": "[8.2.2, )", + "resolved": "8.2.2", + "contentHash": "+zIp8d3sbtYaRbM6hqDs4Ui/z34j7DcUmleruZlYLE4CVxXq+MO8XJyIs42vzeTYFX+k0Iq1dEbBUnQ4z/Gnrw==" + }, "DryIoc.dll": { "type": "CentralTransitive", "requested": "[5.3.1, )", @@ -216,37 +233,36 @@ }, "Microsoft.Extensions.Hosting.Abstractions": { "type": "CentralTransitive", - "requested": "[6.0.0, )", - "resolved": "6.0.0", - "contentHash": "GcT5l2CYXL6Sa27KCSh0TixsRfADUgth+ojQSD5EkzisZxmGFh7CwzkcYuGwvmXLjr27uWRNrJ2vuuEjMhU05Q==", + "requested": "[8.0.0, )", + "resolved": "8.0.0", + "contentHash": "AG7HWwVRdCHlaA++1oKDxLsXIBxmDpMPb3VoyOoAghEWnkUvEAdYQUwnV4jJbAaa/nMYNiEh5ByoLauZBEiovg==", "dependencies": { - "Microsoft.Extensions.Configuration.Abstractions": "6.0.0", - "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", - "Microsoft.Extensions.FileProviders.Abstractions": "6.0.0" + "Microsoft.Extensions.Configuration.Abstractions": "8.0.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "8.0.0", + "Microsoft.Extensions.Diagnostics.Abstractions": "8.0.0", + "Microsoft.Extensions.FileProviders.Abstractions": "8.0.0", + "Microsoft.Extensions.Logging.Abstractions": "8.0.0" } }, "Microsoft.Extensions.Logging.Abstractions": { "type": "CentralTransitive", - "requested": "[6.0.0, )", - "resolved": "6.0.0", - "contentHash": "/HggWBbTwy8TgebGSX5DBZ24ndhzi93sHUBDvP1IxbZD7FDokYzdAr6+vbWGjw2XAfR2EJ1sfKUotpjHnFWPxA==" + "requested": "[8.0.1, )", + "resolved": "8.0.1", + "contentHash": "RIFgaqoaINxkM2KTOw72dmilDmTrYA0ns2KW4lDz4gZ2+o6IQ894CzmdL3StM2oh7QQq44nCWiqKqc4qUI9Jmg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "8.0.1" + } }, "Microsoft.Extensions.Options": { "type": "CentralTransitive", - "requested": "[6.0.0, )", - "resolved": "6.0.0", - "contentHash": "dzXN0+V1AyjOe2xcJ86Qbo233KHuLEY0njf/P2Kw8SfJU+d45HNS2ctJdnEnrWbM9Ye2eFgaC5Mj9otRMU6IsQ==", + "requested": "[8.0.2, )", + "resolved": "8.0.2", + "contentHash": "dWGKvhFybsaZpGmzkGCbNNwBD1rVlWzrZKANLW/CcbFJpCEceMCGzT7zZwHOGBCbwM0SzBuceMj5HN1LKV1QqA==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0", - "Microsoft.Extensions.Primitives": "6.0.0" + "Microsoft.Extensions.DependencyInjection.Abstractions": "8.0.0", + "Microsoft.Extensions.Primitives": "8.0.0" } }, - "Microsoft.Toolkit.HighPerformance": { - "type": "CentralTransitive", - "requested": "[7.1.1, )", - "resolved": "7.1.1", - "contentHash": "TRnvDpZPXO30hTOtjfLw6Y9BtTKtTpzk9lefeh4RMCaUihWrVKQR454nYH4/mMJAh+LXqfAPyk0kfkJs0Amopw==" - }, "NetFabric.Hyperlinq": { "type": "CentralTransitive", "requested": "[3.0.0-beta48, )", @@ -267,12 +283,6 @@ "libortki": "0.0.2" } }, - "Razor.Templating.Core": { - "type": "CentralTransitive", - "requested": "[1.9.0, )", - "resolved": "1.9.0", - "contentHash": "eHNqkpmNcPr5rvP/8/FFkddnvzVMH0BSyrq03H0VLZK2r1GUe3RgIgsoIXnImHMIrBzUS8gOwV65MfRPdYRi6g==" - }, "Singulink.Collections.Weak": { "type": "CentralTransitive", "requested": "[1.0.2, )", @@ -287,9 +297,9 @@ }, "System.Reactive": { "type": "CentralTransitive", - "requested": "[5.0.0, )", - "resolved": "5.0.0", - "contentHash": "erBZjkQHWL9jpasCE/0qKAryzVBJFxGHVBAvgRN1bzM0q2s1S4oYREEEL0Vb+1kA/6BKb5FjUZMp5VXmy+gzkQ==" + "requested": "[6.0.0, )", + "resolved": "6.0.0", + "contentHash": "31kfaW4ZupZzPsI5PVe77VhnvFF55qgma7KZr/E0iFTs6fmdhhG8j0mgEx620iLTey1EynOkEfnyTjtNEpJzGw==" } } } diff --git a/src/Nncase.Cli/packages.lock.json b/src/Nncase.Cli/packages.lock.json index 734ec50218..968cf8bfe4 100644 --- a/src/Nncase.Cli/packages.lock.json +++ b/src/Nncase.Cli/packages.lock.json @@ -765,6 +765,13 @@ "Razor.Templating.Core": "[1.9.0, )" } }, + "nncase.modules.ncnn": { + "type": "Project", + "dependencies": { + "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Passes": "[1.0.0, )" + } + }, "nncase.modules.stackvm": { "type": "Project", "dependencies": { diff --git a/src/Nncase.Compiler/packages.lock.json b/src/Nncase.Compiler/packages.lock.json index d4ebd1cb2a..bf1efa248f 100644 --- a/src/Nncase.Compiler/packages.lock.json +++ b/src/Nncase.Compiler/packages.lock.json @@ -746,6 +746,13 @@ "Razor.Templating.Core": "[1.9.0, )" } }, + "nncase.modules.ncnn": { + "type": "Project", + "dependencies": { + "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Passes": "[1.0.0, )" + } + }, "nncase.modules.stackvm": { "type": "Project", "dependencies": { diff --git a/src/Nncase.Studio/packages.lock.json b/src/Nncase.Studio/packages.lock.json index fab340e44b..df84b3b59f 100644 --- a/src/Nncase.Studio/packages.lock.json +++ b/src/Nncase.Studio/packages.lock.json @@ -1016,6 +1016,13 @@ "Razor.Templating.Core": "[1.9.0, )" } }, + "nncase.modules.ncnn": { + "type": "Project", + "dependencies": { + "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Passes": "[1.0.0, )" + } + }, "nncase.modules.stackvm": { "type": "Project", "dependencies": { diff --git a/src/Nncase.Tests.TestFixture/packages.lock.json b/src/Nncase.Tests.TestFixture/packages.lock.json index 41e5f6199c..e12016f4a6 100644 --- a/src/Nncase.Tests.TestFixture/packages.lock.json +++ b/src/Nncase.Tests.TestFixture/packages.lock.json @@ -818,6 +818,13 @@ "Razor.Templating.Core": "[1.9.0, )" } }, + "nncase.modules.ncnn": { + "type": "Project", + "dependencies": { + "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Passes": "[1.0.0, )" + } + }, "nncase.modules.stackvm": { "type": "Project", "dependencies": { diff --git a/src/Nncase.Tests/packages.lock.json b/src/Nncase.Tests/packages.lock.json index 228298bb8f..dad914a726 100644 --- a/src/Nncase.Tests/packages.lock.json +++ b/src/Nncase.Tests/packages.lock.json @@ -934,6 +934,13 @@ "Razor.Templating.Core": "[1.9.0, )" } }, + "nncase.modules.ncnn": { + "type": "Project", + "dependencies": { + "Nncase.CodeGen": "[1.0.0, )", + "Nncase.Passes": "[1.0.0, )" + } + }, "nncase.modules.stackvm": { "type": "Project", "dependencies": { From 8c62dee5d1e9bad1c5dd269d455bf607cf5f4e49 Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 7 Jun 2024 15:35:14 +0800 Subject: [PATCH 130/132] fix Syntax format --- modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs | 2 +- modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerMatmul.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs index 96ae1c982c..adef2362b8 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerBinary.cs @@ -158,7 +158,7 @@ private bool HasGap(List aShape, List bShape) bShape.RemoveAt(0); } - b = new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputA }, op, 2, constB.ToArray(), bShape.ToArray()), newInputA), newA); + b = new Call(new Fusion("ncnn", NcnnBinary(new Expr[] { newInputA }, op, 2, constB.ToArray(), bShape.ToArray()), new[] { newInputA }), newA); } else { diff --git a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerMatmul.cs b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerMatmul.cs index 2b413b8fe4..5235a913a5 100644 --- a/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerMatmul.cs +++ b/modules/Nncase.Modules.Ncnn/Passes/Rules/Ncnn/LowerMatmul.cs @@ -47,7 +47,7 @@ public partial class LowerMatmul : RewriteRule var newInputA = new Var(inputA.CheckedType); var constB = ((TensorConst)inputB).Value; var constShape = inputB.CheckedShape.ToValueArray(); - return new Call(new Fusion("ncnn", NcnnMatMul(new Expr[] { newInputA }, 2, constB.ToArray(), constShape), newInputA), inputA); + return new Call(new Fusion("ncnn", NcnnMatMul(new Expr[] { newInputA }, 2, constB.ToArray(), constShape), new[] { newInputA }), inputA); } { From 9b0610ec95688d1c768d8031f803321480cb923b Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 7 Jun 2024 15:36:01 +0800 Subject: [PATCH 131/132] divide bin data --- .../CodeGen/Ncnn/NcnnEmitter.cs | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index b0b5713423..eb8182effc 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -17,6 +17,7 @@ namespace Nncase.CodeGen.Ncnn; internal class NcnnEmitter { + private static long _s_pos; private readonly NcnnModel _model; private readonly BinaryWriter _binWriter; private readonly List? _rData; @@ -41,12 +42,27 @@ public void SaveParam(Stream paramStream) public void SaveBin(string dumpPath, uint id) { - using (var fileStream = File.Create(Path.Join(dumpPath, $"/ncnn_{id}.bin"))) + if (id == 0) { - _binWriter.BaseStream.Seek(0, SeekOrigin.Begin); - _binWriter.BaseStream.CopyTo(fileStream); + _s_pos = 0; + } - _binWriter.BaseStream.SetLength(0); + using (var fileStream = File.Create(Path.Join(dumpPath, $"/ncnn_{id}.bin"))) + { + long now = _binWriter.BaseStream.Position; + if (now != 0 && now != _s_pos) + { + using (var tempStream = new MemoryStream()) + { + _binWriter.BaseStream.Position = _s_pos; // 重置位置到新内容的开始 + Console.WriteLine($"now - _s_pos : {now} - {_s_pos}"); + _binWriter.BaseStream.CopyTo(tempStream, (int)(now - _s_pos)); + tempStream.Position = 0; + tempStream.CopyTo(fileStream); + } + + _s_pos = _binWriter.BaseStream.Position; + } } } From 7fd4286bbd673fffc063560439212cb22421782e Mon Sep 17 00:00:00 2001 From: Curio Yang Date: Fri, 7 Jun 2024 15:37:39 +0800 Subject: [PATCH 132/132] add condition for dump ncnn param/bin --- .../CodeGen/Ncnn/NcnnEmitter.cs | 1 - .../CodeGen/Ncnn/NcnnFunctionBuilder.cs | 19 +++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs index eb8182effc..47f24950a5 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnEmitter.cs @@ -55,7 +55,6 @@ public void SaveBin(string dumpPath, uint id) using (var tempStream = new MemoryStream()) { _binWriter.BaseStream.Position = _s_pos; // 重置位置到新内容的开始 - Console.WriteLine($"now - _s_pos : {now} - {_s_pos}"); _binWriter.BaseStream.CopyTo(tempStream, (int)(now - _s_pos)); tempStream.Position = 0; tempStream.CopyTo(fileStream); diff --git a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs index c33d4f93d5..a697a077eb 100644 --- a/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs +++ b/modules/Nncase.Modules.Ncnn/CodeGen/Ncnn/NcnnFunctionBuilder.cs @@ -49,6 +49,8 @@ protected override void Compile(BaseFunction callable) protected override void WriteText() { _emitter.SaveParam(TextWriter.BaseStream); + + // dump param and bin. string dumpPath = Path.Join(DumpScope.Current.Directory, "ncnn_param_dir"); if (!Directory.Exists(dumpPath)) { @@ -56,6 +58,7 @@ protected override void WriteText() } else if (Id == 0) { + // clear dir before single case. foreach (string filePath in Directory.GetFiles(dumpPath, "*", SearchOption.AllDirectories)) { File.SetAttributes(filePath, FileAttributes.Normal); // 移除所有特殊属性以便删除 @@ -63,14 +66,18 @@ protected override void WriteText() } } - // if (DumpScope.Current.IsEnabled(DumpFlags.CodeGen)) - using (var fileStream = File.Create(Path.Join(dumpPath, $"ncnn_{Id}.param"))) + if (DumpScope.Current.IsEnabled(DumpFlags.CodeGen)) { - TextWriter.BaseStream.Seek(0, SeekOrigin.Begin); - TextWriter.BaseStream.CopyTo(fileStream); - } + // dump param. + using (var fileStream = File.Create(Path.Join(dumpPath, $"ncnn_{Id}.param"))) + { + TextWriter.BaseStream.Seek(0, SeekOrigin.Begin); + TextWriter.BaseStream.CopyTo(fileStream); + } - _emitter.SaveBin(dumpPath, Id); + // dump bin. + _emitter.SaveBin(dumpPath, Id); + } } private class CodeGenVisitor : ExprVisitor