diff --git a/ALL.sln b/ALL.sln index cdfe3282919..a0d80166361 100644 --- a/ALL.sln +++ b/ALL.sln @@ -153,8 +153,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Scripts", "Scripts", "{0AB2 Externals\winimerge\ExpandEnvironmenStrings.js = Externals\winimerge\ExpandEnvironmenStrings.js EndProjectSection EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "SampleStatic", "Externals\crystaledit\Sample\SampleStatic.vcxproj", "{C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}" -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "WinIMerge", "WinIMerge", "{0B7E421E-A8CA-444C-B650-D1D7F1D55688}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Frhed", "Frhed", "{97E9C1F1-F87E-4BE6-A66C-6B97DCD80873}" @@ -506,6 +504,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Translations", "Translation EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CrashLogger", "Src\CrashLogger\CrashLogger.vcxitems", "{CDEDDAC4-79B9-4A34-A564-8019F60D241F}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tree-sitter", "Externals\crystaledit\editlib\tree-sitter\tree-sitter.vcxitems", "{A3E7E04F-5D2B-4F3A-B725-8F5C4B1E9D3C}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "SampleStatic", "Externals\crystaledit\Sample\SampleStatic.vcxproj", "{C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|ARM = Debug|ARM @@ -1137,26 +1139,6 @@ Global {49A7104F-08CD-48B1-99E6-6A7C3FFB0EEC}.Test|ARM64.ActiveCfg = Debug|ARM64 {49A7104F-08CD-48B1-99E6-6A7C3FFB0EEC}.Test|x64.ActiveCfg = Debug|x64 {49A7104F-08CD-48B1-99E6-6A7C3FFB0EEC}.Test|x86.ActiveCfg = Debug|Win32 - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Debug|ARM.ActiveCfg = Debug|ARM - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Debug|ARM.Build.0 = Debug|ARM - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Debug|ARM64.ActiveCfg = Debug|ARM64 - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Debug|ARM64.Build.0 = Debug|ARM64 - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Debug|x64.ActiveCfg = Debug|x64 - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Debug|x64.Build.0 = Debug|x64 - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Debug|x86.ActiveCfg = Debug|Win32 - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Debug|x86.Build.0 = Debug|Win32 - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Release|ARM.ActiveCfg = Release|ARM - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Release|ARM.Build.0 = Release|ARM - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Release|ARM64.ActiveCfg = Release|ARM64 - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Release|ARM64.Build.0 = Release|ARM64 - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Release|x64.ActiveCfg = Release|x64 - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Release|x64.Build.0 = Release|x64 - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Release|x86.ActiveCfg = Release|Win32 - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Release|x86.Build.0 = Release|Win32 - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Test|ARM.ActiveCfg = Debug|ARM - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Test|ARM64.ActiveCfg = Debug|ARM64 - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Test|x64.ActiveCfg = Debug|x64 - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Test|x86.ActiveCfg = Debug|Win32 {139D8239-98B4-4B29-A199-B27CDB8B4708}.Debug|ARM.ActiveCfg = Debug|ARM {139D8239-98B4-4B29-A199-B27CDB8B4708}.Debug|ARM.Build.0 = Debug|ARM {139D8239-98B4-4B29-A199-B27CDB8B4708}.Debug|ARM64.ActiveCfg = Debug|ARM64 @@ -1389,6 +1371,30 @@ Global {40AF6A7D-B4E7-4919-B38A-75F625E36AF5}.Test|x64.Build.0 = Debug|x64 {40AF6A7D-B4E7-4919-B38A-75F625E36AF5}.Test|x86.ActiveCfg = Debug|Win32 {40AF6A7D-B4E7-4919-B38A-75F625E36AF5}.Test|x86.Build.0 = Debug|Win32 + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Debug|ARM.ActiveCfg = Debug|ARM + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Debug|ARM.Build.0 = Debug|ARM + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Debug|ARM64.Build.0 = Debug|ARM64 + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Debug|x64.ActiveCfg = Debug|x64 + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Debug|x64.Build.0 = Debug|x64 + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Debug|x86.ActiveCfg = Debug|Win32 + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Debug|x86.Build.0 = Debug|Win32 + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Release|ARM.ActiveCfg = Release|ARM + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Release|ARM.Build.0 = Release|ARM + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Release|ARM64.ActiveCfg = Release|ARM64 + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Release|ARM64.Build.0 = Release|ARM64 + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Release|x64.ActiveCfg = Release|x64 + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Release|x64.Build.0 = Release|x64 + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Release|x86.ActiveCfg = Release|Win32 + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Release|x86.Build.0 = Release|Win32 + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Test|ARM.ActiveCfg = Debug|ARM + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Test|ARM.Build.0 = Debug|ARM + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Test|ARM64.ActiveCfg = Debug|ARM64 + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Test|ARM64.Build.0 = Debug|ARM64 + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Test|x64.ActiveCfg = Release|x64 + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Test|x64.Build.0 = Release|x64 + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Test|x86.ActiveCfg = Debug|Win32 + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}.Test|x86.Build.0 = Debug|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1427,7 +1433,6 @@ Global {49A7104F-08CD-48B1-99E6-6A7C3FFB0EEC} = {0B7E421E-A8CA-444C-B650-D1D7F1D55688} {A9E83C9D-8AD6-4C87-888D-3F91B5656CD1} = {0B7E421E-A8CA-444C-B650-D1D7F1D55688} {0AB212AD-1F11-4BAB-BAD5-8BFC4435F453} = {A9E83C9D-8AD6-4C87-888D-3F91B5656CD1} - {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD} = {6BBF0DEA-C0B8-4B73-B540-3BF8297B49B4} {0B7E421E-A8CA-444C-B650-D1D7F1D55688} = {CE514278-A13F-4F6A-93EB-5653410AC214} {97E9C1F1-F87E-4BE6-A66C-6B97DCD80873} = {CE514278-A13F-4F6A-93EB-5653410AC214} {6BBF0DEA-C0B8-4B73-B540-3BF8297B49B4} = {CE514278-A13F-4F6A-93EB-5653410AC214} @@ -1476,6 +1481,8 @@ Global {DAFD8D9E-6509-4482-A533-44C843C3BCC7} = {220B870C-D051-463E-997B-8C392081EE15} {40AF6A7D-B4E7-4919-B38A-75F625E36AF5} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8} {02828B8D-E4B9-41FE-84C4-D5CDA7761627} = {5360DC20-6552-4F74-8D4D-BF88B23E0B3A} + {A3E7E04F-5D2B-4F3A-B725-8F5C4B1E9D3C} = {6BBF0DEA-C0B8-4B73-B540-3BF8297B49B4} + {C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD} = {6BBF0DEA-C0B8-4B73-B540-3BF8297B49B4} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {CC2E4F75-FADC-4F44-BD62-47A321828081} @@ -1501,6 +1508,7 @@ Global Externals\googletest\googletest\googletest.vcxitems*{9ee35458-b145-444f-92b7-27ff72112c42}*SharedItemsImports = 9 Externals\crystaledit\editlib\editlib.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 Externals\crystaledit\editlib\editlibparsers.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 + Externals\crystaledit\editlib\tree-sitter\tree-sitter.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 Externals\poco\Foundation\Foundation.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 Externals\poco\XML\XML.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 Externals\xdiff\xdiff.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 @@ -1508,6 +1516,7 @@ Global Src\CrashLogger\CrashLogger.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 Src\diffutils\diffutils.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 Src\FilterEngine\FilterEngine.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 + Externals\crystaledit\editlib\tree-sitter\tree-sitter.vcxitems*{a3e7e04f-5d2b-4f3a-b725-8f5c4b1e9d3c}*SharedItemsImports = 9 Plugins\src_VCPP\Common\Common.vcxitems*{a644fba4-d76e-4500-b4b7-04d7a245359a}*SharedItemsImports = 4 Plugins\src_VCPP\Common\Common.vcxitems*{aa88b46e-b2e2-4b03-8cd5-1e9d60db6ab2}*SharedItemsImports = 4 Externals\crystaledit\editlib\editlibparsers.vcxitems*{ab827c6b-5116-408f-b453-e2075e9b73b4}*SharedItemsImports = 4 @@ -1523,6 +1532,7 @@ Global Plugins\src_VCPP\Common\Common.vcxitems*{bd0c5fe1-8457-49c2-8801-0c99a6e6cc03}*SharedItemsImports = 4 Externals\crystaledit\editlib\editlib.vcxitems*{c347d6ae-7a2b-4ed0-97ad-2595e1c5d7dd}*SharedItemsImports = 4 Externals\crystaledit\editlib\editlibparsers.vcxitems*{c347d6ae-7a2b-4ed0-97ad-2595e1c5d7dd}*SharedItemsImports = 4 + Externals\crystaledit\editlib\tree-sitter\tree-sitter.vcxitems*{c347d6ae-7a2b-4ed0-97ad-2595e1c5d7dd}*SharedItemsImports = 4 Src\CrashLogger\CrashLogger.vcxitems*{cdeddac4-79b9-4a34-a564-8019f60d241f}*SharedItemsImports = 9 Externals\poco\XML\XML.vcxitems*{dafd8d9e-6509-4482-a533-44c843c3bcc7}*SharedItemsImports = 9 Src\diffutils\diffutils.vcxitems*{fc3b9df3-2854-4264-ab4b-ee8c43982513}*SharedItemsImports = 9 diff --git a/ALL.vs2017.sln b/ALL.vs2017.sln index 0b09fa68fe6..771fa6ce0e8 100644 --- a/ALL.vs2017.sln +++ b/ALL.vs2017.sln @@ -153,8 +153,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Scripts", "Scripts", "{0AB2 Externals\winimerge\ExpandEnvironmenStrings.js = Externals\winimerge\ExpandEnvironmenStrings.js EndProjectSection EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "SampleStatic", "Externals\crystaledit\Sample\SampleStatic.vcxproj", "{C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}" -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "WinIMerge", "WinIMerge", "{0B7E421E-A8CA-444C-B650-D1D7F1D55688}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Frhed", "Frhed", "{97E9C1F1-F87E-4BE6-A66C-6B97DCD80873}" @@ -437,6 +435,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Manual", "Manual", "{7DF650 Translations\Docs\Manual\English.pot = Translations\Docs\Manual\English.pot Translations\Docs\Manual\French.po = Translations\Docs\Manual\French.po Translations\Docs\Manual\Hebrew.po = Translations\Docs\Manual\Hebrew.po + Translations\Docs\Manual\Italian.po = Translations\Docs\Manual\Italian.po Translations\Docs\Manual\Japanese.po = Translations\Docs\Manual\Japanese.po EndProjectSection EndProject @@ -490,6 +489,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Translations", "Translation EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CrashLogger", "Src\CrashLogger\CrashLogger.vcxitems", "{CDEDDAC4-79B9-4A34-A564-8019F60D241F}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tree-sitter", "Externals\crystaledit\editlib\tree-sitter\tree-sitter.vcxitems", "{A3E7E04F-5D2B-4F3A-B725-8F5C4B1E9D3C}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "SampleStatic", "Externals\crystaledit\Sample\SampleStatic.vcxproj", "{C347D6AE-7A2B-4ED0-97AD-2595E1C5D7DD}" +EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution Plugins\src_VCPP\Common\Common.vcxitems*{098b241d-63af-4683-baf8-6da57fd8fe75}*SharedItemsImports = 4 @@ -514,11 +517,13 @@ Global Externals\crystaledit\editlib\editlibparsers.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 Externals\poco\Foundation\Foundation.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 Externals\poco\XML\XML.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 + Externals\crystaledit\editlib\tree-sitter\tree-sitter.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 Externals\xdiff\xdiff.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 Src\CompareEngines\CompareEngines.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 Src\CrashLogger\CrashLogger.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 Src\diffutils\diffutils.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 Src\FilterEngine\FilterEngine.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 + Externals\crystaledit\editlib\tree-sitter\tree-sitter.vcxitems*{a3e7e04f-5d2b-4f3a-b725-8f5c4b1e9d3c}*SharedItemsImports = 9 Plugins\src_VCPP\Common\Common.vcxitems*{a644fba4-d76e-4500-b4b7-04d7a245359a}*SharedItemsImports = 4 Plugins\src_VCPP\Common\Common.vcxitems*{aa88b46e-b2e2-4b03-8cd5-1e9d60db6ab2}*SharedItemsImports = 4 Externals\crystaledit\editlib\editlibparsers.vcxitems*{ab827c6b-5116-408f-b453-e2075e9b73b4}*SharedItemsImports = 4 @@ -534,6 +539,7 @@ Global Plugins\src_VCPP\Common\Common.vcxitems*{bd0c5fe1-8457-49c2-8801-0c99a6e6cc03}*SharedItemsImports = 4 Externals\crystaledit\editlib\editlib.vcxitems*{c347d6ae-7a2b-4ed0-97ad-2595e1c5d7dd}*SharedItemsImports = 4 Externals\crystaledit\editlib\editlibparsers.vcxitems*{c347d6ae-7a2b-4ed0-97ad-2595e1c5d7dd}*SharedItemsImports = 4 + Externals\crystaledit\editlib\tree-sitter\tree-sitter.vcxitems*{c347d6ae-7a2b-4ed0-97ad-2595e1c5d7dd}*SharedItemsImports = 4 Src\CrashLogger\CrashLogger.vcxitems*{cdeddac4-79b9-4a34-a564-8019f60d241f}*SharedItemsImports = 9 Externals\poco\XML\XML.vcxitems*{dafd8d9e-6509-4482-a533-44c843c3bcc7}*SharedItemsImports = 9 Src\diffutils\diffutils.vcxitems*{fc3b9df3-2854-4264-ab4b-ee8c43982513}*SharedItemsImports = 9 @@ -1423,6 +1429,7 @@ Global {B30837C4-F99F-46F9-ADFB-5FAEFB1D2FBB} = {220B870C-D051-463E-997B-8C392081EE15} {DAFD8D9E-6509-4482-A533-44C843C3BCC7} = {220B870C-D051-463E-997B-8C392081EE15} {02828B8D-E4B9-41FE-84C4-D5CDA7761627} = {5360DC20-6552-4F74-8D4D-BF88B23E0B3A} + {A3E7E04F-5D2B-4F3A-B725-8F5C4B1E9D3C} = {6BBF0DEA-C0B8-4B73-B540-3BF8297B49B4} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {CC2E4F75-FADC-4F44-BD62-47A321828081} diff --git a/Docs/Manual/Italian/Filters.xml b/Docs/Manual/Italian/Filters.xml index 67bf2545950..04d2a15952c 100644 --- a/Docs/Manual/Italian/Filters.xml +++ b/Docs/Manual/Italian/Filters.xml @@ -591,7 +591,7 @@ I seguenti attributi sono progettati per essere usati solo nei filtri i prefissi e:, e!:, fe:, fe!:, de: o de!: in il campo -Filtro). Non devono essere usati nelle regole di inclusione/esclusione del +Filtro). Non devono essere uszati nelle regole di inclusione/esclusione del filtro file (.flt), poiché richiedono che il confronto venga prima completato e potrebbero causare comportamenti imprevisti. diff --git a/Externals/crystaledit/Sample/SampleStatic.vcxproj b/Externals/crystaledit/Sample/SampleStatic.vcxproj index 31e7032a148..ee108d31f76 100644 --- a/Externals/crystaledit/Sample/SampleStatic.vcxproj +++ b/Externals/crystaledit/Sample/SampleStatic.vcxproj @@ -127,6 +127,7 @@ + diff --git a/Externals/crystaledit/editlib/TreeSitterParser.cpp b/Externals/crystaledit/editlib/TreeSitterParser.cpp new file mode 100644 index 00000000000..5238e9130bd --- /dev/null +++ b/Externals/crystaledit/editlib/TreeSitterParser.cpp @@ -0,0 +1,2115 @@ +//////////////////////////////////////////////////////////////////////////// +// File: TreeSitterParser.cpp +// Version: 1.0.1 +// Created: 2026-03-26 +// +// Tree-sitter based syntax highlighting bridge for CrystalEdit. +// +// SPDX-License-Identifier: GPL-2.0-or-later +//////////////////////////////////////////////////////////////////////////// + +#include "StdAfx.h" +#include "TreeSitterParser.h" +#include "ccrystaltextbuffer.h" + +#include + +#include +#include +#include +#include +#include +#include + +#ifdef _DEBUG +#define new DEBUG_NEW +#endif + +namespace +{ +bool HasCapturePrefix(const std::string& captureName, const char* prefix) +{ + const size_t prefixLen = strlen(prefix); + return captureName == prefix || + (captureName.size() > prefixLen && + captureName.compare(0, prefixLen, prefix) == 0 && + captureName[prefixLen] == '.'); +} + +int CountCaptureSegments(const std::string& captureName) +{ + return static_cast(std::count(captureName.begin(), captureName.end(), '.')) + 1; +} + +int MakeCapturePriority(const std::string& captureName, uint32_t startByte, uint32_t endByte) +{ + const uint32_t span = (endByte > startByte) ? (endByte - startByte) : 0; + const int specificityScore = CountCaptureSegments(captureName) * 200000; + const int spanScore = 100000 - static_cast(std::min(span, 100000)); + return specificityScore + spanScore; +} +} + +// ============================================================================ +// CTreeSitterColorMap +// ============================================================================ + +CTreeSitterColorMap::CTreeSitterColorMap() +{ + // Map standard tree-sitter highlight capture names to WinMerge COLORINDEX. + // + // Tree-sitter highlight queries use capture names like @keyword, @string, + // @comment, etc. These names come from nvim-treesitter conventions. + // We collapse them into WinMerge's available color categories. + + // Keywords: control flow, storage, operators as keywords + m_map["keyword"] = COLORINDEX_KEYWORD; + m_map["keyword.function"] = COLORINDEX_KEYWORD; + m_map["keyword.operator"] = COLORINDEX_KEYWORD; + m_map["keyword.import"] = COLORINDEX_KEYWORD; + m_map["keyword.type"] = COLORINDEX_KEYWORD; + m_map["keyword.modifier"] = COLORINDEX_KEYWORD; + m_map["keyword.repeat"] = COLORINDEX_KEYWORD; + m_map["keyword.return"] = COLORINDEX_KEYWORD; + m_map["keyword.conditional"] = COLORINDEX_KEYWORD; + m_map["keyword.exception"] = COLORINDEX_KEYWORD; + m_map["keyword.directive"] = COLORINDEX_PREPROCESSOR; + m_map["keyword.coroutine"] = COLORINDEX_KEYWORD; + m_map["include"] = COLORINDEX_KEYWORD; + m_map["repeat"] = COLORINDEX_KEYWORD; + m_map["conditional"] = COLORINDEX_KEYWORD; + m_map["exception"] = COLORINDEX_KEYWORD; + + // Functions + m_map["function"] = COLORINDEX_FUNCNAME; + m_map["function.call"] = COLORINDEX_FUNCNAME; + m_map["function.builtin"] = COLORINDEX_FUNCNAME; + m_map["function.macro"] = COLORINDEX_FUNCNAME; + m_map["method"] = COLORINDEX_FUNCNAME; + m_map["method.call"] = COLORINDEX_FUNCNAME; + m_map["constructor"] = COLORINDEX_FUNCNAME; + + // Comments + m_map["comment"] = COLORINDEX_COMMENT; + m_map["comment.documentation"] = COLORINDEX_COMMENT; + + // Strings + m_map["string"] = COLORINDEX_STRING; + m_map["string.documentation"] = COLORINDEX_STRING; + m_map["string.regex"] = COLORINDEX_STRING; + m_map["string.escape"] = COLORINDEX_STRING; + m_map["string.special"] = COLORINDEX_STRING; + m_map["character"] = COLORINDEX_STRING; + m_map["character.special"] = COLORINDEX_STRING; + + // Numbers + m_map["number"] = COLORINDEX_NUMBER; + m_map["number.float"] = COLORINDEX_NUMBER; + m_map["float"] = COLORINDEX_NUMBER; + m_map["boolean"] = COLORINDEX_NUMBER; + + // Operators and punctuation + m_map["operator"] = COLORINDEX_OPERATOR; + m_map["punctuation"] = COLORINDEX_OPERATOR; + m_map["punctuation.bracket"] = COLORINDEX_OPERATOR; + m_map["punctuation.delimiter"] = COLORINDEX_OPERATOR; + m_map["punctuation.special"] = COLORINDEX_OPERATOR; + + // Preprocessor / attributes + m_map["preproc"] = COLORINDEX_PREPROCESSOR; + m_map["define"] = COLORINDEX_PREPROCESSOR; + m_map["attribute"] = COLORINDEX_PREPROCESSOR; + m_map["attribute.builtin"] = COLORINDEX_PREPROCESSOR; + + // Types -> USER1 (types are important in F# for merge understanding) + m_map["type"] = COLORINDEX_USER1; + m_map["type.builtin"] = COLORINDEX_USER1; + m_map["type.definition"] = COLORINDEX_USER1; + m_map["type.qualifier"] = COLORINDEX_USER1; + m_map["storageclass"] = COLORINDEX_USER1; + + // Variables / properties / modules -> USER2 or NORMALTEXT + m_map["variable"] = COLORINDEX_NORMALTEXT; + m_map["variable.builtin"] = COLORINDEX_USER2; + m_map["variable.parameter"] = COLORINDEX_NORMALTEXT; + m_map["variable.member"] = COLORINDEX_USER2; + m_map["property"] = COLORINDEX_USER2; + m_map["field"] = COLORINDEX_USER2; + m_map["constant"] = COLORINDEX_USER2; + m_map["constant.builtin"] = COLORINDEX_USER2; + m_map["constant.macro"] = COLORINDEX_USER2; + m_map["module"] = COLORINDEX_USER1; + m_map["namespace"] = COLORINDEX_USER1; + m_map["label"] = COLORINDEX_USER2; + m_map["tag"] = COLORINDEX_KEYWORD; + m_map["tag.attribute"] = COLORINDEX_USER2; + m_map["tag.delimiter"] = COLORINDEX_OPERATOR; +} + +int CTreeSitterColorMap::MapCapture(const std::string& sCaptureName) const +{ + // Try exact match first + auto it = m_map.find(sCaptureName); + if (it != m_map.end()) + return it->second; + + // Try prefix match: e.g. "keyword.control.fsharp" -> "keyword" + std::string prefix = sCaptureName; + while (true) + { + auto pos = prefix.rfind('.'); + if (pos == std::string::npos) + break; + prefix = prefix.substr(0, pos); + it = m_map.find(prefix); + if (it != m_map.end()) + return it->second; + } + + return COLORINDEX_NORMALTEXT; +} + + +// ============================================================================ +// CTreeSitterLanguage +// ============================================================================ + +CTreeSitterLanguage::CTreeSitterLanguage() + : m_hDll(nullptr) + , m_pLanguage(nullptr) + , m_pHighlightQuery(nullptr) + , m_pLocalsQuery(nullptr) + , m_pTagsQuery(nullptr) + , m_pInjectionQuery(nullptr) +{ +} + +CTreeSitterLanguage::~CTreeSitterLanguage() +{ + if (m_pHighlightQuery) + ts_query_delete(m_pHighlightQuery); + if (m_pLocalsQuery) + ts_query_delete(m_pLocalsQuery); + if (m_pTagsQuery) + ts_query_delete(m_pTagsQuery); + if (m_pInjectionQuery) + ts_query_delete(m_pInjectionQuery); + if (m_hDll) + FreeLibrary(m_hDll); +} + +TSQuery* CTreeSitterLanguage::LoadQuery(const std::wstring& sPath) +{ + std::ifstream file(sPath, std::ios::binary); + if (!file.is_open()) + return nullptr; + + std::string source((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + + uint32_t errorOffset = 0; + TSQueryError errorType = TSQueryErrorNone; + TSQuery* pQuery = ts_query_new( + m_pLanguage, + source.c_str(), + static_cast(source.size()), + &errorOffset, + &errorType); + + if (errorType != TSQueryErrorNone) + { + // Query compilation failed - log but continue without this query + return nullptr; + } + + return pQuery; +} + +bool CTreeSitterLanguage::Load(const std::wstring& sGrammarDir, const std::wstring& sLanguage) +{ + m_sName = sLanguage; + + // Load the grammar DLL + // Expected name: tree-sitter-.dll (e.g. tree-sitter-fsharp.dll) + std::wstring sDllPath = sGrammarDir + L"\\tree-sitter-" + sLanguage + L".dll"; + m_hDll = LoadLibraryW(sDllPath.c_str()); + if (!m_hDll) + return false; + + // Get the language function + // Expected export: tree_sitter_ (e.g. tree_sitter_fsharp) + // Note: hyphens in language names are converted to underscores for the + // C export name (e.g. "c-sharp" -> "tree_sitter_c_sharp") + std::string sFuncName = "tree_sitter_"; + for (wchar_t ch : sLanguage) + { + if (ch == L'-') + sFuncName += '_'; + else + sFuncName += static_cast(ch); // ASCII language names only + } + + typedef const TSLanguage* (*TSLanguageFunc)(); + TSLanguageFunc pfnLanguage = reinterpret_cast( + GetProcAddress(m_hDll, sFuncName.c_str())); + if (!pfnLanguage) + { + FreeLibrary(m_hDll); + m_hDll = nullptr; + return false; + } + + m_pLanguage = pfnLanguage(); + if (!m_pLanguage) + { + FreeLibrary(m_hDll); + m_hDll = nullptr; + return false; + } + + // Load highlight query (.scm file) - required + // Expected name: -highlights.scm (e.g. fsharp-highlights.scm) + std::wstring sHighlightPath = sGrammarDir + L"\\" + sLanguage + L"-highlights.scm"; + m_pHighlightQuery = LoadQuery(sHighlightPath); + + // Load locals query (.scm file) - optional + // Expected name: -locals.scm (e.g. fsharp-locals.scm) + std::wstring sLocalsPath = sGrammarDir + L"\\" + sLanguage + L"-locals.scm"; + m_pLocalsQuery = LoadQuery(sLocalsPath); + + // Load tags query (.scm file) - optional + // Expected name: -tags.scm (e.g. python-tags.scm) + std::wstring sTagsPath = sGrammarDir + L"\\" + sLanguage + L"-tags.scm"; + m_pTagsQuery = LoadQuery(sTagsPath); + + // Load injection query (.scm file) - optional + // Expected name: -injections.scm (e.g. html-injections.scm) + std::wstring sInjectionPath = sGrammarDir + L"\\" + sLanguage + L"-injections.scm"; + m_pInjectionQuery = LoadQuery(sInjectionPath); + + return true; +} + + +// ============================================================================ +// CTreeSitterParser +// ============================================================================ + +CTreeSitterParser::CTreeSitterParser() + : m_pParser(nullptr) // Fix #2: lazy-init, don't call ts_parser_new() here + , m_pTree(nullptr) + , m_pLang(nullptr) + , m_bDirty(false) + , m_nLineCount(0) +{ +} + +CTreeSitterParser::~CTreeSitterParser() +{ + if (m_pTree) + ts_tree_delete(m_pTree); + if (m_pParser) + ts_parser_delete(m_pParser); +} + +/** + * @brief Lazily create the TSParser instance on first use. + * + * This avoids calling ts_parser_new() in the constructor, which would + * happen for every CMergeEditView even when tree-sitter isn't needed. + */ +void CTreeSitterParser::EnsureParser() +{ + if (!m_pParser) + m_pParser = ts_parser_new(); +} + +void CTreeSitterParser::SetLanguage(const CTreeSitterLanguage* pLang) +{ + m_pLang = pLang; + if (pLang && pLang->GetLanguage()) + { + EnsureParser(); + if (m_pParser) + ts_parser_set_language(m_pParser, pLang->GetLanguage()); + } + Invalidate(); +} + +void CTreeSitterParser::Invalidate() +{ + if (m_pTree) + { + ts_tree_delete(m_pTree); + m_pTree = nullptr; + } + m_lineBlocks.clear(); + m_lineUtf8.clear(); + m_documentText.clear(); + m_localScopes.clear(); + m_localRefHighlights.clear(); + m_pendingRefs.clear(); + m_tagDefs.clear(); + m_tagRefs.clear(); + m_nLineCount = 0; + m_nextBlockOrder = 0; + m_bDirty = false; +} + +void CTreeSitterParser::ParseDocument(const tchar_t* const* ppszLines, + const int* pnLineLengths, + int nLineCount) +{ + EnsureParser(); + if (!m_pParser || !m_pLang || !m_pLang->GetLanguage()) + return; + + // Build contiguous document text from line pointers. + // Tree-sitter requires UTF-8 input, so we convert from tchar_t (wchar_t on Windows). + m_documentText.clear(); + m_lineUtf8.clear(); + m_nLineCount = nLineCount; + + // Pre-calculate total size estimate + size_t totalEstimate = 0; + for (int i = 0; i < nLineCount; i++) + totalEstimate += static_cast(pnLineLengths[i]) * 3 + 1; // worst case UTF-8 + m_documentText.reserve(totalEstimate); + m_lineUtf8.resize(nLineCount); + + for (int i = 0; i < nLineCount; i++) + { + if (ppszLines[i] && pnLineLengths[i] > 0) + { +#ifdef _UNICODE + // Convert UTF-16 to UTF-8 + int nLen = WideCharToMultiByte(CP_UTF8, 0, + ppszLines[i], pnLineLengths[i], + nullptr, 0, nullptr, nullptr); + if (nLen > 0) + { + std::string lineUtf8(nLen, '\0'); + WideCharToMultiByte(CP_UTF8, 0, + ppszLines[i], pnLineLengths[i], + &lineUtf8[0], nLen, + nullptr, nullptr); + m_lineUtf8[i] = lineUtf8; + m_documentText.append(lineUtf8); + } +#else + m_lineUtf8[i].assign(ppszLines[i], pnLineLengths[i]); + m_documentText.append(ppszLines[i], pnLineLengths[i]); +#endif + } + if (i < nLineCount - 1) + m_documentText += '\n'; + } + + // Parse the document. + // Pass the old tree if available -- tree-sitter can reuse unchanged subtrees + // for faster re-parsing. (For full incremental support, ts_tree_edit() should + // be called on the old tree before re-parsing, but even without edit info + // tree-sitter benefits from having the previous tree as a reference.) + TSTree* pOldTree = m_pTree; + m_pTree = ts_parser_parse_string( + m_pParser, + pOldTree, + m_documentText.c_str(), + static_cast(m_documentText.size())); + + if (pOldTree) + ts_tree_delete(pOldTree); + + if (m_pTree) + { + m_nextBlockOrder = 0; + // 1. Run locals query first to build scope/def/ref information + RunLocalsQuery(); + // 2. Run tags query for same-file symbol definitions/references + RunTagsQuery(); + // 3. Run highlight query (uses locals info for scope-aware coloring) + RunHighlightQuery(); + // 4. Run injection query to handle embedded languages + RunInjectionQuery(); + BuildLineCache(nLineCount); + } + + // Cache is now fresh + m_bDirty = false; +} + +/** + * @brief Notify the parser of an edit for incremental reparsing. + * + * Reads the last UndoRecord from the buffer to get the edit position, + * then calls ts_tree_edit() on the existing tree. This allows tree-sitter + * to reuse unchanged subtrees during the next reparse, which is + * significantly faster for large documents. + * + * Falls back to a simple MarkDirty() if the tree or undo info is unavailable. + */ +void CTreeSitterParser::NotifyEdit(CCrystalTextBuffer* pBuf) +{ + m_bDirty = true; + + // If we don't have a tree, there's nothing to edit incrementally + if (!m_pTree || !pBuf || !pBuf->CanUndo()) + return; + + int nUndoPos = pBuf->GetUndoPosition(); + if (nUndoPos <= 0) + return; + + UndoRecord ur = pBuf->GetUndoRecord(nUndoPos - 1); + bool bInsert = (ur.m_dwFlags & UNDO_INSERT) != 0; + + // Convert CEPoint (char-based, line/col) to UTF-8 byte offsets. + // m_lineUtf8 holds the per-line UTF-8 from the previous parse. + // We need: + // start_byte: absolute byte offset of the edit start in old doc + // old_end_byte: absolute byte offset of the old content end + // new_end_byte: absolute byte offset of the new content end + + // Helper: compute absolute byte offset from (line, charPos) using old m_lineUtf8 + // Each line in m_documentText is followed by '\n' (except the last) + auto charPosToByteOffset = [this](int line, int charPos) -> uint32_t + { + uint32_t byteOffset = 0; + int nLines = static_cast(m_lineUtf8.size()); + + // Sum up all lines before 'line' + for (int i = 0; i < line && i < nLines; i++) + { + byteOffset += static_cast(m_lineUtf8[i].size()); + byteOffset += 1; // for '\n' separator + } + + // Add the byte offset within the target line + if (line >= 0 && line < nLines && charPos > 0) + { +#ifdef _UNICODE + const std::string& utf8Line = m_lineUtf8[line]; + // Convert charPos (UTF-16 units) to UTF-8 byte count + // We need the original line text to do this properly. + // Use the stored UTF-8 line and convert back to count bytes. + int nUtf16Len = MultiByteToWideChar(CP_UTF8, 0, + utf8Line.c_str(), static_cast(utf8Line.size()), + nullptr, 0); + if (charPos >= nUtf16Len) + { + byteOffset += static_cast(utf8Line.size()); + } + else + { + // Walk the UTF-8 bytes counting UTF-16 chars until we reach charPos + int utf16Count = 0; + uint32_t byteIdx = 0; + const uint8_t* p = reinterpret_cast(utf8Line.c_str()); + uint32_t lineLen = static_cast(utf8Line.size()); + while (byteIdx < lineLen && utf16Count < charPos) + { + uint8_t ch = p[byteIdx]; + uint32_t seqLen; + if (ch < 0x80) + seqLen = 1; + else if (ch < 0xE0) + seqLen = 2; + else if (ch < 0xF0) + seqLen = 3; + else + seqLen = 4; + + byteIdx += seqLen; + // A 4-byte UTF-8 sequence produces a surrogate pair (2 UTF-16 units) + utf16Count += (seqLen == 4) ? 2 : 1; + } + byteOffset += byteIdx; + } +#else + byteOffset += static_cast(charPos); +#endif + } + else if (line >= nLines && line > 0) + { + // Line is beyond our cached data -- use end of document + byteOffset = static_cast(m_documentText.size()); + } + + return byteOffset; + }; + + // Helper: compute TSPoint (row, column in bytes) from (line, charPos) + auto charPosToTSPoint = [this](int line, int charPos) -> TSPoint + { + TSPoint pt; + pt.row = static_cast(line); + pt.column = 0; + + if (charPos > 0 && line >= 0 && line < static_cast(m_lineUtf8.size())) + { + const std::string& utf8Line = m_lineUtf8[line]; +#ifdef _UNICODE + int nUtf16Len = MultiByteToWideChar(CP_UTF8, 0, + utf8Line.c_str(), static_cast(utf8Line.size()), + nullptr, 0); + if (charPos >= nUtf16Len) + { + pt.column = static_cast(utf8Line.size()); + } + else + { + int utf16Count = 0; + uint32_t byteIdx = 0; + const uint8_t* p = reinterpret_cast(utf8Line.c_str()); + uint32_t lineLen = static_cast(utf8Line.size()); + while (byteIdx < lineLen && utf16Count < charPos) + { + uint8_t ch = p[byteIdx]; + uint32_t seqLen; + if (ch < 0x80) + seqLen = 1; + else if (ch < 0xE0) + seqLen = 2; + else if (ch < 0xF0) + seqLen = 3; + else + seqLen = 4; + byteIdx += seqLen; + utf16Count += (seqLen == 4) ? 2 : 1; + } + pt.column = byteIdx; + } +#else + pt.column = static_cast(charPos); +#endif + } + return pt; + }; + + TSInputEdit edit; + memset(&edit, 0, sizeof(edit)); + + if (bInsert) + { + // Insert: old range is empty (start == old_end), new range is the inserted text + edit.start_byte = charPosToByteOffset(ur.m_ptStartPos.y, ur.m_ptStartPos.x); + edit.old_end_byte = edit.start_byte; + edit.start_point = charPosToTSPoint(ur.m_ptStartPos.y, ur.m_ptStartPos.x); + edit.old_end_point = edit.start_point; + + // For new_end, we need the end position after insert. + // The UndoRecord's m_ptEndPos gives us the end position in the *new* document. + // But our m_lineUtf8 is from the *old* document, so we can't use + // charPosToByteOffset for the end position directly. + // Instead, compute new_end_byte = start_byte + utf8_length_of_inserted_text. + // Normalize to LF-only: ParseDocument() concatenates lines with '\n' (no '\r'), + // so '\r' bytes must be excluded from all byte-offset calculations. + const tchar_t* pInsText = ur.GetText(); + size_t nInsLen = ur.GetTextLength(); +#ifdef _UNICODE + // Convert to UTF-8 and strip '\r' to match ParseDocument()'s representation. + int nRawUtf8Len = WideCharToMultiByte(CP_UTF8, 0, + pInsText, static_cast(nInsLen), + nullptr, 0, nullptr, nullptr); + std::string insUtf8; + if (nRawUtf8Len > 0) + { + insUtf8.resize(nRawUtf8Len); + WideCharToMultiByte(CP_UTF8, 0, pInsText, static_cast(nInsLen), + &insUtf8[0], nRawUtf8Len, nullptr, nullptr); + insUtf8.erase(std::remove(insUtf8.begin(), insUtf8.end(), '\r'), insUtf8.end()); + } + int nUtf8Len = static_cast(insUtf8.size()); + edit.new_end_byte = edit.start_byte + static_cast(nUtf8Len); +#else + // Exclude '\r' characters to match ParseDocument()'s LF-only representation. + int nCrCount = static_cast(std::count(pInsText, pInsText + nInsLen, static_cast('\r'))); + edit.new_end_byte = edit.start_byte + static_cast(nInsLen - nCrCount); +#endif + edit.new_end_point.row = static_cast(ur.m_ptEndPos.y); + // For the column, we can compute it from the text: count bytes after last newline + uint32_t lastNewlineBytes = 0; + bool foundNewline = false; +#ifdef _UNICODE + // Use the already-normalized UTF-8 string to find the last newline position. + if (nUtf8Len > 0) + { + auto lastNL = insUtf8.rfind('\n'); + if (lastNL != std::string::npos) + { + foundNewline = true; + lastNewlineBytes = static_cast(nUtf8Len - lastNL - 1); + } + } +#else + { + const char* pText = pInsText; + for (int i = static_cast(nInsLen) - 1; i >= 0; i--) + { + if (pText[i] == '\n') { foundNewline = true; lastNewlineBytes = nInsLen - i - 1; break; } + } + } +#endif + if (foundNewline) + { + edit.new_end_point.column = lastNewlineBytes; + } + else + { + // No newline in inserted text: column = start column + inserted byte length + edit.new_end_point.column = edit.start_point.column + + (edit.new_end_byte - edit.start_byte); + } + } + else + { + // Delete: old range is the deleted text, new range is empty (start == new_end) + edit.start_byte = charPosToByteOffset(ur.m_ptStartPos.y, ur.m_ptStartPos.x); + edit.start_point = charPosToTSPoint(ur.m_ptStartPos.y, ur.m_ptStartPos.x); + + // For old_end, we use the old document positions + edit.old_end_byte = charPosToByteOffset(ur.m_ptEndPos.y, ur.m_ptEndPos.x); + edit.old_end_point = charPosToTSPoint(ur.m_ptEndPos.y, ur.m_ptEndPos.x); + + // After deletion, the cursor is at start + edit.new_end_byte = edit.start_byte; + edit.new_end_point = edit.start_point; + } + + ts_tree_edit(m_pTree, &edit); +} + +/** + * @brief Ensure the document is parsed if the cache is dirty. + * + * Called lazily from ParseLine during the paint cycle. This means + * we reparse at most once per paint, not once per keystroke. + */ +void CTreeSitterParser::EnsureParsed(CCrystalTextBuffer* pBuffer) +{ + if (m_bDirty && m_pLang) + { + ParseFromBuffer(pBuffer); + // ParseFromBuffer sets m_bDirty = false via ParseDocument + } +} + +/** + * @brief Convert a UTF-8 byte offset within a line to a UTF-16 character position. + * + * Tree-sitter reports column positions as byte offsets in the UTF-8 text. + * WinMerge's TEXTBLOCK.m_nCharPos expects tchar_t (wchar_t) character indices. + * For pure ASCII, these are identical. For multi-byte UTF-8 / surrogate pairs, + * we need to walk the UTF-8 bytes and count the corresponding UTF-16 code units. + * + * @param nLine Zero-based line index. + * @param byteCol Byte offset within the line's UTF-8 representation. + * @return Character position (index into the wchar_t line). + */ +int CTreeSitterParser::Utf8ByteOffsetToCharPos(int nLine, uint32_t byteCol) const +{ + if (nLine < 0 || nLine >= static_cast(m_lineUtf8.size())) + return static_cast(byteCol); + + const std::string& utf8Line = m_lineUtf8[nLine]; + if (utf8Line.empty() || byteCol == 0) + return 0; + + // Clamp to line length + uint32_t maxByte = static_cast(utf8Line.size()); + if (byteCol > maxByte) + byteCol = maxByte; + +#ifdef _UNICODE + // Convert the prefix [0..byteCol) from UTF-8 to UTF-16 and count chars + int nChars = MultiByteToWideChar(CP_UTF8, 0, + utf8Line.c_str(), static_cast(byteCol), + nullptr, 0); + return nChars; +#else + return static_cast(byteCol); +#endif +} + +/** + * @brief Extract a #set! predicate property value from a query pattern. + * + * Tree-sitter predicates like (#set! injection.language "javascript") are + * encoded as sequences of TSQueryPredicateStep: + * [String "set!", String "injection.language", String "javascript", Done] + * + * @param pQuery The query containing the pattern. + * @param patternIndex The pattern index. + * @param key The property key to look for (e.g. "injection.language"). + * @return The property value, or empty string if not found. + */ +std::string CTreeSitterParser::GetSetProperty(const TSQuery* pQuery, + uint32_t patternIndex, + const std::string& key) +{ + uint32_t stepCount = 0; + const TSQueryPredicateStep* steps = + ts_query_predicates_for_pattern(pQuery, patternIndex, &stepCount); + if (!steps || stepCount == 0) + return std::string(); + + // Walk through predicate steps looking for: "set!" + for (uint32_t i = 0; i + 2 < stepCount; i++) + { + // Look for a string step containing "set!" + if (steps[i].type != TSQueryPredicateStepTypeString) + continue; + + uint32_t nameLen = 0; + const char* name = ts_query_string_value_for_id(pQuery, steps[i].value_id, &nameLen); + if (!name || std::string(name, nameLen) != "set!") + continue; + + // Next step should be the property key (string) + if (i + 1 >= stepCount || steps[i + 1].type != TSQueryPredicateStepTypeString) + continue; + + uint32_t keyLen = 0; + const char* keyStr = ts_query_string_value_for_id(pQuery, steps[i + 1].value_id, &keyLen); + if (!keyStr || std::string(keyStr, keyLen) != key) + { + // Skip to the Done sentinel for this predicate + while (i < stepCount && steps[i].type != TSQueryPredicateStepTypeDone) + i++; + continue; + } + + // Next step should be the property value (string) + if (i + 2 >= stepCount || steps[i + 2].type != TSQueryPredicateStepTypeString) + continue; + + uint32_t valLen = 0; + const char* valStr = ts_query_string_value_for_id(pQuery, steps[i + 2].value_id, &valLen); + if (valStr) + return std::string(valStr, valLen); + } + + return std::string(); +} + +void CTreeSitterParser::RunTagsQuery() +{ + m_tagDefs.clear(); + m_tagRefs.clear(); + + if (!m_pTree || !m_pLang || !m_pLang->GetTagsQuery()) + return; + + const TSQuery* pQuery = m_pLang->GetTagsQuery(); + TSNode rootNode = ts_tree_root_node(m_pTree); + + TSQueryCursor* pCursor = ts_query_cursor_new(); + if (!pCursor) + return; + + ts_query_cursor_exec(pCursor, pQuery, rootNode); + + TSQueryMatch match; + while (ts_query_cursor_next_match(pCursor, &match)) + { + std::string name; + uint32_t nameStart = 0; + uint32_t nameEnd = 0; + bool hasName = false; + + struct Range + { + uint32_t startByte; + uint32_t endByte; + }; + std::vector defs; + std::vector refs; + + for (uint16_t i = 0; i < match.capture_count; ++i) + { + TSQueryCapture capture = match.captures[i]; + TSNode node = capture.node; + + uint32_t captureNameLen = 0; + const char* captureName = ts_query_capture_name_for_id( + pQuery, capture.index, &captureNameLen); + std::string sCapture(captureName, captureNameLen); + + const uint32_t nodeStart = ts_node_start_byte(node); + const uint32_t nodeEnd = ts_node_end_byte(node); + + if (sCapture == "name") + { + if (nodeStart < m_documentText.size() && nodeEnd <= m_documentText.size()) + { + name = m_documentText.substr(nodeStart, nodeEnd - nodeStart); + nameStart = nodeStart; + nameEnd = nodeEnd; + hasName = true; + } + } + else if (HasCapturePrefix(sCapture, "definition")) + { + defs.push_back({ nodeStart, nodeEnd }); + } + else if (HasCapturePrefix(sCapture, "reference")) + { + refs.push_back({ nodeStart, nodeEnd }); + } + } + + if (!hasName) + continue; + + if (defs.empty() && !refs.empty()) + { + for (const auto& ref : refs) + { + const bool useNameRange = + nameStart >= ref.startByte && nameEnd <= ref.endByte; + m_tagRefs.push_back({ + name, + useNameRange ? nameStart : ref.startByte, + useNameRange ? nameEnd : ref.endByte, + }); + } + } + + for (const auto& def : defs) + { + const bool useNameRange = + nameStart >= def.startByte && nameEnd <= def.endByte; + m_tagDefs.push_back({ + name, + useNameRange ? nameStart : def.startByte, + useNameRange ? nameEnd : def.endByte, + }); + } + } + + ts_query_cursor_delete(pCursor); +} + +/** + * @brief Run the locals query to build scope/definition/reference information. + * + * Processes locals.scm captures: + * @local.scope - Defines a scope boundary + * @local.definition - Defines a local variable/symbol + * @local.reference - References a local variable/symbol + * + * The results are stored in m_localScopes and m_localRefHighlights, + * which are used by RunHighlightQuery to provide scope-aware coloring. + */ +void CTreeSitterParser::RunLocalsQuery() +{ + m_localScopes.clear(); + m_localRefHighlights.clear(); + + if (!m_pTree || !m_pLang || !m_pLang->GetLocalsQuery()) + return; + + const TSQuery* pQuery = m_pLang->GetLocalsQuery(); + TSNode rootNode = ts_tree_root_node(m_pTree); + + TSQueryCursor* pCursor = ts_query_cursor_new(); + if (!pCursor) + return; + + ts_query_cursor_exec(pCursor, pQuery, rootNode); + + // Temporary storage for references (resolved after all defs are collected) + std::vector references; + + TSQueryMatch match; + while (ts_query_cursor_next_match(pCursor, &match)) + { + for (uint16_t i = 0; i < match.capture_count; i++) + { + TSQueryCapture capture = match.captures[i]; + TSNode node = capture.node; + + uint32_t nameLen = 0; + const char* captureName = ts_query_capture_name_for_id( + pQuery, capture.index, &nameLen); + std::string sCapture(captureName, nameLen); + + uint32_t nodeStart = ts_node_start_byte(node); + uint32_t nodeEnd = ts_node_end_byte(node); + + if (HasCapturePrefix(sCapture, "local.scope")) + { + // Check for #set! local.scope-inherits predicate + bool inherits = true; + std::string inheritVal = GetSetProperty(pQuery, match.pattern_index, "local.scope-inherits"); + if (inheritVal == "false") + inherits = false; + + LocalScope scope; + scope.startByte = nodeStart; + scope.endByte = nodeEnd; + scope.inherits = inherits; + m_localScopes.push_back(scope); + } + else if (HasCapturePrefix(sCapture, "local.definition")) + { + // Extract the text of the definition node as the symbol name + if (nodeStart < m_documentText.size() && nodeEnd <= m_documentText.size()) + { + std::string defName = m_documentText.substr(nodeStart, nodeEnd - nodeStart); + + // Find the innermost enclosing scope and add this definition + LocalScope* pBestScope = nullptr; + for (auto& scope : m_localScopes) + { + if (nodeStart >= scope.startByte && nodeEnd <= scope.endByte) + { + if (!pBestScope || + (scope.endByte - scope.startByte) < (pBestScope->endByte - pBestScope->startByte)) + { + pBestScope = &scope; + } + } + } + if (pBestScope) + { + LocalDef def; + def.name = defName; + def.startByte = nodeStart; + def.endByte = nodeEnd; + def.highlight = -1; // Will be resolved during RunHighlightQuery + pBestScope->defs.push_back(def); + } + } + } + else if (HasCapturePrefix(sCapture, "local.reference")) + { + if (nodeStart < m_documentText.size() && nodeEnd <= m_documentText.size()) + { + PendingRef ref; + ref.name = m_documentText.substr(nodeStart, nodeEnd - nodeStart); + ref.startByte = nodeStart; + ref.endByte = nodeEnd; + ref.scopeStartByte = nodeStart; + references.push_back(ref); + } + } + } + } + + ts_query_cursor_delete(pCursor); + + // Sort scopes by start byte, then by size (smallest/innermost first for lookup) + std::sort(m_localScopes.begin(), m_localScopes.end(), + [](const LocalScope& a, const LocalScope& b) + { + if (a.startByte != b.startByte) + return a.startByte < b.startByte; + return (a.endByte - a.startByte) < (b.endByte - b.startByte); + }); + + // Store references for later resolution in RunHighlightQuery. + // We can't resolve them yet because definition highlights haven't been + // determined. Instead, store them and resolve after RunHighlightQuery + // has assigned highlights to definitions. + // + // Actually, we'll store the reference info and do a two-pass approach: + // RunHighlightQuery will first assign highlights to definition nodes, + // then we resolve references. + // + // For now, store references as pending. The key is (startByte << 32 | endByte). + // We'll store reference name -> node range for later lookup. + m_pendingRefs = std::move(references); +} + +/** + * @brief Run the highlight query against the parsed tree and collect token ranges. + * + * This produces per-line block arrays by walking all highlight query matches + * and mapping tree-sitter byte positions to WinMerge character positions. + * + * If locals information is available, definition nodes get their highlights + * recorded, and references are resolved to use the same highlight as their + * matching definition. + */ +void CTreeSitterParser::RunHighlightQuery() +{ + if (!m_pTree || !m_pLang || !m_pLang->GetHighlightQuery()) + return; + + const TSQuery* pQuery = m_pLang->GetHighlightQuery(); + TSNode rootNode = ts_tree_root_node(m_pTree); + + TSQueryCursor* pCursor = ts_query_cursor_new(); + if (!pCursor) + return; + + ts_query_cursor_exec(pCursor, pQuery, rootNode); + + // Temporary structure to collect all highlights + struct HighlightEntry + { + uint32_t startRow; + uint32_t startCol; // UTF-8 byte offset in line + uint32_t endRow; + uint32_t endCol; // UTF-8 byte offset in line + uint32_t startByte; + uint32_t endByte; + int colorIndex; + int priority; + uint32_t order; + }; + std::vector highlights; + + // Map from (startByte, endByte) to colorIndex for definition resolution. + // Key: (startByte << 32 | endByte). This assumes files < 4GB and that + // nodes with identical byte ranges should share the same highlight. + std::unordered_map nodeHighlightMap; + + TSQueryMatch match; + while (ts_query_cursor_next_match(pCursor, &match)) + { + for (uint16_t i = 0; i < match.capture_count; i++) + { + TSQueryCapture capture = match.captures[i]; + TSNode node = capture.node; + + uint32_t captureNameLen = 0; + const char* captureName = ts_query_capture_name_for_id( + pQuery, capture.index, &captureNameLen); + + std::string sName(captureName, captureNameLen); + int colorIndex = m_colorMap.MapCapture(sName); + + uint32_t nodeStartByte = ts_node_start_byte(node); + uint32_t nodeEndByte = ts_node_end_byte(node); + TSPoint startPoint = ts_node_start_point(node); + TSPoint endPoint = ts_node_end_point(node); + + HighlightEntry entry; + entry.startRow = startPoint.row; + entry.startCol = startPoint.column; + entry.endRow = endPoint.row; + entry.endCol = endPoint.column; + entry.startByte = nodeStartByte; + entry.endByte = nodeEndByte; + entry.colorIndex = colorIndex; + entry.priority = MakeCapturePriority(sName, nodeStartByte, nodeEndByte); + entry.order = NextBlockOrder(); + + highlights.push_back(entry); + + // Record this node's highlight for locals resolution + uint64_t nodeKey = (static_cast(nodeStartByte) << 32) | + static_cast(nodeEndByte); + nodeHighlightMap[nodeKey] = colorIndex; + } + } + + ts_query_cursor_delete(pCursor); + + // --- Locals resolution --- + // Now that we know the highlight for each node, assign highlights to + // definitions and resolve references. + if (!m_localScopes.empty()) + { + // Pass 1: Assign highlights to definitions based on their node's highlight + for (auto& scope : m_localScopes) + { + for (auto& def : scope.defs) + { + uint64_t defKey = (static_cast(def.startByte) << 32) | + static_cast(def.endByte); + auto it = nodeHighlightMap.find(defKey); + if (it != nodeHighlightMap.end()) + def.highlight = it->second; + } + } + + // Pass 2: Resolve references — find matching definition in enclosing scopes + for (const auto& ref : m_pendingRefs) + { + int resolvedHighlight = -1; + + // Search scopes from innermost to outermost + // Find all scopes that contain this reference + std::vector enclosingScopes; + for (const auto& scope : m_localScopes) + { + if (ref.startByte >= scope.startByte && ref.endByte <= scope.endByte) + enclosingScopes.push_back(&scope); + } + + // Sort by size (smallest = innermost first) + std::sort(enclosingScopes.begin(), enclosingScopes.end(), + [](const LocalScope* a, const LocalScope* b) + { + return (a->endByte - a->startByte) < (b->endByte - b->startByte); + }); + + // Search for a matching definition + for (const auto* pScope : enclosingScopes) + { + for (const auto& def : pScope->defs) + { + // Match by name and ensure the definition appears before the reference + if (def.name == ref.name && def.startByte <= ref.startByte && def.highlight >= 0) + { + resolvedHighlight = def.highlight; + break; + } + } + if (resolvedHighlight >= 0) + break; + // If this scope doesn't inherit, stop searching + if (!pScope->inherits) + break; + } + + if (resolvedHighlight >= 0) + { + uint64_t refKey = (static_cast(ref.startByte) << 32) | + static_cast(ref.endByte); + m_localRefHighlights[refKey] = resolvedHighlight; + } + } + + // Pass 3: Apply resolved reference highlights to the highlight entries + for (auto& h : highlights) + { + uint64_t key = (static_cast(h.startByte) << 32) | + static_cast(h.endByte); + auto it = m_localRefHighlights.find(key); + if (it != m_localRefHighlights.end()) + h.colorIndex = it->second; + } + } + + // Sort by start position (row, then column) + std::sort(highlights.begin(), highlights.end(), + [](const HighlightEntry& a, const HighlightEntry& b) + { + if (a.startRow != b.startRow) + return a.startRow < b.startRow; + return a.startCol < b.startCol; + }); + + // Build per-line block arrays + m_lineBlocks.clear(); + m_lineBlocks.resize(m_nLineCount); + + for (const auto& h : highlights) + { + // Handle tokens that span multiple lines (e.g. multi-line strings/comments) + for (uint32_t row = h.startRow; row <= h.endRow && row < static_cast(m_nLineCount); row++) + { + uint32_t byteCol = (row == h.startRow) ? h.startCol : 0; + + // Convert UTF-8 byte offset to UTF-16 character position + int charPos = Utf8ByteOffsetToCharPos(static_cast(row), byteCol); + + TreeSitterLineBlock block; + block.nCharPos = charPos; + block.nColorIndex = h.colorIndex; + block.nPriority = h.priority; + block.nOrder = h.order; + m_lineBlocks[row].push_back(block); + } + + // Emit a block at the end of the capture to restore normal color. + // This prevents the token's color from "bleeding" past its end. + if (h.endRow < static_cast(m_nLineCount)) + { + uint32_t endByteCol = h.endCol; + + // Convert UTF-8 byte offset at token end to UTF-16 character position + int endCharPos = Utf8ByteOffsetToCharPos(static_cast(h.endRow), endByteCol); + + TreeSitterLineBlock endBlock; + endBlock.nCharPos = endCharPos; + endBlock.nColorIndex = COLORINDEX_NORMALTEXT; + endBlock.nPriority = INT_MIN; + endBlock.nOrder = h.order; + m_lineBlocks[h.endRow].push_back(endBlock); + } + } +} + +/** + * @brief Run the injection query to find embedded language regions. + * + * Processes injections.scm captures: + * @injection.content - The node whose content should be parsed as another language + * @injection.language - The node whose text specifies the language name + * (#set! injection.language "xxx") - Hard-coded language name + * + * For each injection region, this spawns a sub-parser with the appropriate + * language grammar, runs highlights on the injected content, and merges + * the results into the main highlight blocks. + */ +void CTreeSitterParser::RunInjectionQuery() +{ + if (!m_pTree || !m_pLang || !m_pLang->GetInjectionQuery()) + return; + + const TSQuery* pQuery = m_pLang->GetInjectionQuery(); + TSNode rootNode = ts_tree_root_node(m_pTree); + + TSQueryCursor* pCursor = ts_query_cursor_new(); + if (!pCursor) + return; + + ts_query_cursor_exec(pCursor, pQuery, rootNode); + + // Collect injection regions + struct InjectionRegion + { + std::string language; // Target language name + uint32_t contentStart; // Byte offset in document + uint32_t contentEnd; + TSPoint startPoint; + TSPoint endPoint; + }; + std::vector injections; + + TSQueryMatch match; + while (ts_query_cursor_next_match(pCursor, &match)) + { + std::string language; + TSNode contentNode = {}; + bool hasContent = false; + + // Check for #set! injection.language predicate first + language = GetSetProperty(pQuery, match.pattern_index, "injection.language"); + + // Also check for #set! injection.self + if (language.empty()) + { + std::string selfVal = GetSetProperty(pQuery, match.pattern_index, "injection.self"); + if (!selfVal.empty()) + { + // Use the current language + const std::wstring& wname = m_pLang->GetName(); + for (wchar_t ch : wname) + language += static_cast(ch); // ASCII only + } + } + + for (uint16_t i = 0; i < match.capture_count; i++) + { + TSQueryCapture capture = match.captures[i]; + + uint32_t nameLen = 0; + const char* captureName = ts_query_capture_name_for_id( + pQuery, capture.index, &nameLen); + std::string sCapture(captureName, nameLen); + + if (sCapture == "injection.content") + { + contentNode = capture.node; + hasContent = true; + } + else if (sCapture == "injection.language" && language.empty()) + { + // The captured node's text is the language name + uint32_t start = ts_node_start_byte(capture.node); + uint32_t end = ts_node_end_byte(capture.node); + if (start < m_documentText.size() && end <= m_documentText.size()) + language = m_documentText.substr(start, end - start); + } + } + + if (hasContent && !language.empty()) + { + InjectionRegion region; + region.language = language; + region.contentStart = ts_node_start_byte(contentNode); + region.contentEnd = ts_node_end_byte(contentNode); + region.startPoint = ts_node_start_point(contentNode); + region.endPoint = ts_node_end_point(contentNode); + injections.push_back(region); + } + } + + ts_query_cursor_delete(pCursor); + + if (injections.empty()) + return; + + // Process each injection: look up the language, parse the content, run highlights + TreeSitterRegistry& registry = TreeSitterRegistry::Instance(); + + for (const auto& inj : injections) + { + // Convert language name to wstring for registry lookup + std::wstring wLangName; + for (char ch : inj.language) + wLangName += static_cast(ch); + + // Try to find the language — look it up by name directly in the registry's + // available languages (we need a way to get language by name, not just by ext). + // For now, try common mappings. The language name from injections.scm + // is typically the tree-sitter language name (e.g. "javascript", "css"). + // We can look it up as an extension since many languages use their name + // as the extension. + const CTreeSitterLanguage* pInjLang = registry.GetLanguageForName(wLangName); + if (!pInjLang || !pInjLang->GetHighlightQuery()) + continue; + + // Extract the injection content + if (inj.contentStart >= m_documentText.size() || inj.contentEnd > m_documentText.size()) + continue; + + std::string injContent = m_documentText.substr(inj.contentStart, inj.contentEnd - inj.contentStart); + if (injContent.empty()) + continue; + + // Create a temporary parser for the injected content + TSParser* pInjParser = ts_parser_new(); + if (!pInjParser) + continue; + + ts_parser_set_language(pInjParser, pInjLang->GetLanguage()); + + TSTree* pInjTree = ts_parser_parse_string( + pInjParser, nullptr, + injContent.c_str(), + static_cast(injContent.size())); + + if (pInjTree) + { + // Run highlight query on the injected tree + const TSQuery* pInjQuery = pInjLang->GetHighlightQuery(); + TSNode injRoot = ts_tree_root_node(pInjTree); + TSQueryCursor* pInjCursor = ts_query_cursor_new(); + + if (pInjCursor) + { + ts_query_cursor_exec(pInjCursor, pInjQuery, injRoot); + + TSQueryMatch injMatch; + while (ts_query_cursor_next_match(pInjCursor, &injMatch)) + { + for (uint16_t ci = 0; ci < injMatch.capture_count; ci++) + { + TSQueryCapture cap = injMatch.captures[ci]; + TSNode capNode = cap.node; + + uint32_t capNameLen = 0; + const char* capName = ts_query_capture_name_for_id( + pInjQuery, cap.index, &capNameLen); + std::string sCapName(capName, capNameLen); + int colorIndex = m_colorMap.MapCapture(sCapName); + + // Map the injected node's position back to the parent document. + // The injection content starts at inj.startPoint in the parent doc. + TSPoint capStart = ts_node_start_point(capNode); + TSPoint capEnd = ts_node_end_point(capNode); + + // Translate rows/columns to parent document coordinates. + // Since the injection content is a contiguous substring of + // m_documentText, continuation line columns in the sub-parse + // are already correct relative to the parent document lines. + // NOTE: This does NOT handle injection.combined (multiple + // disjoint ranges concatenated into one parse), but we don't + // support that feature currently. + uint32_t parentStartRow = inj.startPoint.row + capStart.row; + uint32_t parentEndRow = inj.startPoint.row + capEnd.row; + uint32_t parentStartCol = (capStart.row == 0) + ? inj.startPoint.column + capStart.column + : capStart.column; + + // Add to parent's line blocks + for (uint32_t row = parentStartRow; + row <= parentEndRow && row < static_cast(m_nLineCount); + row++) + { + uint32_t byteCol = (row == parentStartRow) ? parentStartCol : 0; + int charPos = Utf8ByteOffsetToCharPos(static_cast(row), byteCol); + + TreeSitterLineBlock block; + block.nCharPos = charPos; + block.nColorIndex = colorIndex; + block.nPriority = MakeCapturePriority(sCapName, + ts_node_start_byte(capNode), ts_node_end_byte(capNode)); + block.nOrder = NextBlockOrder(); + m_lineBlocks[row].push_back(block); + } + } + } + + ts_query_cursor_delete(pInjCursor); + } + + ts_tree_delete(pInjTree); + } + + ts_parser_delete(pInjParser); + } +} + +void CTreeSitterParser::BuildLineCache(int nLineCount) +{ + // Sort each line's blocks by character position and deterministic precedence. + // For identical start positions, keep the most specific / shortest capture. + for (int i = 0; i < nLineCount && i < static_cast(m_lineBlocks.size()); i++) + { + auto& blocks = m_lineBlocks[i]; + std::sort(blocks.begin(), blocks.end(), + [](const TreeSitterLineBlock& a, const TreeSitterLineBlock& b) + { + if (a.nCharPos != b.nCharPos) + return a.nCharPos < b.nCharPos; + if (a.nPriority != b.nPriority) + return a.nPriority < b.nPriority; + return a.nOrder < b.nOrder; + }); + + // Remove consecutive entries at the same position (keep the last one, + // which is typically the more specific/inner match) + if (blocks.size() > 1) + { + std::vector deduped; + deduped.reserve(blocks.size()); + for (size_t j = 0; j < blocks.size(); j++) + { + if (j + 1 < blocks.size() && blocks[j].nCharPos == blocks[j + 1].nCharPos) + continue; // skip, keep the later one + deduped.push_back(blocks[j]); + } + blocks = std::move(deduped); + } + } +} + +void CTreeSitterParser::GetLineBlocks(int nLineIndex, + CrystalLineParser::TEXTBLOCK* pBuf, + int& nActualItems, + int nMaxBlocks) const +{ + // Cookie-only mode (pBuf == nullptr): caller just wants the cookie. + // Tree-sitter doesn't use cookies, so nothing to do. + if (!pBuf) + return; + + // Fix #4: bounds check against cached line count + if (nLineIndex < 0 || nLineIndex >= static_cast(m_lineBlocks.size())) + return; + + const auto& blocks = m_lineBlocks[nLineIndex]; + + // The caller (GetTextBlocks) pre-inserts a NORMALTEXT block at position 0 + // and sets nActualItems = 1 before calling ParseLine. We follow the same + // convention as existing parsers: append our blocks starting at the current + // nActualItems, but overwrite the caller's default block if we have our own + // block at position 0. + + for (const auto& block : blocks) + { + // If the caller's last block is at the same position, overwrite it + // (same logic as DEFINE_BLOCK macro in crystallineparser.h) + if (nActualItems > 0 && pBuf[nActualItems - 1].m_nCharPos == block.nCharPos) + { + pBuf[nActualItems - 1].m_nColorIndex = block.nColorIndex; + pBuf[nActualItems - 1].m_nBgColorIndex = COLORINDEX_BKGND; + continue; + } + + // Skip if same color as previous block (no visible change) + if (nActualItems > 0 && pBuf[nActualItems - 1].m_nColorIndex == block.nColorIndex) + continue; + + // Bounds check: stop if we'd overflow the buffer + if (nMaxBlocks > 0 && nActualItems >= nMaxBlocks) + break; + + pBuf[nActualItems].m_nCharPos = block.nCharPos; + pBuf[nActualItems].m_nColorIndex = block.nColorIndex; + pBuf[nActualItems].m_nBgColorIndex = COLORINDEX_BKGND; + nActualItems++; + } +} + + +// ============================================================================ +// TreeSitterRegistry +// ============================================================================ + +// Default extension -> language mappings. +// Users can extend this via configuration. +static const struct +{ + const wchar_t* ext; + const wchar_t* language; +} s_defaultExtMap[] = +{ + // F# (primary target) + { L"fs", L"fsharp" }, + { L"fsx", L"fsharp" }, + { L"fsi", L"fsharp_signature" }, + + // Common languages + { L"c", L"c" }, + { L"h", L"c" }, + { L"cpp", L"cpp" }, + { L"cxx", L"cpp" }, + { L"cc", L"cpp" }, + { L"hpp", L"cpp" }, + { L"hxx", L"cpp" }, + { L"cs", L"c-sharp" }, + { L"py", L"python" }, + { L"js", L"javascript" }, + { L"ts", L"typescript" }, + { L"tsx", L"tsx" }, + { L"jsx", L"javascript" }, + { L"java", L"java" }, + { L"go", L"go" }, + { L"rs", L"rust" }, + { L"rb", L"ruby" }, + { L"lua", L"lua" }, + { L"sh", L"bash" }, + { L"bash", L"bash" }, + { L"json", L"json" }, + { L"yaml", L"yaml" }, + { L"yml", L"yaml" }, + { L"xml", L"xml" }, + { L"html", L"html" }, + { L"htm", L"html" }, + { L"css", L"css" }, + { L"sql", L"sql" }, + { L"ps1", L"powershell" }, + { L"psm1", L"powershell" }, + { L"php", L"php" }, + { L"pl", L"perl" }, + { L"swift", L"swift" }, + { L"kt", L"kotlin" }, + { L"scala", L"scala" }, + { L"hs", L"haskell" }, + { L"ml", L"ocaml" }, + { L"mli", L"ocaml" }, + { L"ex", L"elixir" }, + { L"exs", L"elixir" }, + { L"zig", L"zig" }, + { L"nim", L"nim" }, + { L"toml", L"toml" }, + { L"md", L"markdown" }, +}; + +TreeSitterRegistry& TreeSitterRegistry::Instance() +{ + static TreeSitterRegistry instance; + return instance; +} + +void TreeSitterRegistry::Initialize(const std::wstring& sGrammarDir) +{ + if (m_bInitialized) + return; + + // Determine grammar directory + if (sGrammarDir.empty()) + { + // Use /TreeSitterGrammars/ + wchar_t szExePath[MAX_PATH] = {}; + GetModuleFileNameW(nullptr, szExePath, MAX_PATH); + std::wstring sExeDir(szExePath); + auto pos = sExeDir.rfind(L'\\'); + if (pos != std::wstring::npos) + sExeDir = sExeDir.substr(0, pos); + m_sGrammarDir = sExeDir + L"\\TreeSitterGrammars"; + } + else + { + m_sGrammarDir = sGrammarDir; + } + + // Register default extension mappings + for (const auto& mapping : s_defaultExtMap) + { + m_extMap[mapping.ext] = mapping.language; + } + + // Check if the grammar directory exists + DWORD dwAttrib = GetFileAttributesW(m_sGrammarDir.c_str()); + if (dwAttrib == INVALID_FILE_ATTRIBUTES || !(dwAttrib & FILE_ATTRIBUTE_DIRECTORY)) + { + // Directory doesn't exist - that's OK, just means no tree-sitter support + m_bInitialized = true; + return; + } + + // Fix #3: Only discover available grammar DLLs, don't load them yet. + // Grammars are loaded lazily on first request in GetLanguageForExt(). + WIN32_FIND_DATAW findData; + std::wstring searchPattern = m_sGrammarDir + L"\\tree-sitter-*.dll"; + HANDLE hFind = FindFirstFileW(searchPattern.c_str(), &findData); + if (hFind != INVALID_HANDLE_VALUE) + { + do + { + // Extract language name from "tree-sitter-.dll" + std::wstring sFileName(findData.cFileName); + const std::wstring prefix = L"tree-sitter-"; + const std::wstring suffix = L".dll"; + if (sFileName.size() > prefix.size() + suffix.size()) + { + std::wstring sLangName = sFileName.substr( + prefix.size(), + sFileName.size() - prefix.size() - suffix.size()); + + // Just record that this language is available + m_availableLanguages.insert(sLangName); + } + } while (FindNextFileW(hFind, &findData)); + FindClose(hFind); + } + + m_bInitialized = true; +} + +const CTreeSitterLanguage* TreeSitterRegistry::GetLanguageForExt(const std::wstring& sExt) +{ + // Look up extension -> language name + auto itExt = m_extMap.find(sExt); + if (itExt == m_extMap.end()) + return nullptr; + + const std::wstring& sLangName = itExt->second; + + // Check if already loaded + auto itLang = m_languages.find(sLangName); + if (itLang != m_languages.end()) + { + // Already loaded - return if it has a valid highlight query + if (!itLang->second->GetHighlightQuery()) + return nullptr; + return itLang->second.get(); + } + + // Check if this language previously failed to load + if (m_failedLanguages.count(sLangName) > 0) + return nullptr; + + // Check if a DLL is available for this language + if (m_availableLanguages.count(sLangName) == 0) + return nullptr; + + // Lazy load: load the grammar DLL now + auto pLang = std::make_unique(); + if (!pLang->Load(m_sGrammarDir, sLangName)) + { + // Record failure so we don't retry + m_failedLanguages.insert(sLangName); + return nullptr; + } + + // Check if the loaded grammar has a highlight query + if (!pLang->GetHighlightQuery()) + { + m_failedLanguages.insert(sLangName); + return nullptr; + } + + const CTreeSitterLanguage* pResult = pLang.get(); + m_languages[sLangName] = std::move(pLang); + return pResult; +} + +const CTreeSitterLanguage* TreeSitterRegistry::GetLanguageForName(const std::wstring& sLangName) +{ + // Check if already loaded + auto itLang = m_languages.find(sLangName); + if (itLang != m_languages.end()) + { + if (!itLang->second->GetHighlightQuery()) + return nullptr; + return itLang->second.get(); + } + + // Check if this language previously failed to load + if (m_failedLanguages.count(sLangName) > 0) + return nullptr; + + // Check if a DLL is available for this language + if (m_availableLanguages.count(sLangName) == 0) + return nullptr; + + // Lazy load: load the grammar DLL now + auto pLang = std::make_unique(); + if (!pLang->Load(m_sGrammarDir, sLangName)) + { + m_failedLanguages.insert(sLangName); + return nullptr; + } + + if (!pLang->GetHighlightQuery()) + { + m_failedLanguages.insert(sLangName); + return nullptr; + } + + const CTreeSitterLanguage* pResult = pLang.get(); + m_languages[sLangName] = std::move(pLang); + return pResult; +} + +void TreeSitterRegistry::RegisterExtension(const std::wstring& sExt, const std::wstring& sLanguage) +{ + m_extMap[sExt] = sLanguage; +} + +// ============================================================================ +// CTreeSitterParser - Additional Methods +// ============================================================================ + +/** + * @brief Convenience: parse document from a text buffer. + */ +void CTreeSitterParser::ParseFromBuffer(CCrystalTextBuffer* pBuffer) +{ + if (!pBuffer) + return; + + const int nLineCount = pBuffer->GetLineCount(); + if (nLineCount == 0) + return; + + // Collect line pointers and lengths + std::vector ppszLines(nLineCount); + std::vector pnLineLengths(nLineCount); + + for (int i = 0; i < nLineCount; i++) + { + ppszLines[i] = pBuffer->GetLineChars(i); + pnLineLengths[i] = pBuffer->GetLineLength(i); + } + + ParseDocument(ppszLines.data(), pnLineLengths.data(), nLineCount); +} + +/** + * @brief Get the node type name at a specific position. + * + * This is used for comment filtering and other syntax-aware operations. + */ +std::wstring CTreeSitterParser::GetNodeTypeAt(int nLineIndex, int nCharPos) const +{ + if (!m_pTree || nLineIndex < 0 || nLineIndex >= m_nLineCount) + return _T(""); + + // Convert line + character position to byte offset + if (nLineIndex >= static_cast(m_lineUtf8.size())) + return _T(""); + + // Calculate byte offset + uint32_t byteOffset = 0; + + // Add bytes from all previous lines + for (int i = 0; i < nLineIndex; i++) + { + if (i >= static_cast(m_lineUtf8.size())) + return _T(""); + byteOffset += static_cast(m_lineUtf8[i].size()); + byteOffset++; // newline character + } + + // Add bytes from current line up to nCharPos + const std::string& lineUtf8 = m_lineUtf8[nLineIndex]; + int charCount = 0; + for (size_t i = 0; i < lineUtf8.size() && charCount < nCharPos; ) + { + unsigned char byte = lineUtf8[i]; + + // UTF-8 character length determination + int charLen = 1; + if ((byte & 0x80) == 0x00) + charLen = 1; // ASCII + else if ((byte & 0xE0) == 0xC0) + charLen = 2; + else if ((byte & 0xF0) == 0xE0) + charLen = 3; + else if ((byte & 0xF8) == 0xF0) + charLen = 4; + + i += charLen; + byteOffset += charLen; + charCount++; + } + + // Get the tree-sitter node at this byte position + TSNode rootNode = ts_tree_root_node(m_pTree); + TSNode node = ts_node_descendant_for_byte_range(rootNode, byteOffset, byteOffset); + + if (ts_node_is_null(node)) + return _T(""); + + // Get node type + const char* pszType = ts_node_type(node); + if (!pszType) + return _T(""); + + // Convert from UTF-8 to wstring/String +#ifdef UNICODE + int nLen = MultiByteToWideChar(CP_UTF8, 0, pszType, -1, nullptr, 0); + if (nLen == 0) + return _T(""); + + std::vector buffer(nLen); + MultiByteToWideChar(CP_UTF8, 0, pszType, -1, buffer.data(), nLen); + return std::wstring(buffer.data()); +#else + return String(pszType); +#endif +} + +bool CTreeSitterParser::IsCommentPosition(int nLineIndex, int nCharPos) const +{ + const std::wstring sNodeType = GetNodeTypeAt(nLineIndex, nCharPos); + return sNodeType.find(L"comment") != std::wstring::npos; +} + +bool CTreeSitterParser::TryGetDefinitionByteRangeAt(uint32_t byteOffset, uint32_t& defStartByte, uint32_t& defEndByte) const +{ + for (const auto& scope : m_localScopes) + { + for (const auto& def : scope.defs) + { + if (byteOffset >= def.startByte && byteOffset < def.endByte) + { + defStartByte = def.startByte; + defEndByte = def.endByte; + return true; + } + } + } + + for (const auto& ref : m_pendingRefs) + { + if (byteOffset < ref.startByte || byteOffset >= ref.endByte) + continue; + + std::vector enclosingScopes; + for (const auto& scope : m_localScopes) + { + if (ref.startByte >= scope.startByte && ref.endByte <= scope.endByte) + enclosingScopes.push_back(&scope); + } + + std::sort(enclosingScopes.begin(), enclosingScopes.end(), + [](const LocalScope* a, const LocalScope* b) + { + return (a->endByte - a->startByte) < (b->endByte - b->startByte); + }); + + for (const auto* pScope : enclosingScopes) + { + for (const auto& def : pScope->defs) + { + if (def.name == ref.name && def.startByte <= ref.startByte) + { + defStartByte = def.startByte; + defEndByte = def.endByte; + return true; + } + } + if (!pScope->inherits) + break; + } + + return false; + } + + for (const auto& def : m_tagDefs) + { + if (byteOffset >= def.startByte && byteOffset < def.endByte) + { + defStartByte = def.startByte; + defEndByte = def.endByte; + return true; + } + } + + for (const auto& ref : m_tagRefs) + { + if (byteOffset < ref.startByte || byteOffset >= ref.endByte) + continue; + + const TagDef* pBestDef = nullptr; + for (const auto& def : m_tagDefs) + { + if (def.name != ref.name) + continue; + if (!pBestDef || def.startByte < pBestDef->startByte) + pBestDef = &def; + } + + if (pBestDef) + { + defStartByte = pBestDef->startByte; + defEndByte = pBestDef->endByte; + return true; + } + + return false; + } + + return false; +} + +bool CTreeSitterParser::ByteOffsetToLineChar(uint32_t byteOffset, int& nLineIndex, int& nCharPos) const +{ + uint32_t currentOffset = 0; + for (int i = 0; i < static_cast(m_lineUtf8.size()); ++i) + { + const std::string& utf8Line = m_lineUtf8[i]; + const uint32_t lineStart = currentOffset; + const uint32_t lineEnd = lineStart + static_cast(utf8Line.size()); + if (byteOffset <= lineEnd) + { + nLineIndex = i; + nCharPos = Utf8ByteOffsetToCharPos(i, byteOffset - lineStart); + return true; + } + currentOffset = lineEnd + 1; + } + + if (!m_lineUtf8.empty() && byteOffset == currentOffset - 1) + { + nLineIndex = static_cast(m_lineUtf8.size()) - 1; + nCharPos = static_cast(m_lineUtf8.back().size()); + return true; + } + + return false; +} + +bool CTreeSitterParser::TryGetTagDefinitionByNameAt(int nLineIndex, int nCharPos, uint32_t& defStartByte, uint32_t& defEndByte) const +{ + if (!m_pBuffer || nLineIndex < 0 || nLineIndex >= m_pBuffer->GetLineCount()) + return false; + + const tchar_t* pszLine = m_pBuffer->GetLineChars(nLineIndex); + const int nLineLength = m_pBuffer->GetLineLength(nLineIndex); + if (!pszLine || nLineLength <= 0) + return false; + + auto IsIdentChar = [](tchar_t ch) + { + return _istalnum(ch) || ch == _T('_') || ch == _T('`') || ch == _T('\''); + }; + + int nIndex = nCharPos; + if (nIndex >= nLineLength) + nIndex = nLineLength - 1; + if (nIndex < 0) + return false; + + if (!IsIdentChar(pszLine[nIndex]) && nIndex > 0 && IsIdentChar(pszLine[nIndex - 1])) + --nIndex; + if (!IsIdentChar(pszLine[nIndex])) + return false; + + int nStart = nIndex; + while (nStart > 0 && IsIdentChar(pszLine[nStart - 1])) + --nStart; + + int nEnd = nIndex + 1; + while (nEnd < nLineLength && IsIdentChar(pszLine[nEnd])) + ++nEnd; + + std::wstring symbolW(pszLine + nStart, nEnd - nStart); +#ifdef _UNICODE + int nUtf8Len = WideCharToMultiByte(CP_UTF8, 0, symbolW.c_str(), static_cast(symbolW.size()), nullptr, 0, nullptr, nullptr); + if (nUtf8Len <= 0) + return false; + std::string symbol(nUtf8Len, '\0'); + WideCharToMultiByte(CP_UTF8, 0, symbolW.c_str(), static_cast(symbolW.size()), symbol.data(), nUtf8Len, nullptr, nullptr); +#else + std::string symbol(symbolW.begin(), symbolW.end()); +#endif + + const TagDef* pBestDef = nullptr; + for (const auto& def : m_tagDefs) + { + if (def.name != symbol) + continue; + if (!pBestDef || def.startByte < pBestDef->startByte) + pBestDef = &def; + } + + if (!pBestDef) + return false; + + defStartByte = pBestDef->startByte; + defEndByte = pBestDef->endByte; + return true; +} + +bool CTreeSitterParser::FindDefinition(int nLineIndex, int nCharPos, int& nDefLine, int& nDefChar) const +{ + if (!m_pTree || nLineIndex < 0 || nLineIndex >= m_nLineCount) + return false; + + uint32_t byteOffset = 0; + for (int i = 0; i < nLineIndex; ++i) + byteOffset += static_cast(m_lineUtf8[i].size()) + 1; + + const std::string& lineUtf8 = m_lineUtf8[nLineIndex]; + int charCount = 0; + for (size_t i = 0; i < lineUtf8.size() && charCount < nCharPos; ) + { + const unsigned char byte = static_cast(lineUtf8[i]); + int charLen = 1; + if ((byte & 0x80) == 0x00) + charLen = 1; + else if ((byte & 0xE0) == 0xC0) + charLen = 2; + else if ((byte & 0xF0) == 0xE0) + charLen = 3; + else if ((byte & 0xF8) == 0xF0) + charLen = 4; + i += charLen; + byteOffset += charLen; + charCount++; + } + + uint32_t defStartByte = 0; + uint32_t defEndByte = 0; + const bool foundAtPosition = TryGetDefinitionByteRangeAt(byteOffset, defStartByte, defEndByte); + + uint32_t tagDefStartByte = 0; + uint32_t tagDefEndByte = 0; + if (!foundAtPosition) + { + if (!TryGetTagDefinitionByNameAt( + nLineIndex, nCharPos, tagDefStartByte, tagDefEndByte)) + return false; + + defStartByte = tagDefStartByte; + defEndByte = tagDefEndByte; + } + else + { + int posDefLine = 0; + int posDefChar = 0; + const bool positionResolved = + ByteOffsetToLineChar(defStartByte, posDefLine, posDefChar); + + // If the position-based lookup resolves back to the clicked symbol, + // prefer the tag-definition target when it points somewhere else. + // This avoids getting stuck on the clicked type reference after the + // context-menu click has moved the caret onto the symbol. + if (!positionResolved || (posDefLine == nLineIndex && posDefChar == nCharPos)) + { + if (TryGetTagDefinitionByNameAt( + nLineIndex, nCharPos, tagDefStartByte, tagDefEndByte) && + (!positionResolved || tagDefStartByte != defStartByte)) + { + defStartByte = tagDefStartByte; + defEndByte = tagDefEndByte; + } + } + } + + return ByteOffsetToLineChar(defStartByte, nDefLine, nDefChar); +} diff --git a/Externals/crystaledit/editlib/TreeSitterParser.h b/Externals/crystaledit/editlib/TreeSitterParser.h new file mode 100644 index 00000000000..af3dafb4574 --- /dev/null +++ b/Externals/crystaledit/editlib/TreeSitterParser.h @@ -0,0 +1,441 @@ +//////////////////////////////////////////////////////////////////////////// +// File: TreeSitterParser.h +// Version: 1.0.1 +// Created: 2026-03-26 +// +// Tree-sitter based syntax highlighting bridge for CrystalEdit. +// +// Loads tree-sitter grammar DLLs and highlight query (.scm) files +// at runtime. Parses full documents into ASTs and extracts per-line +// color blocks mapped to WinMerge's COLORINDEX values. +// +// Grammar DLLs are loaded from a "TreeSitterGrammars" directory +// alongside the WinMerge executable. Each DLL exports a function +// named tree_sitter_() returning const TSLanguage*. +// +// Highlight queries are loaded from .scm files in the same directory. +// +// SPDX-License-Identifier: GPL-2.0-or-later +//////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "SyntaxColors.h" +#include "parsers/crystallineparser.h" + +#include +#include +#include +#include +#include + +// Forward declarations for tree-sitter C API types. +// The actual tree_sitter/api.h header is included only in the .cpp file. +typedef struct TSParser TSParser; +typedef struct TSTree TSTree; +typedef struct TSQuery TSQuery; +typedef struct TSLanguage TSLanguage; + +class CCrystalTextBuffer; + +/** + * @brief Manages a tree-sitter grammar loaded from a DLL. + * + * Each instance holds a loaded grammar DLL, the TSLanguage pointer, + * and the compiled highlight, locals, tags, and injection queries for that language. + */ +class CTreeSitterLanguage +{ +public: + CTreeSitterLanguage(); + ~CTreeSitterLanguage(); + + CTreeSitterLanguage(const CTreeSitterLanguage&) = delete; + CTreeSitterLanguage& operator=(const CTreeSitterLanguage&) = delete; + + /** + * @brief Load a grammar DLL and its query files. + * @param sGrammarDir Directory containing grammar DLLs and .scm files. + * @param sLanguage Language name (e.g. "fsharp", "python", "cpp"). + * @return true if both the DLL and highlight query loaded successfully. + * + * Also attempts to load locals.scm, tags.scm, and injections.scm if present. + * Failure to load optional queries is not fatal. + */ + bool Load(const std::wstring& sGrammarDir, const std::wstring& sLanguage); + + /** @brief Get the loaded TSLanguage pointer (or nullptr). */ + const TSLanguage* GetLanguage() const { return m_pLanguage; } + + /** @brief Get the compiled highlight TSQuery (or nullptr). */ + const TSQuery* GetHighlightQuery() const { return m_pHighlightQuery; } + + /** @brief Get the compiled locals TSQuery (or nullptr). */ + const TSQuery* GetLocalsQuery() const { return m_pLocalsQuery; } + + /** @brief Get the compiled tags TSQuery (or nullptr). */ + const TSQuery* GetTagsQuery() const { return m_pTagsQuery; } + + /** @brief Get the compiled injection TSQuery (or nullptr). */ + const TSQuery* GetInjectionQuery() const { return m_pInjectionQuery; } + + /** @brief Get the language name. */ + const std::wstring& GetName() const { return m_sName; } + +private: + /** @brief Helper to load and compile a .scm query file. */ + TSQuery* LoadQuery(const std::wstring& sPath); + + HMODULE m_hDll; + const TSLanguage* m_pLanguage; + TSQuery* m_pHighlightQuery; + TSQuery* m_pLocalsQuery; + TSQuery* m_pTagsQuery; + TSQuery* m_pInjectionQuery; + std::wstring m_sName; +}; + + +/** + * @brief Maps tree-sitter highlight capture names to COLORINDEX values. + * + * Standard capture names from nvim-treesitter / tree-sitter highlight queries: + * @keyword, @function, @function.call, @string, @comment, @number, + * @operator, @type, @variable, @property, @constant, @punctuation, + * @constructor, @module, @attribute, @label, @preproc, etc. + * + * These are collapsed into WinMerge's 9 syntax COLORINDEX values. + */ +class CTreeSitterColorMap +{ +public: + CTreeSitterColorMap(); + + /** + * @brief Map a tree-sitter capture name to a COLORINDEX. + * @param sCaptureName The capture name (e.g. "keyword", "string"). + * @return The COLORINDEX, or COLORINDEX_NORMALTEXT if unknown. + */ + int MapCapture(const std::string& sCaptureName) const; + +private: + std::unordered_map m_map; +}; + + +/** + * @brief Per-line cached highlight result. + */ +struct TreeSitterLineBlock +{ + int nCharPos; + int nColorIndex; + int nPriority; + uint32_t nOrder; +}; + + +/** + * @brief Tree-sitter based syntax parser for CrystalEdit views. + * + * This class replaces the line-by-line keyword parsers with full-document + * AST-based parsing via tree-sitter. It: + * 1. Parses the entire document into a tree-sitter AST. + * 2. Runs highlight queries to extract token ranges. + * 3. Caches per-line color block arrays. + * 4. Serves cached results on per-line ParseLine() calls. + * 5. Supports lazy re-parsing: edits mark the cache dirty, + * and the next paint cycle triggers a single reparse. + * + * Override ParseLine() in the view to call this parser's GetLineBlocks(). + * + * Usage: + * @code + * auto pLang = TreeSitterRegistry::Instance().GetLanguageForExt(L"fs"); + * if (pLang) { + * m_treeSitterParser.SetLanguage(pLang); + * m_treeSitterParser.ParseFromView(pView); + * // In ParseLine override: + * m_treeSitterParser.GetLineBlocks(nLineIndex, pBuf, nActualItems); + * } + * @endcode + */ +class CTreeSitterParser +{ +public: + CTreeSitterParser(); + ~CTreeSitterParser(); + + CTreeSitterParser(const CTreeSitterParser&) = delete; + CTreeSitterParser& operator=(const CTreeSitterParser&) = delete; + + /** + * @brief Set the language to use for parsing. + * @param pLang Pointer to a loaded CTreeSitterLanguage. + */ + void SetLanguage(const CTreeSitterLanguage* pLang); + + /** + * @brief Parse (or re-parse) the full document. + * @param ppszLines Array of line pointers (from CCrystalTextBuffer). + * @param pnLineLengths Array of line lengths. + * @param nLineCount Number of lines in the document. + */ + void ParseDocument(const tchar_t* const* ppszLines, const int* pnLineLengths, int nLineCount); + + /** + * @brief Mark the parse cache as dirty (e.g. after an edit). + * + * The next call to EnsureParsed() (from ParseLine/GetLineBlocks) + * will trigger a single reparse. This avoids reparsing on every + * keystroke and instead defers to the next paint cycle. + */ + void MarkDirty() { m_bDirty = true; } + + /** + * @brief Notify the parser of an edit for incremental reparsing. + * + * Extracts the last edit's position from the buffer's UndoRecord + * and calls ts_tree_edit() on the existing tree so tree-sitter + * can reuse unchanged subtrees during the next reparse. + * + * Also marks the cache dirty. + * + * @param pBuf The text buffer that was edited. + */ + void NotifyEdit(class CCrystalTextBuffer* pBuf); + + /** + * @brief Ensure the document is parsed and cache is up-to-date. + * @param pView The text view to read line data from (if reparse needed). + * + * Called lazily from ParseLine. Only reparses if marked dirty. + */ + void EnsureParsed(class CCrystalTextBuffer* pBuf); + + /** + * @brief Get the cached color blocks for a specific line. + * @param nLineIndex Zero-based line index. + * @param pBuf Output buffer for TEXTBLOCK entries (may be nullptr for cookie-only). + * @param nActualItems In/out: on entry, the number of blocks already in pBuf + * (caller pre-inserts a default NORMALTEXT block at position 0); + * on return, the total number of blocks. + * @param nMaxBlocks Maximum number of TEXTBLOCK entries that fit in pBuf. + * Pass 0 to skip bounds checking (legacy behavior). + */ + void GetLineBlocks(int nLineIndex, + CrystalLineParser::TEXTBLOCK* pBuf, + int& nActualItems, + int nMaxBlocks = 0) const; + + /** @brief Check if a valid tree is available. */ + bool HasTree() const { return m_pTree != nullptr; } + + /** @brief Check if a language is set. */ + bool HasLanguage() const { return m_pLang != nullptr; } + + /** @brief Get the number of lines in the cached parse result. */ + int GetCachedLineCount() const { return m_nLineCount; } + + /** @brief Check if cache needs rebuilding. */ + bool IsDirty() const { return m_bDirty; } + + /** @brief Invalidate cached results and free tree. */ + void Invalidate(); + + /** @brief Get the language object (for service layer). */ + const CTreeSitterLanguage* GetLanguage() const { return m_pLang; } + + /** + * @brief Get the node type name at a specific position. + * @param nLineIndex Zero-based line index. + * @param nCharPos Zero-based character position in line. + * @return Node type name (e.g. "comment", "string_literal"), or empty string. + * + * This is used by TreeSitterHighlightService to determine if a position + * is within a comment or string for filtering purposes. + */ + std::wstring GetNodeTypeAt(int nLineIndex, int nCharPos) const; + + bool IsCommentPosition(int nLineIndex, int nCharPos) const; + + bool FindDefinition(int nLineIndex, int nCharPos, int& nDefLine, int& nDefChar) const; + + /** + * @brief Convenience: parse document from a text buffer. + * @param pBuffer The text buffer to read line data from. + */ + void ParseFromBuffer(class CCrystalTextBuffer* pBuffer); + + void SetBuffer(class CCrystalTextBuffer* pBuffer) { m_pBuffer = pBuffer; } + CCrystalTextBuffer* GetBuffer() const { return m_pBuffer; } + +private: + void EnsureParser(); + void RunHighlightQuery(); + void RunLocalsQuery(); + void RunTagsQuery(); + void RunInjectionQuery(); + void BuildLineCache(int nLineCount); + int Utf8ByteOffsetToCharPos(int nLine, uint32_t byteCol) const; + bool TryGetDefinitionByteRangeAt(uint32_t byteOffset, uint32_t& defStartByte, uint32_t& defEndByte) const; + bool ByteOffsetToLineChar(uint32_t byteOffset, int& nLineIndex, int& nCharPos) const; + bool TryGetTagDefinitionByNameAt(int nLineIndex, int nCharPos, uint32_t& defStartByte, uint32_t& defEndByte) const; + uint32_t NextBlockOrder() { return m_nextBlockOrder++; } + + /** + * @brief Extract #set! predicate properties from a query pattern. + * @param pQuery The query containing the pattern. + * @param patternIndex The pattern index. + * @param key The property key to look for (e.g. "injection.language"). + * @return The property value, or empty string if not found. + */ + static std::string GetSetProperty(const TSQuery* pQuery, + uint32_t patternIndex, + const std::string& key); + + TSParser* m_pParser; // Created lazily on first use + TSTree* m_pTree; + const CTreeSitterLanguage* m_pLang; + CTreeSitterColorMap m_colorMap; + bool m_bDirty; // True when cache needs rebuild + + // Cached per-line highlight blocks + std::vector> m_lineBlocks; + + // Per-line UTF-8 content (for byte offset -> character position mapping) + std::vector m_lineUtf8; + + // Full document text (needed for tree-sitter, which requires contiguous input + // or a custom read callback) + std::string m_documentText; + int m_nLineCount; + uint32_t m_nextBlockOrder; + + // --- Locals support --- + // Maps (startByte, endByte) of definition nodes to their highlight color. + // Built by RunLocalsQuery + RunHighlightQuery cross-referencing. + struct LocalDef + { + std::string name; // Variable/symbol name + uint32_t startByte; // Definition node start + uint32_t endByte; // Definition node end + int highlight; // COLORINDEX from highlights.scm (-1 = unknown) + }; + + struct LocalScope + { + uint32_t startByte; + uint32_t endByte; + bool inherits; + std::vector defs; + }; + + // Scopes from locals.scm, sorted by startByte + std::vector m_localScopes; + + // Map from reference node byte range -> resolved highlight color. + // Key: (startByte << 32 | endByte). Assumes files < 4GB. + std::unordered_map m_localRefHighlights; + + // Pending references from RunLocalsQuery, resolved during RunHighlightQuery + struct PendingRef + { + std::string name; + uint32_t startByte; + uint32_t endByte; + uint32_t scopeStartByte; + }; + std::vector m_pendingRefs; + + struct TagDef + { + std::string name; + uint32_t startByte; + uint32_t endByte; + }; + + struct TagRef + { + std::string name; + uint32_t startByte; + uint32_t endByte; + }; + + std::vector m_tagDefs; + std::vector m_tagRefs; + CCrystalTextBuffer* m_pBuffer; // Needed to get line/char info for unresolved references +}; + + +/** + * @brief Global registry for tree-sitter grammars. + * + * Scans the grammar directory on first use to discover available grammar DLLs, + * but loads them lazily on first request. Maps file extensions to languages. + */ +class TreeSitterRegistry +{ +public: + /** + * @brief Get the singleton instance. + */ + static TreeSitterRegistry& Instance(); + + /** + * @brief Initialize the registry by scanning the grammar directory. + * @param sGrammarDir Path to directory containing grammar DLLs and .scm files. + * If empty, uses "/TreeSitterGrammars/". + * + * This only discovers available DLL filenames. Grammars are loaded + * lazily on first request via GetLanguageForExt(). + */ + void Initialize(const std::wstring& sGrammarDir = L""); + + /** + * @brief Find a loaded language for a file extension. + * @param sExt File extension without dot (e.g. "fs", "py", "cpp"). + * @return Pointer to the language, or nullptr if not available. + * + * Loads the grammar DLL lazily on first request for each language. + */ + const CTreeSitterLanguage* GetLanguageForExt(const std::wstring& sExt); + + /** + * @brief Find a loaded language by language name. + * @param sLangName Language name (e.g. "javascript", "css", "python"). + * @return Pointer to the language, or nullptr if not available. + * + * Used by injection processing to look up grammars for embedded languages. + * Loads the grammar DLL lazily on first request. + */ + const CTreeSitterLanguage* GetLanguageForName(const std::wstring& sLangName); + + /** + * @brief Register a file extension mapping to a language name. + * @param sExt Extension (e.g. "fs"). + * @param sLanguage Language name (e.g. "fsharp"). + */ + void RegisterExtension(const std::wstring& sExt, const std::wstring& sLanguage); + + /** @brief Check if the registry has been initialized. */ + bool IsInitialized() const { return m_bInitialized; } + +private: + TreeSitterRegistry() : m_bInitialized(false) {} + + bool m_bInitialized; + std::wstring m_sGrammarDir; + + // language name -> loaded grammar (loaded lazily) + std::unordered_map> m_languages; + + // language names that have a DLL available but haven't been loaded yet + std::unordered_set m_availableLanguages; + + // language names that failed to load (don't retry) + std::unordered_set m_failedLanguages; + + // file extension -> language name + std::unordered_map m_extMap; +}; diff --git a/Externals/crystaledit/editlib/TreeSitterWrapper.cpp b/Externals/crystaledit/editlib/TreeSitterWrapper.cpp new file mode 100644 index 00000000000..c5bc3626828 --- /dev/null +++ b/Externals/crystaledit/editlib/TreeSitterWrapper.cpp @@ -0,0 +1,201 @@ +/** + * @file TreeSitterWrapper.cpp + * + * @brief TreeSitter wrapper implementation for integration with TextDefinition::ParseLineX + */ + +#include "pch.h" +#include +#include "TreeSitterWrapper.h" +#include "TreeSitterParser.h" +#include "utils/ctchar.h" +#include + + /** + * @brief TreeSitter-based ParseLineX implementation + * + * Can be registered as TextDefinition::ParseLineX. + * Cookie upper 16 bits: BufferID + * Cookie lower 16 bits: Line number + * + * @param dwCookie State cookie (BufferID | LineIndex) + * @param pszChars Line text (unused - TreeSitter parses the entire content) + * @param nLength Line length (unused) + * @param pBuf Output buffer + * @param nActualItems Output item count + * @return Cookie for the next line + */ +unsigned ParseLineTreeSitter(unsigned dwCookie, int nLineIndex, const tchar_t* pszChars, + int nLength, CrystalLineParser::TEXTBLOCK* pBuf, + int& nActualItems, void* pContext) +{ + // Get parser from global context + CTreeSitterParser* pParser = reinterpret_cast(pContext); + if (pParser != nullptr) + { + // Lazy reparse: if cache is dirty, reparse now (once per paint cycle) + if (pParser->IsDirty()) + pParser->EnsureParsed(pParser->GetBuffer()); + + if (pParser->HasTree()) + { + // Bounds safety: use the parser's cached line count to avoid + // accessing stale data. We intentionally avoid calling + // LocateTextBuffer() here since ParseLine may be called + // before the window handle is fully set up. + if (nLineIndex >= 0 && nLineIndex < pParser->GetCachedLineCount()) + { + // Buffer size: GetTextBlocks allocates (nLength+1)*3 TEXTBLOCK entries + int nMaxBlocks = (nLength + 1) * 3; + + pParser->GetLineBlocks(nLineIndex, pBuf, nActualItems, 0); + + // Return cookie for next line + return 0; + } + } + } + + // If parser is not available, set default NORMALTEXT block + if (pBuf != nullptr) + { + nActualItems = 1; + pBuf[0].m_nCharPos = 0; + pBuf[0].m_nColorIndex = COLORINDEX_NORMALTEXT; + pBuf[0].m_nBgColorIndex = COLORINDEX_BKGND; + } + + // Return cookie for next line + return 0; +} + +/** + * @brief Dynamically create TextDefinition for TreeSitter + * + * @param pszExt File extension (without dot, e.g., "cpp", "fs") + * @param pszName Language name (for display, e.g., "C++", "F#") + * @param nBufferId Buffer ID (for context identification) + * @return New TextDefinition if TreeSitter is available, nullptr otherwise + * + * @note The returned TextDefinition must be freed by the caller using FreeTreeSitterTextDefinition() + */ +CrystalLineParser::TextDefinition* CreateTreeSitterTextDefinition( + const tchar_t* pszExt, const tchar_t* pszName, int nBufferId) +{ + if (pszExt == nullptr || pszName == nullptr) + return nullptr; + + // Check if TreeSitterRegistry has a language for this extension + const CTreeSitterLanguage* pLang = TreeSitterRegistry::Instance().GetLanguageForExt(pszExt); + if (pLang == nullptr || pLang->GetLanguage() == nullptr) + { + // Return nullptr if TreeSitter language is not available + return nullptr; + } + + // Dynamically allocate new TextDefinition + auto pDef = new CrystalLineParser::TextDefinition(); + if (pDef == nullptr) + return nullptr; + + // Initialize TextDefinition fields + pDef->type = CrystalLineParser::SRC_PLAIN; // TreeSitter does not have a specific type + pDef->ParseLineX = ParseLineTreeSitter; + pDef->flags = 0; + pDef->encoding = 0; + + // Copy language name (dynamic allocation) + const size_t nNameLen = tc::tcslen(pszName); + pDef->name = new tchar_t[nNameLen + 1]; +#ifdef _UNICODE + _wcslcpy(const_cast(pDef->name), nNameLen + 1, pszName); +#else + _strlcpy(const_cast(pDef->name), nNameLen + 1, pszName); +#endif + + // Copy extension (dynamic allocation) + const size_t nExtLen = tc::tcslen(pszExt); + pDef->exts = new tchar_t[nExtLen + 1]; +#ifdef _UNICODE + _wcslcpy(pDef->exts, nExtLen + 1, pszExt); +#else + _strlcpy(pDef->exts, nExtLen + 1, pszExt); +#endif + pDef->extsIsDynamic = true; + + // Comment syntax is empty (TreeSitter is AST-based, so not needed) + pDef->opencomment[0] = _T('\0'); + pDef->closecomment[0] = _T('\0'); + pDef->commentline[0] = _T('\0'); + + return pDef; +} + +/** + * @brief Free TreeSitter TextDefinition + * @param pDef TextDefinition to free + */ +void FreeTreeSitterTextDefinition(CrystalLineParser::TextDefinition* pDef) +{ + if (pDef == nullptr) + return; + + // Free dynamically allocated fields + if (pDef->name != nullptr) + { + delete[] pDef->name; + } + + if (pDef->extsIsDynamic && pDef->exts != nullptr) + { + delete[] pDef->exts; + } + + // Free TextDefinition itself + delete pDef; +} + +void* CreateTreeSitterParseContextForDiff(const std::wstring& filePath, const std::vector& lines) +{ + std::wstring ext = filePath; + size_t posOfDot = ext.rfind('.'); + if (posOfDot != std::wstring::npos) + ext.erase(0, posOfDot + 1); + + TreeSitterRegistry& registry = TreeSitterRegistry::Instance(); + if (!registry.IsInitialized()) + registry.Initialize(); + + const CTreeSitterLanguage* pLang = registry.GetLanguageForExt(ext.c_str()); + if (pLang == nullptr || pLang->GetLanguage() == nullptr) + return nullptr; + + auto* pParser = new CTreeSitterParser(); + pParser->SetLanguage(pLang); + + std::vector linePtrs; + std::vector lineLens; + linePtrs.reserve(lines.size()); + lineLens.reserve(lines.size()); + for (const auto& line : lines) + { + linePtrs.push_back(line.c_str()); + lineLens.push_back(static_cast(line.size())); + } + pParser->ParseDocument(linePtrs.data(), lineLens.data(), static_cast(linePtrs.size())); + return pParser; +} + +void DestroyTreeSitterParseContextForDiff(void* parseContext) +{ + delete reinterpret_cast(parseContext); +} + +bool IsTreeSitterCommentPositionForDiff(void* parseContext, int nLineIndex, int nCharPos) +{ + auto* pParser = reinterpret_cast(parseContext); + if (pParser == nullptr) + return false; + + return pParser->IsCommentPosition(nLineIndex, nCharPos); +} diff --git a/Externals/crystaledit/editlib/TreeSitterWrapper.h b/Externals/crystaledit/editlib/TreeSitterWrapper.h new file mode 100644 index 00000000000..0c532998a14 --- /dev/null +++ b/Externals/crystaledit/editlib/TreeSitterWrapper.h @@ -0,0 +1,53 @@ +/** + * @file TreeSitterWrapper.h + * + * @brief TreeSitter wrapper for integration with TextDefinition::ParseLineX + */ +#pragma once + +#include +#include +#include "parsers/crystallineparser.h" +#include +#include + +/** + * @brief TreeSitter-based ParseLineX implementation + * + * Can be registered as TextDefinition::ParseLineX. + * Cookie upper 16 bits: BufferID + * Cookie lower 16 bits: Line number + * + * @param dwCookie State cookie (BufferID | LineIndex) + * @param pszChars Line text (unused) + * @param nLength Line length (unused) + * @param pBuf Output buffer + * @param nActualItems Output item count + * @return Cookie for the next line + */ +unsigned ParseLineTreeSitter(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, + int nLength, CrystalLineParser::TEXTBLOCK * pBuf, + int &nActualItems, void* pContext); + +/** + * @brief Dynamically create TextDefinition for TreeSitter + * + * @param pszExt File extension + * @param pszName Language name (for display) + * @param nBufferId Buffer ID (for context identification) + * @return New TextDefinition if TreeSitter is available, nullptr otherwise + * + * @note The returned TextDefinition must be freed by the caller using delete + */ +CrystalLineParser::TextDefinition* CreateTreeSitterTextDefinition( + const tchar_t* pszExt, const tchar_t* pszName, int nBufferId); + +/** + * @brief Free TreeSitter TextDefinition + * @param pDef TextDefinition to free + */ +void FreeTreeSitterTextDefinition(CrystalLineParser::TextDefinition* pDef); + +void* CreateTreeSitterParseContextForDiff(const std::wstring& filePath, const std::vector& lines); +void DestroyTreeSitterParseContextForDiff(void* parseContext); +bool IsTreeSitterCommentPositionForDiff(void* parseContext, int nLineIndex, int nCharPos); diff --git a/Externals/crystaledit/editlib/ccrystaltextbuffer.h b/Externals/crystaledit/editlib/ccrystaltextbuffer.h index 6fe3c94d7c4..69e97237b26 100644 --- a/Externals/crystaledit/editlib/ccrystaltextbuffer.h +++ b/Externals/crystaledit/editlib/ccrystaltextbuffer.h @@ -155,6 +155,8 @@ public : }; std::shared_ptr m_pSharedTableProps; + void* pParseContext; // context for incremental parsing, owned by the parser + // Helper methods void InsertLine (const tchar_t* pszLine, size_t nLength, int nPosition = -1, int nCount = 1); void AppendLine (int nLineIndex, const tchar_t* pszChars, size_t nLength, bool bDetectEol = true); @@ -169,7 +171,6 @@ public : // [JRT] Support For Descriptions On Undo/Redo Actions virtual void AddUndoRecord (bool bInsert, const CEPoint & ptStartPos, const CEPoint & ptEndPos, const tchar_t* pszText, size_t cchText, int nActionType = CE_ACTION_UNKNOWN, std::vector *paSavedRevisionNumbers = nullptr); - virtual UndoRecord GetUndoRecord (int nUndoPos) const { return m_aUndoBuf[nUndoPos]; } virtual std::vector *CopyRevisionNumbers(int nStartLine, int nEndLine) const; virtual void RestoreRevisionNumbers(int nStartLine, std::vector *psaSavedRevisionNumbers); @@ -246,6 +247,8 @@ public : // Undo/Redo bool CanUndo () const; bool CanRedo () const; + int GetUndoPosition () const { return m_nUndoPosition; } + virtual UndoRecord GetUndoRecord (int nUndoPos) const { return m_aUndoBuf[nUndoPos]; } virtual bool Undo (CCrystalTextView * pSource, CEPoint & ptCursorPos); virtual bool UndoInsert (CCrystalTextView * pSource, CEPoint & ptCursorPos, const CEPoint apparent_ptStartPos, CEPoint const apparent_ptEndPos, const UndoRecord & ur); virtual bool Redo (CCrystalTextView * pSource, CEPoint & ptCursorPos); @@ -316,6 +319,9 @@ public : int FindNextBookmarkLine (int nCurrentLine = 0) const; int FindPrevBookmarkLine (int nCurrentLine = 0) const; + void* GetParseContext() const { return pParseContext; } + void SetParseContext(void* pContext) { pParseContext = pContext; } + // Overrides // ClassWizard generated virtual function overrides //{{AFX_VIRTUAL(CCrystalTextBuffer) diff --git a/Externals/crystaledit/editlib/ccrystaltextview.cpp b/Externals/crystaledit/editlib/ccrystaltextview.cpp index 26f3b35c644..23defd01b2f 100644 --- a/Externals/crystaledit/editlib/ccrystaltextview.cpp +++ b/Externals/crystaledit/editlib/ccrystaltextview.cpp @@ -1459,7 +1459,7 @@ GetParseCookie (int nLineIndex) if (L > 0) dwCookie = (*m_ParseCookies)[L - 1]; ASSERT (dwCookie != - 1); - (*m_ParseCookies)[L] = ParseLine (dwCookie, GetLineChars(L), GetLineLength(L), nullptr, nBlocks); + (*m_ParseCookies)[L] = ParseLine (dwCookie, L, nullptr, nBlocks, m_pTextBuffer->GetParseContext()); ASSERT ((*m_ParseCookies)[L] != - 1); L++; } @@ -1939,7 +1939,7 @@ CCrystalTextView::GetTextBlocks(int nLineIndex) blocks[0].m_nColorIndex = COLORINDEX_NORMALTEXT; blocks[0].m_nBgColorIndex = COLORINDEX_BKGND; nBlocks++; - (*m_ParseCookies)[nLineIndex] = ParseLine(dwCookie, GetLineChars(nLineIndex), GetLineLength(nLineIndex), blocks.data(), nBlocks); + (*m_ParseCookies)[nLineIndex] = ParseLine(dwCookie, nLineIndex, blocks.data(), nBlocks, m_pTextBuffer->GetParseContext()); ASSERT((*m_ParseCookies)[nLineIndex] != -1 && nBlocks < static_cast(blocks.size())); blocks.resize(nBlocks); @@ -4775,9 +4775,9 @@ OnSetFocus (CWnd * pOldWnd) } unsigned CCrystalTextView:: -ParseLine (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +ParseLine (unsigned dwCookie, int nLineIndex, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { - return m_CurSourceDef->ParseLineX (dwCookie, pszChars, nLength, pBuf, nActualItems); + return m_CurSourceDef->ParseLineX (dwCookie, nLineIndex, GetLineChars(nLineIndex), GetLineLength(nLineIndex), pBuf, nActualItems, pContext); } int CCrystalTextView:: diff --git a/Externals/crystaledit/editlib/ccrystaltextview.h b/Externals/crystaledit/editlib/ccrystaltextview.h index 8eb80c44ab0..800d28121e0 100644 --- a/Externals/crystaledit/editlib/ccrystaltextview.h +++ b/Externals/crystaledit/editlib/ccrystaltextview.h @@ -607,6 +607,7 @@ protected : */ virtual void InvalidateLineCache( int nLineIndex1, int nLineIndex2 ); virtual void InvalidateSubLineIndexCache( int nLineIndex1 ); + void InvalidateParseCookies() { m_ParseCookies->clear(); } void InvalidateScreenRect(bool bInvalidateView = true); void InvalidateVertScrollBar(); void InvalidateHorzScrollBar(); @@ -695,7 +696,7 @@ protected : public : void GoToLine (int nLine, bool bRelative); - unsigned ParseLine (unsigned dwCookie, const tchar_t *pszChars, int nLength, CrystalLineParser::TEXTBLOCK * pBuf, int &nActualItems); + unsigned ParseLine (unsigned dwCookie, int nLineIndex, CrystalLineParser::TEXTBLOCK * pBuf, int &nActualItems, void *pContext); // Attributes public : diff --git a/Externals/crystaledit/editlib/editlib.vcxitems.filters b/Externals/crystaledit/editlib/editlib.vcxitems.filters index 03f50f5a66b..3cf534ecd9f 100644 --- a/Externals/crystaledit/editlib/editlib.vcxitems.filters +++ b/Externals/crystaledit/editlib/editlib.vcxitems.filters @@ -1,209 +1,209 @@ - - - - - {751f47a6-029e-4270-ae4b-0934e924e6ac} - - - {130c92db-be9c-4f56-9280-35528bd2d346} - - - {140ff583-705c-43c8-9970-86fe11cd2513} - - - {3fe0d48d-7410-4c0b-83dc-0c1d7a146ea9} - - - {f9221a78-ee81-4242-b77b-d48189ff3f1c} - - - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Utils - - - Utils - - - Utils - - - Utils - - - Utils - - - Utils - - - Renderers - - - Renderers - - - Dialogs - - - Dialogs - - - Dialogs - - - Dialogs - - - Dialogs - - - Dialogs - - - Utils - - - - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Renderers - - - Renderers - - - Renderers - - - Dialogs - - - Dialogs - - - Dialogs - - - Dialogs - - - Dialogs - - - Dialogs - - - Header Files - - - Utils - - - Header Files - - - Utils - - - Utils - - - Utils - - - Utils - - - Utils - - - Utils - - - - Header Files - - + + + + + {751f47a6-029e-4270-ae4b-0934e924e6ac} + + + {130c92db-be9c-4f56-9280-35528bd2d346} + + + {140ff583-705c-43c8-9970-86fe11cd2513} + + + {3fe0d48d-7410-4c0b-83dc-0c1d7a146ea9} + + + {f9221a78-ee81-4242-b77b-d48189ff3f1c} + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Utils + + + Utils + + + Utils + + + Utils + + + Utils + + + Utils + + + Renderers + + + Renderers + + + Dialogs + + + Dialogs + + + Dialogs + + + Dialogs + + + Dialogs + + + Dialogs + + + Utils + + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Renderers + + + Renderers + + + Renderers + + + Dialogs + + + Dialogs + + + Dialogs + + + Dialogs + + + Dialogs + + + Dialogs + + + Header Files + + + Utils + + + Header Files + + + Utils + + + Utils + + + Utils + + + Utils + + + Utils + + + Utils + + + + Header Files + + \ No newline at end of file diff --git a/Externals/crystaledit/editlib/editlibparsers.vcxitems b/Externals/crystaledit/editlib/editlibparsers.vcxitems index 47fe6073211..added2ff775 100644 --- a/Externals/crystaledit/editlib/editlibparsers.vcxitems +++ b/Externals/crystaledit/editlib/editlibparsers.vcxitems @@ -202,6 +202,14 @@ pch.h $(IntDir)$(TargetName)2.pch + + stdafx.h + $(IntDir)$(TargetName).pch + + + pch.h + $(IntDir)$(TargetName)2.pch + pch.h $(IntDir)$(TargetName)2.pch @@ -213,6 +221,8 @@ + + diff --git a/Externals/crystaledit/editlib/editlibparsers.vcxitems.filters b/Externals/crystaledit/editlib/editlibparsers.vcxitems.filters index c340bc2ba8b..1f0d6e103d7 100644 --- a/Externals/crystaledit/editlib/editlibparsers.vcxitems.filters +++ b/Externals/crystaledit/editlib/editlibparsers.vcxitems.filters @@ -162,6 +162,8 @@ Utils + + @@ -173,5 +175,7 @@ Utils + + \ No newline at end of file diff --git a/Externals/crystaledit/editlib/parsers/abap.cpp b/Externals/crystaledit/editlib/parsers/abap.cpp index 3283960aefd..654a96e39a2 100644 --- a/Externals/crystaledit/editlib/parsers/abap.cpp +++ b/Externals/crystaledit/editlib/parsers/abap.cpp @@ -1065,7 +1065,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineAbap(unsigned dwCookie, const tchar_t* pszChars, int nLength, TEXTBLOCK* pBuf, int& nActualItems) +CrystalLineParser::ParseLineAbap(unsigned dwCookie, int nLineIndex, const tchar_t* pszChars, int nLength, TEXTBLOCK* pBuf, int& nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/ada.cpp b/Externals/crystaledit/editlib/parsers/ada.cpp index 93d62fd4cf6..97c54ba5f18 100644 --- a/Externals/crystaledit/editlib/parsers/ada.cpp +++ b/Externals/crystaledit/editlib/parsers/ada.cpp @@ -439,7 +439,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineAda (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineAda (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/asp.cpp b/Externals/crystaledit/editlib/parsers/asp.cpp index e813e956036..715e4121929 100644 --- a/Externals/crystaledit/editlib/parsers/asp.cpp +++ b/Externals/crystaledit/editlib/parsers/asp.cpp @@ -18,7 +18,7 @@ #include "crystallineparser.h" unsigned -CrystalLineParser::ParseLineAsp (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineAsp (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { - return ParseLineHtmlEx(dwCookie, pszChars, nLength, pBuf, nActualItems, SRC_BASIC); + return ParseLineHtmlEx(dwCookie, nLineIndex, pszChars, nLength, pBuf, nActualItems, SRC_BASIC, pContext); } diff --git a/Externals/crystaledit/editlib/parsers/autoit.cpp b/Externals/crystaledit/editlib/parsers/autoit.cpp index 0135e786bb2..b00e91324ad 100644 --- a/Externals/crystaledit/editlib/parsers/autoit.cpp +++ b/Externals/crystaledit/editlib/parsers/autoit.cpp @@ -522,7 +522,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineAutoIt (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineAutoIt (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/basic.cpp b/Externals/crystaledit/editlib/parsers/basic.cpp index 42eb61ba40b..8770306937b 100644 --- a/Externals/crystaledit/editlib/parsers/basic.cpp +++ b/Externals/crystaledit/editlib/parsers/basic.cpp @@ -347,7 +347,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineBasic (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineBasic (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/batch.cpp b/Externals/crystaledit/editlib/parsers/batch.cpp index d27ff45ba3f..63e2d839c26 100644 --- a/Externals/crystaledit/editlib/parsers/batch.cpp +++ b/Externals/crystaledit/editlib/parsers/batch.cpp @@ -714,7 +714,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineBatch (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineBatch (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie; diff --git a/Externals/crystaledit/editlib/parsers/cplusplus.cpp b/Externals/crystaledit/editlib/parsers/cplusplus.cpp index db1f0ac00ad..5e85986727c 100644 --- a/Externals/crystaledit/editlib/parsers/cplusplus.cpp +++ b/Externals/crystaledit/editlib/parsers/cplusplus.cpp @@ -317,15 +317,16 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } } unsigned -CrystalLineParser::ParseLineC (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineC (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { - return ParseLineCJava (dwCookie, pszChars, nLength, pBuf, nActualItems, IsCppKeyword, IsUser1Keyword); + return ParseLineCJava (dwCookie, nLineIndex, pszChars, nLength, pBuf, nActualItems, IsCppKeyword, IsUser1Keyword, pContext); } unsigned -CrystalLineParser::ParseLineCJava (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, +CrystalLineParser::ParseLineCJava (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, bool (*IsKeyword)(const tchar_t *pszChars, int nLength), - bool (*IsUser1Keyword)(const tchar_t *pszChars, int nLength)) + bool (*IsUser1Keyword)(const tchar_t *pszChars, int nLength), + void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/crystallineparser.h b/Externals/crystaledit/editlib/parsers/crystallineparser.h index b2f1e897140..316086df0d6 100644 --- a/Externals/crystaledit/editlib/parsers/crystallineparser.h +++ b/Externals/crystaledit/editlib/parsers/crystallineparser.h @@ -136,7 +136,7 @@ struct TextDefinition const tchar_t* name; tchar_t* exts; bool extsIsDynamic; - unsigned (* ParseLineX) (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); + unsigned (* ParseLineX) (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); unsigned flags; // unsigned tabsize; tchar_t opencomment[8]; @@ -157,54 +157,54 @@ TextDefinition *GetTextType(const tchar_t *pszExt); TextDefinition* GetTextType(int index); void SetExtension(int index, const tchar_t *pszExts); -unsigned ParseLinePlain(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineAbap(unsigned dwCookie, const tchar_t* pszChars, int nLength, TEXTBLOCK* pBuf, int& nActualItems); -unsigned ParseLineAda(unsigned dwCookie, const tchar_t* pszChars, int nLength, TEXTBLOCK* pBuf, int& nActualItems); -unsigned ParseLineAsp(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineAutoIt(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineBasic(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineBatch(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineC(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineCJava(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, bool (*IsKeyword)(const tchar_t *pszChars, int nLength), bool (*IsUser1Keyword)(const tchar_t *pszChars, int nLength)); -unsigned ParseLineCSharp(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineCss(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineDcl(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineDlang(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineFortran(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineFSharp(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineGo(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineHtml(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineHtmlEx(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, int nEmbeddedLanguage); -unsigned ParseLineIni(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineInnoSetup(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineIS(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineJava(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineJavaScript(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineLisp(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineLua(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineMatlab(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineNsis(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLinePascal(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLinePerl(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLinePhp(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLinePhpLanguage(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLinePo(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLinePowerShell(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLinePython(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineRexx(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineRsrc(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineRuby(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineRust(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineSgml(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineSh(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineSiod(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineSmarty(unsigned dwCookie, const tchar_t* pszChars, int nLength, TEXTBLOCK* pBuf, int& nActualItems); -unsigned ParseLineSmartyLanguage(unsigned dwCookie, const tchar_t* pszChars, int nLength, TEXTBLOCK* pBuf, int& nActualItems); -unsigned ParseLineSql(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineTcl(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineTex(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineVerilog(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineVhdl(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); -unsigned ParseLineXml(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems); +unsigned ParseLinePlain(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineAbap(unsigned dwCookie, int nLineIndex, const tchar_t* pszChars, int nLength, TEXTBLOCK* pBuf, int& nActualItems, void *pContext); +unsigned ParseLineAda(unsigned dwCookie, int nLineIndex, const tchar_t* pszChars, int nLength, TEXTBLOCK* pBuf, int& nActualItems, void *pContext); +unsigned ParseLineAsp(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineAutoIt(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineBasic(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineBatch(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineC(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineCJava(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, bool (*IsKeyword)(const tchar_t *pszChars, int nLength), bool (*IsUser1Keyword)(const tchar_t *pszChars, int nLength), void *pContext); +unsigned ParseLineCSharp(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineCss(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineDcl(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineDlang(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineFortran(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineFSharp(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineGo(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineHtml(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineHtmlEx(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, int nEmbeddedLanguage, void *pContext); +unsigned ParseLineIni(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineInnoSetup(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineIS(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineJava(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineJavaScript(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineLisp(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineLua(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineMatlab(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineNsis(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLinePascal(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLinePerl(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLinePhp(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLinePhpLanguage(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLinePo(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLinePowerShell(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLinePython(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineRexx(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineRsrc(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineRuby(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineRust(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineSgml(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineSh(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineSiod(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineSmarty(unsigned dwCookie, int nLineIndex, const tchar_t* pszChars, int nLength, TEXTBLOCK* pBuf, int& nActualItems, void *pContext); +unsigned ParseLineSmartyLanguage(unsigned dwCookie, int nLineIndex, const tchar_t* pszChars, int nLength, TEXTBLOCK* pBuf, int& nActualItems, void *pContext); +unsigned ParseLineSql(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineTcl(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineTex(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineVerilog(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineVhdl(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); +unsigned ParseLineXml(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext); } diff --git a/Externals/crystaledit/editlib/parsers/csharp.cpp b/Externals/crystaledit/editlib/parsers/csharp.cpp index 9656b43528b..2cd2ec50078 100644 --- a/Externals/crystaledit/editlib/parsers/csharp.cpp +++ b/Externals/crystaledit/editlib/parsers/csharp.cpp @@ -134,7 +134,7 @@ IsCppKeyword (const tchar_t *pszChars, int nLength) } unsigned -CrystalLineParser::ParseLineCSharp (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineCSharp (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { - return ParseLineCJava (dwCookie, pszChars, nLength, pBuf, nActualItems, IsCppKeyword, nullptr); + return ParseLineCJava (dwCookie, nLineIndex, pszChars, nLength, pBuf, nActualItems, IsCppKeyword, nullptr, pContext); } diff --git a/Externals/crystaledit/editlib/parsers/css.cpp b/Externals/crystaledit/editlib/parsers/css.cpp index 1163b976776..93469033ce5 100644 --- a/Externals/crystaledit/editlib/parsers/css.cpp +++ b/Externals/crystaledit/editlib/parsers/css.cpp @@ -534,7 +534,7 @@ IsCssExKeyword(const tchar_t *pszChars, int nLength) } unsigned -CrystalLineParser::ParseLineCss (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineCss (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & (COOKIE_EXT_COMMENT|COOKIE_EXT_DEFINITION|COOKIE_EXT_VALUE); diff --git a/Externals/crystaledit/editlib/parsers/dcl.cpp b/Externals/crystaledit/editlib/parsers/dcl.cpp index e22d8e621bf..2aa380cd31f 100644 --- a/Externals/crystaledit/editlib/parsers/dcl.cpp +++ b/Externals/crystaledit/editlib/parsers/dcl.cpp @@ -179,7 +179,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineDcl (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineDcl (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/dlang.cpp b/Externals/crystaledit/editlib/parsers/dlang.cpp index 924ed633f1f..b61ddd32899 100644 --- a/Externals/crystaledit/editlib/parsers/dlang.cpp +++ b/Externals/crystaledit/editlib/parsers/dlang.cpp @@ -200,7 +200,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineDlang (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineDlang (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & (COOKIE_EXT_COMMENT | COOKIE_RAWSTRING | COOKIE_STRING | 0xFF000000); diff --git a/Externals/crystaledit/editlib/parsers/fortran.cpp b/Externals/crystaledit/editlib/parsers/fortran.cpp index 20ce2531493..0fe2694c18c 100644 --- a/Externals/crystaledit/editlib/parsers/fortran.cpp +++ b/Externals/crystaledit/editlib/parsers/fortran.cpp @@ -253,7 +253,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineFortran (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineFortran (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/fsharp.cpp b/Externals/crystaledit/editlib/parsers/fsharp.cpp index a1d7198055b..36b2a8d96c2 100644 --- a/Externals/crystaledit/editlib/parsers/fsharp.cpp +++ b/Externals/crystaledit/editlib/parsers/fsharp.cpp @@ -233,7 +233,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineFSharp (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineFSharp (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & (COOKIE_EXT_COMMENT | COOKIE_RAWSTRING); diff --git a/Externals/crystaledit/editlib/parsers/go.cpp b/Externals/crystaledit/editlib/parsers/go.cpp index 79780429dd7..1ab7cc5d81f 100644 --- a/Externals/crystaledit/editlib/parsers/go.cpp +++ b/Externals/crystaledit/editlib/parsers/go.cpp @@ -146,7 +146,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineGo (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineGo (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & (COOKIE_EXT_COMMENT | COOKIE_RAWSTRING); diff --git a/Externals/crystaledit/editlib/parsers/html.cpp b/Externals/crystaledit/editlib/parsers/html.cpp index 2ec035c7ec8..d9c82da0f46 100644 --- a/Externals/crystaledit/editlib/parsers/html.cpp +++ b/Externals/crystaledit/editlib/parsers/html.cpp @@ -30,7 +30,7 @@ static void AdjustCharPosInTextBlocks(CrystalLineParser::TEXTBLOCK* pBuf, int st } unsigned -CrystalLineParser::ParseLineHtmlEx (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, int nEmbeddedLanguage) +CrystalLineParser::ParseLineHtmlEx (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, int nEmbeddedLanguage, void *pContext) { if (nLength == 0) { @@ -102,7 +102,7 @@ CrystalLineParser::ParseLineHtmlEx (unsigned dwCookie, const tchar_t *pszChars, const tchar_t *pszEnd = tc::tcsstr(pszChars + I, _T("")); int nextI = pszEnd ? static_cast(pszEnd - pszChars) : nLength; int nActualItemsEmbedded = 0; - dwCookie = ParseLineJavaScript(dwCookie & ~COOKIE_BLOCK_SCRIPT, pszChars + I, nextI - I, pBuf + nActualItems, nActualItemsEmbedded); + dwCookie = ParseLineJavaScript(dwCookie & ~COOKIE_BLOCK_SCRIPT, nLineIndex, pszChars + I, nextI - I, pBuf + nActualItems, nActualItemsEmbedded, pContext); AdjustCharPosInTextBlocks(pBuf, nActualItems, nActualItems + nActualItemsEmbedded - 1, I); nActualItems += nActualItemsEmbedded; if (!pszEnd) @@ -119,7 +119,7 @@ CrystalLineParser::ParseLineHtmlEx (unsigned dwCookie, const tchar_t *pszChars, const tchar_t *pszEnd = tc::tcsstr(pszChars + I, _T("")); int nextI = pszEnd ? static_cast(pszEnd - pszChars) : nLength; int nActualItemsEmbedded = 0; - dwCookie = ParseLineCss(dwCookie & ~COOKIE_BLOCK_STYLE, pszChars + I, nextI - I, pBuf + nActualItems, nActualItemsEmbedded); + dwCookie = ParseLineCss(dwCookie & ~COOKIE_BLOCK_STYLE, nLineIndex, pszChars + I, nextI - I, pBuf + nActualItems, nActualItemsEmbedded, pContext); AdjustCharPosInTextBlocks(pBuf, nActualItems, nActualItems + nActualItemsEmbedded - 1, I); nActualItems += nActualItemsEmbedded; if (!pszEnd) @@ -139,7 +139,7 @@ CrystalLineParser::ParseLineHtmlEx (unsigned dwCookie, const tchar_t *pszChars, if (!pszEnd) pszEnd = tc::tcsstr(pszChars + I, _T("%>")); int nextI = pszEnd ? static_cast(pszEnd - pszChars) : nLength; - unsigned (*pParseLineFunc)(unsigned, const tchar_t *, int, TEXTBLOCK *, int &); + unsigned (*pParseLineFunc)(unsigned, int, const tchar_t *, int, TEXTBLOCK *, int &, void *); switch (nEmbeddedLanguage) { case SRC_BASIC: pParseLineFunc = ParseLineBasic; break; @@ -147,7 +147,7 @@ CrystalLineParser::ParseLineHtmlEx (unsigned dwCookie, const tchar_t *pszChars, default: pParseLineFunc = ParseLineJavaScript; break; } int nActualItemsEmbedded = 0; - dwCookie = pParseLineFunc(dwCookie & ~COOKIE_EXT_USER1, pszChars + I, nextI - I, pBuf + nActualItems, nActualItemsEmbedded); + dwCookie = pParseLineFunc(dwCookie & ~COOKIE_EXT_USER1, nLineIndex, pszChars + I, nextI - I, pBuf + nActualItems, nActualItemsEmbedded, pContext); AdjustCharPosInTextBlocks(pBuf, nActualItems, nActualItems + nActualItemsEmbedded - 1, I); nActualItems += nActualItemsEmbedded; if (!pszEnd) @@ -185,7 +185,7 @@ CrystalLineParser::ParseLineHtmlEx (unsigned dwCookie, const tchar_t *pszChars, int nextI = pszEnd ? static_cast(pszEnd - pszChars) : nLength; int nActualItemsEmbedded = 0; int nOffset = (I > 0 && pszChars[I - 1] == '{') ? (I - 1) : I; - dwCookie = ParseLineSmartyLanguage(dwCookie & ~COOKIE_EXT_USER1, pszChars + nOffset, nextI - nOffset + 1, pBuf + nActualItems, nActualItemsEmbedded); + dwCookie = ParseLineSmartyLanguage(dwCookie & ~COOKIE_EXT_USER1, nLineIndex, pszChars + nOffset, nextI - nOffset + 1, pBuf + nActualItems, nActualItemsEmbedded, pContext); AdjustCharPosInTextBlocks(pBuf, nActualItems, nActualItems + nActualItemsEmbedded - 1, nOffset); nActualItems += nActualItemsEmbedded; if (!pszEnd) @@ -433,7 +433,7 @@ CrystalLineParser::ParseLineHtmlEx (unsigned dwCookie, const tchar_t *pszChars, } unsigned -CrystalLineParser::ParseLineHtml (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineHtml (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { - return ParseLineHtmlEx(dwCookie, pszChars, nLength, pBuf, nActualItems, SRC_JAVA); + return ParseLineHtmlEx(dwCookie, nLineIndex, pszChars, nLength, pBuf, nActualItems, SRC_JAVA, pContext); } diff --git a/Externals/crystaledit/editlib/parsers/ini.cpp b/Externals/crystaledit/editlib/parsers/ini.cpp index e18d3ae95ee..671ba33704d 100644 --- a/Externals/crystaledit/editlib/parsers/ini.cpp +++ b/Externals/crystaledit/editlib/parsers/ini.cpp @@ -34,7 +34,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineIni (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineIni (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/innosetup.cpp b/Externals/crystaledit/editlib/parsers/innosetup.cpp index 8cfd9141079..14681586fb9 100644 --- a/Externals/crystaledit/editlib/parsers/innosetup.cpp +++ b/Externals/crystaledit/editlib/parsers/innosetup.cpp @@ -335,7 +335,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineInnoSetup (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineInnoSetup (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & (COOKIE_EXT_USER1 | COOKIE_EXT_COMMENT | COOKIE_EXT_COMMENT2); @@ -343,7 +343,7 @@ CrystalLineParser::ParseLineInnoSetup (unsigned dwCookie, const tchar_t *pszChar bool bCodeSection; if ((dwCookie & COOKIE_EXT_USER1) && !IsLineSection(pszChars, nLength, bCodeSection)) { - dwCookie = CrystalLineParser::ParseLinePascal(dwCookie & ~COOKIE_EXT_USER1, pszChars, nLength, pBuf, nActualItems); + dwCookie = CrystalLineParser::ParseLinePascal(dwCookie & ~COOKIE_EXT_USER1, nLineIndex, pszChars, nLength, pBuf, nActualItems, pContext); return dwCookie | COOKIE_EXT_USER1; } dwCookie = 0; diff --git a/Externals/crystaledit/editlib/parsers/is.cpp b/Externals/crystaledit/editlib/parsers/is.cpp index 29934c27078..3640753e97e 100644 --- a/Externals/crystaledit/editlib/parsers/is.cpp +++ b/Externals/crystaledit/editlib/parsers/is.cpp @@ -471,7 +471,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineIS (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineIS (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/java.cpp b/Externals/crystaledit/editlib/parsers/java.cpp index 3ec13435e10..bbc520a276d 100644 --- a/Externals/crystaledit/editlib/parsers/java.cpp +++ b/Externals/crystaledit/editlib/parsers/java.cpp @@ -91,7 +91,7 @@ IsJavaKeyword (const tchar_t *pszChars, int nLength) } unsigned -CrystalLineParser::ParseLineJava (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineJava (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { - return ParseLineCJava (dwCookie, pszChars, nLength, pBuf, nActualItems, IsJavaKeyword, nullptr); + return ParseLineCJava (dwCookie, nLineIndex, pszChars, nLength, pBuf, nActualItems, IsJavaKeyword, nullptr, pContext); } diff --git a/Externals/crystaledit/editlib/parsers/javascript.cpp b/Externals/crystaledit/editlib/parsers/javascript.cpp index ad7af9f0d5c..4c71df61912 100644 --- a/Externals/crystaledit/editlib/parsers/javascript.cpp +++ b/Externals/crystaledit/editlib/parsers/javascript.cpp @@ -87,7 +87,7 @@ IsJavaScriptKeyword (const tchar_t *pszChars, int nLength) } unsigned -CrystalLineParser::ParseLineJavaScript (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineJavaScript (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { - return ParseLineCJava (dwCookie, pszChars, nLength, pBuf, nActualItems, IsJavaScriptKeyword, nullptr); + return ParseLineCJava (dwCookie, nLineIndex, pszChars, nLength, pBuf, nActualItems, IsJavaScriptKeyword, nullptr, pContext); } diff --git a/Externals/crystaledit/editlib/parsers/lisp.cpp b/Externals/crystaledit/editlib/parsers/lisp.cpp index 34b4a39e03e..7db76914440 100644 --- a/Externals/crystaledit/editlib/parsers/lisp.cpp +++ b/Externals/crystaledit/editlib/parsers/lisp.cpp @@ -315,7 +315,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineLisp (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineLisp (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/lua.cpp b/Externals/crystaledit/editlib/parsers/lua.cpp index d163afe4648..bbc4b9e95ca 100644 --- a/Externals/crystaledit/editlib/parsers/lua.cpp +++ b/Externals/crystaledit/editlib/parsers/lua.cpp @@ -90,7 +90,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineLua (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineLua (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & (COOKIE_EXT_COMMENT | COOKIE_RAWSTRING | 0xFF000000); diff --git a/Externals/crystaledit/editlib/parsers/matlab.cpp b/Externals/crystaledit/editlib/parsers/matlab.cpp index fe64820eba1..da49a048e0c 100644 --- a/Externals/crystaledit/editlib/parsers/matlab.cpp +++ b/Externals/crystaledit/editlib/parsers/matlab.cpp @@ -121,7 +121,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineMatlab (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineMatlab (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & (COOKIE_EXT_COMMENT | 0xFF000000); diff --git a/Externals/crystaledit/editlib/parsers/nsis.cpp b/Externals/crystaledit/editlib/parsers/nsis.cpp index 42bb1b0275c..2be35c4aa37 100644 --- a/Externals/crystaledit/editlib/parsers/nsis.cpp +++ b/Externals/crystaledit/editlib/parsers/nsis.cpp @@ -472,7 +472,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineNsis (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineNsis (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/pascal.cpp b/Externals/crystaledit/editlib/parsers/pascal.cpp index f7fd085dfea..0eefce03b2f 100644 --- a/Externals/crystaledit/editlib/parsers/pascal.cpp +++ b/Externals/crystaledit/editlib/parsers/pascal.cpp @@ -148,7 +148,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLinePascal (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLinePascal (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & (COOKIE_EXT_COMMENT | COOKIE_EXT_COMMENT2 | COOKIE_RAWSTRING); diff --git a/Externals/crystaledit/editlib/parsers/perl.cpp b/Externals/crystaledit/editlib/parsers/perl.cpp index bf8f8ee5322..ab1bb096123 100644 --- a/Externals/crystaledit/editlib/parsers/perl.cpp +++ b/Externals/crystaledit/editlib/parsers/perl.cpp @@ -290,7 +290,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLinePerl (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLinePerl (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/php.cpp b/Externals/crystaledit/editlib/parsers/php.cpp index 2d264df0fa7..9421606ad2c 100644 --- a/Externals/crystaledit/editlib/parsers/php.cpp +++ b/Externals/crystaledit/editlib/parsers/php.cpp @@ -299,13 +299,13 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLinePhp(unsigned dwCookie, const tchar_t* pszChars, int nLength, TEXTBLOCK* pBuf, int& nActualItems) +CrystalLineParser::ParseLinePhp(unsigned dwCookie, int nLineIndex, const tchar_t* pszChars, int nLength, TEXTBLOCK* pBuf, int& nActualItems, void *pContext) { - return ParseLineHtmlEx(dwCookie, pszChars, nLength, pBuf, nActualItems, SRC_PHP); + return ParseLineHtmlEx(dwCookie, nLineIndex, pszChars, nLength, pBuf, nActualItems, SRC_PHP, pContext); } unsigned -CrystalLineParser::ParseLinePhpLanguage (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLinePhpLanguage (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & (COOKIE_EXT_COMMENT | COOKIE_STRING | COOKIE_CHAR); diff --git a/Externals/crystaledit/editlib/parsers/plain.cpp b/Externals/crystaledit/editlib/parsers/plain.cpp index 959f773eff0..dc1ca25e789 100644 --- a/Externals/crystaledit/editlib/parsers/plain.cpp +++ b/Externals/crystaledit/editlib/parsers/plain.cpp @@ -2,7 +2,7 @@ #include "crystallineparser.h" unsigned -CrystalLineParser::ParseLinePlain (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLinePlain (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { return 0; } diff --git a/Externals/crystaledit/editlib/parsers/po.cpp b/Externals/crystaledit/editlib/parsers/po.cpp index 15a3c57811a..2b2a66f8ea4 100644 --- a/Externals/crystaledit/editlib/parsers/po.cpp +++ b/Externals/crystaledit/editlib/parsers/po.cpp @@ -86,7 +86,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLinePo (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLinePo (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/powershell.cpp b/Externals/crystaledit/editlib/parsers/powershell.cpp index 2e27c20044b..32a5e1a74df 100644 --- a/Externals/crystaledit/editlib/parsers/powershell.cpp +++ b/Externals/crystaledit/editlib/parsers/powershell.cpp @@ -359,7 +359,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLinePowerShell (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLinePowerShell (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/python.cpp b/Externals/crystaledit/editlib/parsers/python.cpp index 44876ce27d4..80af51bd746 100644 --- a/Externals/crystaledit/editlib/parsers/python.cpp +++ b/Externals/crystaledit/editlib/parsers/python.cpp @@ -434,7 +434,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLinePython (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLinePython (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & (COOKIE_EXT_COMMENT | COOKIE_RAWSTRING | 0xFF000000); diff --git a/Externals/crystaledit/editlib/parsers/rexx.cpp b/Externals/crystaledit/editlib/parsers/rexx.cpp index 79d78c4da14..98a0d4f174e 100644 --- a/Externals/crystaledit/editlib/parsers/rexx.cpp +++ b/Externals/crystaledit/editlib/parsers/rexx.cpp @@ -149,7 +149,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineRexx (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineRexx (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/rsrc.cpp b/Externals/crystaledit/editlib/parsers/rsrc.cpp index 69e1bbc541b..8510e3324f0 100644 --- a/Externals/crystaledit/editlib/parsers/rsrc.cpp +++ b/Externals/crystaledit/editlib/parsers/rsrc.cpp @@ -213,7 +213,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineRsrc (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineRsrc (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/ruby.cpp b/Externals/crystaledit/editlib/parsers/ruby.cpp index 4fc063d7ed0..d85291025b2 100644 --- a/Externals/crystaledit/editlib/parsers/ruby.cpp +++ b/Externals/crystaledit/editlib/parsers/ruby.cpp @@ -158,7 +158,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineRuby(unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineRuby(unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/rust.cpp b/Externals/crystaledit/editlib/parsers/rust.cpp index 032c281ac29..81daa6bbddf 100644 --- a/Externals/crystaledit/editlib/parsers/rust.cpp +++ b/Externals/crystaledit/editlib/parsers/rust.cpp @@ -161,7 +161,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineRust (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineRust (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & (COOKIE_EXT_COMMENT | COOKIE_RAWSTRING | COOKIE_STRING | 0xFF000000); diff --git a/Externals/crystaledit/editlib/parsers/sgml.cpp b/Externals/crystaledit/editlib/parsers/sgml.cpp index 4d125f422ef..e745c3d4568 100644 --- a/Externals/crystaledit/editlib/parsers/sgml.cpp +++ b/Externals/crystaledit/editlib/parsers/sgml.cpp @@ -104,7 +104,7 @@ IsUser1Keyword (const tchar_t *pszChars, int nLength) } unsigned -CrystalLineParser::ParseLineSgml (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineSgml (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/sh.cpp b/Externals/crystaledit/editlib/parsers/sh.cpp index b83cb379bc5..6c7d625161a 100644 --- a/Externals/crystaledit/editlib/parsers/sh.cpp +++ b/Externals/crystaledit/editlib/parsers/sh.cpp @@ -118,7 +118,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineSh (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineSh (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/siod.cpp b/Externals/crystaledit/editlib/parsers/siod.cpp index 7159ac20eb3..31007a75383 100644 --- a/Externals/crystaledit/editlib/parsers/siod.cpp +++ b/Externals/crystaledit/editlib/parsers/siod.cpp @@ -399,7 +399,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineSiod (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineSiod (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/smarty.cpp b/Externals/crystaledit/editlib/parsers/smarty.cpp index cb0e926c4df..096b94dd533 100644 --- a/Externals/crystaledit/editlib/parsers/smarty.cpp +++ b/Externals/crystaledit/editlib/parsers/smarty.cpp @@ -211,13 +211,13 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineSmarty(unsigned dwCookie, const tchar_t* pszChars, int nLength, TEXTBLOCK* pBuf, int& nActualItems) +CrystalLineParser::ParseLineSmarty(unsigned dwCookie, int nLineIndex, const tchar_t* pszChars, int nLength, TEXTBLOCK* pBuf, int& nActualItems, void *pContext) { - return ParseLineHtmlEx(dwCookie, pszChars, nLength, pBuf, nActualItems, SRC_SMARTY); + return ParseLineHtmlEx(dwCookie, nLineIndex, pszChars, nLength, pBuf, nActualItems, SRC_SMARTY, pContext); } unsigned -CrystalLineParser::ParseLineSmartyLanguage (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineSmartyLanguage (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & (COOKIE_EXT_COMMENT | COOKIE_STRING | COOKIE_CHAR); diff --git a/Externals/crystaledit/editlib/parsers/sql.cpp b/Externals/crystaledit/editlib/parsers/sql.cpp index a268dc2230e..d1d6dcab911 100644 --- a/Externals/crystaledit/editlib/parsers/sql.cpp +++ b/Externals/crystaledit/editlib/parsers/sql.cpp @@ -491,7 +491,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineSql (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineSql (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/tcl.cpp b/Externals/crystaledit/editlib/parsers/tcl.cpp index 620433886c2..35753c22cbf 100644 --- a/Externals/crystaledit/editlib/parsers/tcl.cpp +++ b/Externals/crystaledit/editlib/parsers/tcl.cpp @@ -82,7 +82,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineTcl (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineTcl (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/tex.cpp b/Externals/crystaledit/editlib/parsers/tex.cpp index 66abf001b81..dcd724c513b 100644 --- a/Externals/crystaledit/editlib/parsers/tex.cpp +++ b/Externals/crystaledit/editlib/parsers/tex.cpp @@ -895,7 +895,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineTex (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineTex (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/verilog.cpp b/Externals/crystaledit/editlib/parsers/verilog.cpp index 246b28f4d48..094d868f3f4 100644 --- a/Externals/crystaledit/editlib/parsers/verilog.cpp +++ b/Externals/crystaledit/editlib/parsers/verilog.cpp @@ -349,7 +349,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineVerilog (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineVerilog (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/vhdl.cpp b/Externals/crystaledit/editlib/parsers/vhdl.cpp index 62c70452c26..6d020bd90c0 100644 --- a/Externals/crystaledit/editlib/parsers/vhdl.cpp +++ b/Externals/crystaledit/editlib/parsers/vhdl.cpp @@ -363,7 +363,7 @@ DefineIdentiferBlock(const tchar_t *pszChars, int nLength, CrystalLineParser::TE } unsigned -CrystalLineParser::ParseLineVhdl (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineVhdl (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/parsers/xml.cpp b/Externals/crystaledit/editlib/parsers/xml.cpp index 8d066eec84f..b75dad1652f 100644 --- a/Externals/crystaledit/editlib/parsers/xml.cpp +++ b/Externals/crystaledit/editlib/parsers/xml.cpp @@ -73,7 +73,7 @@ IsUser1Keyword (const tchar_t *pszChars, int nLength) } unsigned -CrystalLineParser::ParseLineXml (unsigned dwCookie, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems) +CrystalLineParser::ParseLineXml (unsigned dwCookie, int nLineIndex, const tchar_t *pszChars, int nLength, TEXTBLOCK * pBuf, int &nActualItems, void *pContext) { if (nLength == 0) return dwCookie & COOKIE_EXT_COMMENT; diff --git a/Externals/crystaledit/editlib/tree-sitter/LICENSE b/Externals/crystaledit/editlib/tree-sitter/LICENSE new file mode 100644 index 00000000000..451fe1d26ee --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018-2024 Max Brunsfeld + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Externals/crystaledit/editlib/tree-sitter/README.md b/Externals/crystaledit/editlib/tree-sitter/README.md new file mode 100644 index 00000000000..d378215eb0e --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/README.md @@ -0,0 +1,21 @@ +# tree-sitter + +[![DOI](https://zenodo.org/badge/14164618.svg)](https://zenodo.org/badge/latestdoi/14164618) +[![discord][discord]](https://discord.gg/w7nTvsVJhm) +[![matrix][matrix]](https://matrix.to/#/#tree-sitter-chat:matrix.org) + +Tree-sitter is a parser generator tool and an incremental parsing library. It can build a concrete syntax tree for a source file and efficiently update the syntax tree as the source file is edited. Tree-sitter aims to be: + +- **General** enough to parse any programming language +- **Fast** enough to parse on every keystroke in a text editor +- **Robust** enough to provide useful results even in the presence of syntax errors +- **Dependency-free** so that the runtime library (which is written in pure C) can be embedded in any application + +## Links +- [Documentation](https://tree-sitter.github.io) +- [Rust binding](lib/binding_rust/README.md) +- [WASM binding](lib/binding_web/README.md) +- [Command-line interface](cli/README.md) + +[discord]: https://img.shields.io/discord/1063097320771698699?logo=discord&label=discord +[matrix]: https://img.shields.io/matrix/tree-sitter-chat%3Amatrix.org?logo=matrix&label=matrix diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/include/tree_sitter/api.h b/Externals/crystaledit/editlib/tree-sitter/lib/include/tree_sitter/api.h new file mode 100644 index 00000000000..2bbfe66ffe3 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/include/tree_sitter/api.h @@ -0,0 +1,1478 @@ +#ifndef TREE_SITTER_API_H_ +#define TREE_SITTER_API_H_ + +#ifndef TREE_SITTER_HIDE_SYMBOLS +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC visibility push(default) +#endif +#endif + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/****************************/ +/* Section - ABI Versioning */ +/****************************/ + +/** + * The latest ABI version that is supported by the current version of the + * library. When Languages are generated by the Tree-sitter CLI, they are + * assigned an ABI version number that corresponds to the current CLI version. + * The Tree-sitter library is generally backwards-compatible with languages + * generated using older CLI versions, but is not forwards-compatible. + */ +#define TREE_SITTER_LANGUAGE_VERSION 15 + +/** + * The earliest ABI version that is supported by the current version of the + * library. + */ +#define TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION 13 + +/*******************/ +/* Section - Types */ +/*******************/ + +typedef uint16_t TSStateId; +typedef uint16_t TSSymbol; +typedef uint16_t TSFieldId; +typedef struct TSLanguage TSLanguage; +typedef struct TSParser TSParser; +typedef struct TSTree TSTree; +typedef struct TSQuery TSQuery; +typedef struct TSQueryCursor TSQueryCursor; +typedef struct TSLookaheadIterator TSLookaheadIterator; + +// This function signature reads one code point from the given string, +// returning the number of bytes consumed. It should write the code point +// to the `code_point` pointer, or write -1 if the input is invalid. +typedef uint32_t (*DecodeFunction)( + const uint8_t *string, + uint32_t length, + int32_t *code_point +); + +typedef enum TSInputEncoding { + TSInputEncodingUTF8, + TSInputEncodingUTF16LE, + TSInputEncodingUTF16BE, + TSInputEncodingCustom +} TSInputEncoding; + +typedef enum TSSymbolType { + TSSymbolTypeRegular, + TSSymbolTypeAnonymous, + TSSymbolTypeSupertype, + TSSymbolTypeAuxiliary, +} TSSymbolType; + +typedef struct TSPoint { + uint32_t row; + uint32_t column; +} TSPoint; + +typedef struct TSRange { + TSPoint start_point; + TSPoint end_point; + uint32_t start_byte; + uint32_t end_byte; +} TSRange; + +typedef struct TSInput { + void *payload; + const char *(*read)(void *payload, uint32_t byte_index, TSPoint position, uint32_t *bytes_read); + TSInputEncoding encoding; + DecodeFunction decode; +} TSInput; + +typedef struct TSParseState { + void *payload; + uint32_t current_byte_offset; + bool has_error; +} TSParseState; + +typedef struct TSParseOptions { + void *payload; + bool (*progress_callback)(TSParseState *state); +} TSParseOptions; + +typedef enum TSLogType { + TSLogTypeParse, + TSLogTypeLex, +} TSLogType; + +typedef struct TSLogger { + void *payload; + void (*log)(void *payload, TSLogType log_type, const char *buffer); +} TSLogger; + +typedef struct TSInputEdit { + uint32_t start_byte; + uint32_t old_end_byte; + uint32_t new_end_byte; + TSPoint start_point; + TSPoint old_end_point; + TSPoint new_end_point; +} TSInputEdit; + +typedef struct TSNode { + uint32_t context[4]; + const void *id; + const TSTree *tree; +} TSNode; + +typedef struct TSTreeCursor { + const void *tree; + const void *id; + uint32_t context[3]; +} TSTreeCursor; + +typedef struct TSQueryCapture { + TSNode node; + uint32_t index; +} TSQueryCapture; + +typedef enum TSQuantifier { + TSQuantifierZero = 0, // must match the array initialization value + TSQuantifierZeroOrOne, + TSQuantifierZeroOrMore, + TSQuantifierOne, + TSQuantifierOneOrMore, +} TSQuantifier; + +typedef struct TSQueryMatch { + uint32_t id; + uint16_t pattern_index; + uint16_t capture_count; + const TSQueryCapture *captures; +} TSQueryMatch; + +typedef enum TSQueryPredicateStepType { + TSQueryPredicateStepTypeDone, + TSQueryPredicateStepTypeCapture, + TSQueryPredicateStepTypeString, +} TSQueryPredicateStepType; + +typedef struct TSQueryPredicateStep { + TSQueryPredicateStepType type; + uint32_t value_id; +} TSQueryPredicateStep; + +typedef enum TSQueryError { + TSQueryErrorNone = 0, + TSQueryErrorSyntax, + TSQueryErrorNodeType, + TSQueryErrorField, + TSQueryErrorCapture, + TSQueryErrorStructure, + TSQueryErrorLanguage, +} TSQueryError; + +typedef struct TSQueryCursorState { + void *payload; + uint32_t current_byte_offset; +} TSQueryCursorState; + +typedef struct TSQueryCursorOptions { + void *payload; + bool (*progress_callback)(TSQueryCursorState *state); +} TSQueryCursorOptions; + +/** + * The metadata associated with a language. + * + * Currently, this metadata can be used to check the [Semantic Version](https://semver.org/) + * of the language. This version information should be used to signal if a given parser might + * be incompatible with existing queries when upgrading between major versions, or minor versions + * if it's in zerover. + */ +typedef struct TSLanguageMetadata { + uint8_t major_version; + uint8_t minor_version; + uint8_t patch_version; +} TSLanguageMetadata; + +/********************/ +/* Section - Parser */ +/********************/ + +/** + * Create a new parser. + */ +TSParser *ts_parser_new(void); + +/** + * Delete the parser, freeing all of the memory that it used. + */ +void ts_parser_delete(TSParser *self); + +/** + * Get the parser's current language. + */ +const TSLanguage *ts_parser_language(const TSParser *self); + +/** + * Set the language that the parser should use for parsing. + * + * Returns a boolean indicating whether or not the language was successfully + * assigned. True means assignment succeeded. False means there was a version + * mismatch: the language was generated with an incompatible version of the + * Tree-sitter CLI. Check the language's ABI version using [`ts_language_abi_version`] + * and compare it to this library's [`TREE_SITTER_LANGUAGE_VERSION`] and + * [`TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION`] constants. + */ +bool ts_parser_set_language(TSParser *self, const TSLanguage *language); + +/** + * Set the ranges of text that the parser should include when parsing. + * + * By default, the parser will always include entire documents. This function + * allows you to parse only a *portion* of a document but still return a syntax + * tree whose ranges match up with the document as a whole. You can also pass + * multiple disjoint ranges. + * + * The second and third parameters specify the location and length of an array + * of ranges. The parser does *not* take ownership of these ranges; it copies + * the data, so it doesn't matter how these ranges are allocated. + * + * If `count` is zero, then the entire document will be parsed. Otherwise, + * the given ranges must be ordered from earliest to latest in the document, + * and they must not overlap. That is, the following must hold for all: + * + * `i < count - 1`: `ranges[i].end_byte <= ranges[i + 1].start_byte` + * + * If this requirement is not satisfied, the operation will fail, the ranges + * will not be assigned, and this function will return `false`. On success, + * this function returns `true` + */ +bool ts_parser_set_included_ranges( + TSParser *self, + const TSRange *ranges, + uint32_t count +); + +/** + * Get the ranges of text that the parser will include when parsing. + * + * The returned pointer is owned by the parser. The caller should not free it + * or write to it. The length of the array will be written to the given + * `count` pointer. + */ +const TSRange *ts_parser_included_ranges( + const TSParser *self, + uint32_t *count +); + +/** + * Use the parser to parse some source code and create a syntax tree. + * + * If you are parsing this document for the first time, pass `NULL` for the + * `old_tree` parameter. Otherwise, if you have already parsed an earlier + * version of this document and the document has since been edited, pass the + * previous syntax tree so that the unchanged parts of it can be reused. + * This will save time and memory. For this to work correctly, you must have + * already edited the old syntax tree using the [`ts_tree_edit`] function in a + * way that exactly matches the source code changes. + * + * The [`TSInput`] parameter lets you specify how to read the text. It has the + * following three fields: + * 1. [`read`]: A function to retrieve a chunk of text at a given byte offset + * and (row, column) position. The function should return a pointer to the + * text and write its length to the [`bytes_read`] pointer. The parser does + * not take ownership of this buffer; it just borrows it until it has + * finished reading it. The function should write a zero value to the + * [`bytes_read`] pointer to indicate the end of the document. + * 2. [`payload`]: An arbitrary pointer that will be passed to each invocation + * of the [`read`] function. + * 3. [`encoding`]: An indication of how the text is encoded. Either + * `TSInputEncodingUTF8` or `TSInputEncodingUTF16`. + * + * This function returns a syntax tree on success, and `NULL` on failure. There + * are four possible reasons for failure: + * 1. The parser does not have a language assigned. Check for this using the + [`ts_parser_language`] function. + * 2. Parsing was cancelled due to a timeout that was set by an earlier call to + * the [`ts_parser_set_timeout_micros`] function. You can resume parsing from + * where the parser left out by calling [`ts_parser_parse`] again with the + * same arguments. Or you can start parsing from scratch by first calling + * [`ts_parser_reset`]. + * 3. Parsing was cancelled using a cancellation flag that was set by an + * earlier call to [`ts_parser_set_cancellation_flag`]. You can resume parsing + * from where the parser left out by calling [`ts_parser_parse`] again with + * the same arguments. + * 4. Parsing was cancelled due to the progress callback returning true. This callback + * is passed in [`ts_parser_parse_with_options`] inside the [`TSParseOptions`] struct. + * + * [`read`]: TSInput::read + * [`payload`]: TSInput::payload + * [`encoding`]: TSInput::encoding + * [`bytes_read`]: TSInput::read + */ +TSTree *ts_parser_parse( + TSParser *self, + const TSTree *old_tree, + TSInput input +); + +/** + * Use the parser to parse some source code and create a syntax tree, with some options. + * + * See [`ts_parser_parse`] for more details. + * + * See [`TSParseOptions`] for more details on the options. + */ +TSTree* ts_parser_parse_with_options( + TSParser *self, + const TSTree *old_tree, + TSInput input, + TSParseOptions parse_options +); + +/** + * Use the parser to parse some source code stored in one contiguous buffer. + * The first two parameters are the same as in the [`ts_parser_parse`] function + * above. The second two parameters indicate the location of the buffer and its + * length in bytes. + */ +TSTree *ts_parser_parse_string( + TSParser *self, + const TSTree *old_tree, + const char *string, + uint32_t length +); + +/** + * Use the parser to parse some source code stored in one contiguous buffer with + * a given encoding. The first four parameters work the same as in the + * [`ts_parser_parse_string`] method above. The final parameter indicates whether + * the text is encoded as UTF8 or UTF16. + */ +TSTree *ts_parser_parse_string_encoding( + TSParser *self, + const TSTree *old_tree, + const char *string, + uint32_t length, + TSInputEncoding encoding +); + +/** + * Instruct the parser to start the next parse from the beginning. + * + * If the parser previously failed because of a timeout or a cancellation, then + * by default, it will resume where it left off on the next call to + * [`ts_parser_parse`] or other parsing functions. If you don't want to resume, + * and instead intend to use this parser to parse some other document, you must + * call [`ts_parser_reset`] first. + */ +void ts_parser_reset(TSParser *self); + +/** + * @deprecated use [`ts_parser_parse_with_options`] and pass in a callback instead, this will be removed in 0.26. + * + * Set the maximum duration in microseconds that parsing should be allowed to + * take before halting. + * + * If parsing takes longer than this, it will halt early, returning NULL. + * See [`ts_parser_parse`] for more information. + */ +void ts_parser_set_timeout_micros(TSParser *self, uint64_t timeout_micros); + +/** + * @deprecated use [`ts_parser_parse_with_options`] and pass in a callback instead, this will be removed in 0.26. + * + * Get the duration in microseconds that parsing is allowed to take. + */ +uint64_t ts_parser_timeout_micros(const TSParser *self); + +/** + * @deprecated use [`ts_parser_parse_with_options`] and pass in a callback instead, this will be removed in 0.26. + * + * Set the parser's current cancellation flag pointer. + * + * If a non-null pointer is assigned, then the parser will periodically read + * from this pointer during parsing. If it reads a non-zero value, it will + * halt early, returning NULL. See [`ts_parser_parse`] for more information. + */ +void ts_parser_set_cancellation_flag(TSParser *self, const size_t *flag); + +/** + * @deprecated use [`ts_parser_parse_with_options`] and pass in a callback instead, this will be removed in 0.26. + * + * Get the parser's current cancellation flag pointer. + */ +const size_t *ts_parser_cancellation_flag(const TSParser *self); + +/** + * Set the logger that a parser should use during parsing. + * + * The parser does not take ownership over the logger payload. If a logger was + * previously assigned, the caller is responsible for releasing any memory + * owned by the previous logger. + */ +void ts_parser_set_logger(TSParser *self, TSLogger logger); + +/** + * Get the parser's current logger. + */ +TSLogger ts_parser_logger(const TSParser *self); + +/** + * Set the file descriptor to which the parser should write debugging graphs + * during parsing. The graphs are formatted in the DOT language. You may want + * to pipe these graphs directly to a `dot(1)` process in order to generate + * SVG output. You can turn off this logging by passing a negative number. + */ +void ts_parser_print_dot_graphs(TSParser *self, int fd); + +/******************/ +/* Section - Tree */ +/******************/ + +/** + * Create a shallow copy of the syntax tree. This is very fast. + * + * You need to copy a syntax tree in order to use it on more than one thread at + * a time, as syntax trees are not thread safe. + */ +TSTree *ts_tree_copy(const TSTree *self); + +/** + * Delete the syntax tree, freeing all of the memory that it used. + */ +void ts_tree_delete(TSTree *self); + +/** + * Get the root node of the syntax tree. + */ +TSNode ts_tree_root_node(const TSTree *self); + +/** + * Get the root node of the syntax tree, but with its position + * shifted forward by the given offset. + */ +TSNode ts_tree_root_node_with_offset( + const TSTree *self, + uint32_t offset_bytes, + TSPoint offset_extent +); + +/** + * Get the language that was used to parse the syntax tree. + */ +const TSLanguage *ts_tree_language(const TSTree *self); + +/** + * Get the array of included ranges that was used to parse the syntax tree. + * + * The returned pointer must be freed by the caller. + */ +TSRange *ts_tree_included_ranges(const TSTree *self, uint32_t *length); + +/** + * Edit the syntax tree to keep it in sync with source code that has been + * edited. + * + * You must describe the edit both in terms of byte offsets and in terms of + * (row, column) coordinates. + */ +void ts_tree_edit(TSTree *self, const TSInputEdit *edit); + +/** + * Compare an old edited syntax tree to a new syntax tree representing the same + * document, returning an array of ranges whose syntactic structure has changed. + * + * For this to work correctly, the old syntax tree must have been edited such + * that its ranges match up to the new tree. Generally, you'll want to call + * this function right after calling one of the [`ts_parser_parse`] functions. + * You need to pass the old tree that was passed to parse, as well as the new + * tree that was returned from that function. + * + * The returned ranges indicate areas where the hierarchical structure of syntax + * nodes (from root to leaf) has changed between the old and new trees. Characters + * outside these ranges have identical ancestor nodes in both trees. + * + * Note that the returned ranges may be slightly larger than the exact changed areas, + * but Tree-sitter attempts to make them as small as possible. + * + * The returned array is allocated using `malloc` and the caller is responsible + * for freeing it using `free`. The length of the array will be written to the + * given `length` pointer. + */ +TSRange *ts_tree_get_changed_ranges( + const TSTree *old_tree, + const TSTree *new_tree, + uint32_t *length +); + +/** + * Write a DOT graph describing the syntax tree to the given file. + */ +void ts_tree_print_dot_graph(const TSTree *self, int file_descriptor); + +/******************/ +/* Section - Node */ +/******************/ + +/** + * Get the node's type as a null-terminated string. + */ +const char *ts_node_type(TSNode self); + +/** + * Get the node's type as a numerical id. + */ +TSSymbol ts_node_symbol(TSNode self); + +/** + * Get the node's language. + */ +const TSLanguage *ts_node_language(TSNode self); + +/** + * Get the node's type as it appears in the grammar ignoring aliases as a + * null-terminated string. + */ +const char *ts_node_grammar_type(TSNode self); + +/** + * Get the node's type as a numerical id as it appears in the grammar ignoring + * aliases. This should be used in [`ts_language_next_state`] instead of + * [`ts_node_symbol`]. + */ +TSSymbol ts_node_grammar_symbol(TSNode self); + +/** + * Get the node's start byte. + */ +uint32_t ts_node_start_byte(TSNode self); + +/** + * Get the node's start position in terms of rows and columns. + */ +TSPoint ts_node_start_point(TSNode self); + +/** + * Get the node's end byte. + */ +uint32_t ts_node_end_byte(TSNode self); + +/** + * Get the node's end position in terms of rows and columns. + */ +TSPoint ts_node_end_point(TSNode self); + +/** + * Get an S-expression representing the node as a string. + * + * This string is allocated with `malloc` and the caller is responsible for + * freeing it using `free`. + */ +char *ts_node_string(TSNode self); + +/** + * Check if the node is null. Functions like [`ts_node_child`] and + * [`ts_node_next_sibling`] will return a null node to indicate that no such node + * was found. + */ +bool ts_node_is_null(TSNode self); + +/** + * Check if the node is *named*. Named nodes correspond to named rules in the + * grammar, whereas *anonymous* nodes correspond to string literals in the + * grammar. + */ +bool ts_node_is_named(TSNode self); + +/** + * Check if the node is *missing*. Missing nodes are inserted by the parser in + * order to recover from certain kinds of syntax errors. + */ +bool ts_node_is_missing(TSNode self); + +/** + * Check if the node is *extra*. Extra nodes represent things like comments, + * which are not required the grammar, but can appear anywhere. + */ +bool ts_node_is_extra(TSNode self); + +/** + * Check if a syntax node has been edited. + */ +bool ts_node_has_changes(TSNode self); + +/** + * Check if the node is a syntax error or contains any syntax errors. + */ +bool ts_node_has_error(TSNode self); + +/** + * Check if the node is a syntax error. +*/ +bool ts_node_is_error(TSNode self); + +/** + * Get this node's parse state. +*/ +TSStateId ts_node_parse_state(TSNode self); + +/** + * Get the parse state after this node. +*/ +TSStateId ts_node_next_parse_state(TSNode self); + +/** + * Get the node's immediate parent. + * Prefer [`ts_node_child_with_descendant`] for + * iterating over the node's ancestors. + */ +TSNode ts_node_parent(TSNode self); + +/** + * Get the node that contains `descendant`. + * + * Note that this can return `descendant` itself. + */ +TSNode ts_node_child_with_descendant(TSNode self, TSNode descendant); + +/** + * Get the node's child at the given index, where zero represents the first + * child. + */ +TSNode ts_node_child(TSNode self, uint32_t child_index); + +/** + * Get the field name for node's child at the given index, where zero represents + * the first child. Returns NULL, if no field is found. + */ +const char *ts_node_field_name_for_child(TSNode self, uint32_t child_index); + +/** + * Get the field name for node's named child at the given index, where zero + * represents the first named child. Returns NULL, if no field is found. + */ +const char *ts_node_field_name_for_named_child(TSNode self, uint32_t named_child_index); + +/** + * Get the node's number of children. + */ +uint32_t ts_node_child_count(TSNode self); + +/** + * Get the node's *named* child at the given index. + * + * See also [`ts_node_is_named`]. + */ +TSNode ts_node_named_child(TSNode self, uint32_t child_index); + +/** + * Get the node's number of *named* children. + * + * See also [`ts_node_is_named`]. + */ +uint32_t ts_node_named_child_count(TSNode self); + +/** + * Get the node's child with the given field name. + */ +TSNode ts_node_child_by_field_name( + TSNode self, + const char *name, + uint32_t name_length +); + +/** + * Get the node's child with the given numerical field id. + * + * You can convert a field name to an id using the + * [`ts_language_field_id_for_name`] function. + */ +TSNode ts_node_child_by_field_id(TSNode self, TSFieldId field_id); + +/** + * Get the node's next / previous sibling. + */ +TSNode ts_node_next_sibling(TSNode self); +TSNode ts_node_prev_sibling(TSNode self); + +/** + * Get the node's next / previous *named* sibling. + */ +TSNode ts_node_next_named_sibling(TSNode self); +TSNode ts_node_prev_named_sibling(TSNode self); + +/** + * Get the node's first child that contains or starts after the given byte offset. + */ +TSNode ts_node_first_child_for_byte(TSNode self, uint32_t byte); + +/** + * Get the node's first named child that contains or starts after the given byte offset. + */ +TSNode ts_node_first_named_child_for_byte(TSNode self, uint32_t byte); + +/** + * Get the node's number of descendants, including one for the node itself. + */ +uint32_t ts_node_descendant_count(TSNode self); + +/** + * Get the smallest node within this node that spans the given range of bytes + * or (row, column) positions. + */ +TSNode ts_node_descendant_for_byte_range(TSNode self, uint32_t start, uint32_t end); +TSNode ts_node_descendant_for_point_range(TSNode self, TSPoint start, TSPoint end); + +/** + * Get the smallest named node within this node that spans the given range of + * bytes or (row, column) positions. + */ +TSNode ts_node_named_descendant_for_byte_range(TSNode self, uint32_t start, uint32_t end); +TSNode ts_node_named_descendant_for_point_range(TSNode self, TSPoint start, TSPoint end); + +/** + * Edit the node to keep it in-sync with source code that has been edited. + * + * This function is only rarely needed. When you edit a syntax tree with the + * [`ts_tree_edit`] function, all of the nodes that you retrieve from the tree + * afterward will already reflect the edit. You only need to use [`ts_node_edit`] + * when you have a [`TSNode`] instance that you want to keep and continue to use + * after an edit. + */ +void ts_node_edit(TSNode *self, const TSInputEdit *edit); + +/** + * Check if two nodes are identical. + */ +bool ts_node_eq(TSNode self, TSNode other); + +/************************/ +/* Section - TreeCursor */ +/************************/ + +/** + * Create a new tree cursor starting from the given node. + * + * A tree cursor allows you to walk a syntax tree more efficiently than is + * possible using the [`TSNode`] functions. It is a mutable object that is always + * on a certain syntax node, and can be moved imperatively to different nodes. + * + * Note that the given node is considered the root of the cursor, + * and the cursor cannot walk outside this node. + */ +TSTreeCursor ts_tree_cursor_new(TSNode node); + +/** + * Delete a tree cursor, freeing all of the memory that it used. + */ +void ts_tree_cursor_delete(TSTreeCursor *self); + +/** + * Re-initialize a tree cursor to start at the original node that the cursor was + * constructed with. + */ +void ts_tree_cursor_reset(TSTreeCursor *self, TSNode node); + +/** + * Re-initialize a tree cursor to the same position as another cursor. + * + * Unlike [`ts_tree_cursor_reset`], this will not lose parent information and + * allows reusing already created cursors. +*/ +void ts_tree_cursor_reset_to(TSTreeCursor *dst, const TSTreeCursor *src); + +/** + * Get the tree cursor's current node. + */ +TSNode ts_tree_cursor_current_node(const TSTreeCursor *self); + +/** + * Get the field name of the tree cursor's current node. + * + * This returns `NULL` if the current node doesn't have a field. + * See also [`ts_node_child_by_field_name`]. + */ +const char *ts_tree_cursor_current_field_name(const TSTreeCursor *self); + +/** + * Get the field id of the tree cursor's current node. + * + * This returns zero if the current node doesn't have a field. + * See also [`ts_node_child_by_field_id`], [`ts_language_field_id_for_name`]. + */ +TSFieldId ts_tree_cursor_current_field_id(const TSTreeCursor *self); + +/** + * Move the cursor to the parent of its current node. + * + * This returns `true` if the cursor successfully moved, and returns `false` + * if there was no parent node (the cursor was already on the root node). + * + * Note that the node the cursor was constructed with is considered the root + * of the cursor, and the cursor cannot walk outside this node. + */ +bool ts_tree_cursor_goto_parent(TSTreeCursor *self); + +/** + * Move the cursor to the next sibling of its current node. + * + * This returns `true` if the cursor successfully moved, and returns `false` + * if there was no next sibling node. + * + * Note that the node the cursor was constructed with is considered the root + * of the cursor, and the cursor cannot walk outside this node. + */ +bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self); + +/** + * Move the cursor to the previous sibling of its current node. + * + * This returns `true` if the cursor successfully moved, and returns `false` if + * there was no previous sibling node. + * + * Note, that this function may be slower than + * [`ts_tree_cursor_goto_next_sibling`] due to how node positions are stored. In + * the worst case, this will need to iterate through all the children up to the + * previous sibling node to recalculate its position. Also note that the node the cursor + * was constructed with is considered the root of the cursor, and the cursor cannot + * walk outside this node. + */ +bool ts_tree_cursor_goto_previous_sibling(TSTreeCursor *self); + +/** + * Move the cursor to the first child of its current node. + * + * This returns `true` if the cursor successfully moved, and returns `false` + * if there were no children. + */ +bool ts_tree_cursor_goto_first_child(TSTreeCursor *self); + +/** + * Move the cursor to the last child of its current node. + * + * This returns `true` if the cursor successfully moved, and returns `false` if + * there were no children. + * + * Note that this function may be slower than [`ts_tree_cursor_goto_first_child`] + * because it needs to iterate through all the children to compute the child's + * position. + */ +bool ts_tree_cursor_goto_last_child(TSTreeCursor *self); + +/** + * Move the cursor to the node that is the nth descendant of + * the original node that the cursor was constructed with, where + * zero represents the original node itself. + */ +void ts_tree_cursor_goto_descendant(TSTreeCursor *self, uint32_t goal_descendant_index); + +/** + * Get the index of the cursor's current node out of all of the + * descendants of the original node that the cursor was constructed with. + */ +uint32_t ts_tree_cursor_current_descendant_index(const TSTreeCursor *self); + +/** + * Get the depth of the cursor's current node relative to the original + * node that the cursor was constructed with. + */ +uint32_t ts_tree_cursor_current_depth(const TSTreeCursor *self); + +/** + * Move the cursor to the first child of its current node that contains or starts after + * the given byte offset or point. + * + * This returns the index of the child node if one was found, and returns -1 + * if no such child was found. + */ +int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *self, uint32_t goal_byte); +int64_t ts_tree_cursor_goto_first_child_for_point(TSTreeCursor *self, TSPoint goal_point); + +TSTreeCursor ts_tree_cursor_copy(const TSTreeCursor *cursor); + +/*******************/ +/* Section - Query */ +/*******************/ + +/** + * Create a new query from a string containing one or more S-expression + * patterns. The query is associated with a particular language, and can + * only be run on syntax nodes parsed with that language. + * + * If all of the given patterns are valid, this returns a [`TSQuery`]. + * If a pattern is invalid, this returns `NULL`, and provides two pieces + * of information about the problem: + * 1. The byte offset of the error is written to the `error_offset` parameter. + * 2. The type of error is written to the `error_type` parameter. + */ +TSQuery *ts_query_new( + const TSLanguage *language, + const char *source, + uint32_t source_len, + uint32_t *error_offset, + TSQueryError *error_type +); + +/** + * Delete a query, freeing all of the memory that it used. + */ +void ts_query_delete(TSQuery *self); + +/** + * Get the number of patterns, captures, or string literals in the query. + */ +uint32_t ts_query_pattern_count(const TSQuery *self); +uint32_t ts_query_capture_count(const TSQuery *self); +uint32_t ts_query_string_count(const TSQuery *self); + +/** + * Get the byte offset where the given pattern starts in the query's source. + * + * This can be useful when combining queries by concatenating their source + * code strings. + */ +uint32_t ts_query_start_byte_for_pattern(const TSQuery *self, uint32_t pattern_index); + +/** + * Get the byte offset where the given pattern ends in the query's source. + * + * This can be useful when combining queries by concatenating their source + * code strings. + */ +uint32_t ts_query_end_byte_for_pattern(const TSQuery *self, uint32_t pattern_index); + +/** + * Get all of the predicates for the given pattern in the query. + * + * The predicates are represented as a single array of steps. There are three + * types of steps in this array, which correspond to the three legal values for + * the `type` field: + * - `TSQueryPredicateStepTypeCapture` - Steps with this type represent names + * of captures. Their `value_id` can be used with the + * [`ts_query_capture_name_for_id`] function to obtain the name of the capture. + * - `TSQueryPredicateStepTypeString` - Steps with this type represent literal + * strings. Their `value_id` can be used with the + * [`ts_query_string_value_for_id`] function to obtain their string value. + * - `TSQueryPredicateStepTypeDone` - Steps with this type are *sentinels* + * that represent the end of an individual predicate. If a pattern has two + * predicates, then there will be two steps with this `type` in the array. + */ +const TSQueryPredicateStep *ts_query_predicates_for_pattern( + const TSQuery *self, + uint32_t pattern_index, + uint32_t *step_count +); + +/* + * Check if the given pattern in the query has a single root node. + */ +bool ts_query_is_pattern_rooted(const TSQuery *self, uint32_t pattern_index); + +/* + * Check if the given pattern in the query is 'non local'. + * + * A non-local pattern has multiple root nodes and can match within a + * repeating sequence of nodes, as specified by the grammar. Non-local + * patterns disable certain optimizations that would otherwise be possible + * when executing a query on a specific range of a syntax tree. + */ +bool ts_query_is_pattern_non_local(const TSQuery *self, uint32_t pattern_index); + +/* + * Check if a given pattern is guaranteed to match once a given step is reached. + * The step is specified by its byte offset in the query's source code. + */ +bool ts_query_is_pattern_guaranteed_at_step(const TSQuery *self, uint32_t byte_offset); + +/** + * Get the name and length of one of the query's captures, or one of the + * query's string literals. Each capture and string is associated with a + * numeric id based on the order that it appeared in the query's source. + */ +const char *ts_query_capture_name_for_id( + const TSQuery *self, + uint32_t index, + uint32_t *length +); + +/** + * Get the quantifier of the query's captures. Each capture is * associated + * with a numeric id based on the order that it appeared in the query's source. + */ +TSQuantifier ts_query_capture_quantifier_for_id( + const TSQuery *self, + uint32_t pattern_index, + uint32_t capture_index +); + +const char *ts_query_string_value_for_id( + const TSQuery *self, + uint32_t index, + uint32_t *length +); + +/** + * Disable a certain capture within a query. + * + * This prevents the capture from being returned in matches, and also avoids + * any resource usage associated with recording the capture. Currently, there + * is no way to undo this. + */ +void ts_query_disable_capture(TSQuery *self, const char *name, uint32_t length); + +/** + * Disable a certain pattern within a query. + * + * This prevents the pattern from matching and removes most of the overhead + * associated with the pattern. Currently, there is no way to undo this. + */ +void ts_query_disable_pattern(TSQuery *self, uint32_t pattern_index); + +/** + * Create a new cursor for executing a given query. + * + * The cursor stores the state that is needed to iteratively search + * for matches. To use the query cursor, first call [`ts_query_cursor_exec`] + * to start running a given query on a given syntax node. Then, there are + * two options for consuming the results of the query: + * 1. Repeatedly call [`ts_query_cursor_next_match`] to iterate over all of the + * *matches* in the order that they were found. Each match contains the + * index of the pattern that matched, and an array of captures. Because + * multiple patterns can match the same set of nodes, one match may contain + * captures that appear *before* some of the captures from a previous match. + * 2. Repeatedly call [`ts_query_cursor_next_capture`] to iterate over all of the + * individual *captures* in the order that they appear. This is useful if + * don't care about which pattern matched, and just want a single ordered + * sequence of captures. + * + * If you don't care about consuming all of the results, you can stop calling + * [`ts_query_cursor_next_match`] or [`ts_query_cursor_next_capture`] at any point. + * You can then start executing another query on another node by calling + * [`ts_query_cursor_exec`] again. + */ +TSQueryCursor *ts_query_cursor_new(void); + +/** + * Delete a query cursor, freeing all of the memory that it used. + */ +void ts_query_cursor_delete(TSQueryCursor *self); + +/** + * Start running a given query on a given node. + */ +void ts_query_cursor_exec(TSQueryCursor *self, const TSQuery *query, TSNode node); + +/** + * Start running a given query on a given node, with some options. + */ +void ts_query_cursor_exec_with_options( + TSQueryCursor *self, + const TSQuery *query, + TSNode node, + const TSQueryCursorOptions *query_options +); + +/** + * Manage the maximum number of in-progress matches allowed by this query + * cursor. + * + * Query cursors have an optional maximum capacity for storing lists of + * in-progress captures. If this capacity is exceeded, then the + * earliest-starting match will silently be dropped to make room for further + * matches. This maximum capacity is optional — by default, query cursors allow + * any number of pending matches, dynamically allocating new space for them as + * needed as the query is executed. + */ +bool ts_query_cursor_did_exceed_match_limit(const TSQueryCursor *self); +uint32_t ts_query_cursor_match_limit(const TSQueryCursor *self); +void ts_query_cursor_set_match_limit(TSQueryCursor *self, uint32_t limit); + +/** + * @deprecated use [`ts_query_cursor_exec_with_options`] and pass in a callback instead, this will be removed in 0.26. + * + * Set the maximum duration in microseconds that query execution should be allowed to + * take before halting. + * + * If query execution takes longer than this, it will halt early, returning NULL. + * See [`ts_query_cursor_next_match`] or [`ts_query_cursor_next_capture`] for more information. + */ +void ts_query_cursor_set_timeout_micros(TSQueryCursor *self, uint64_t timeout_micros); + +/** + * @deprecated use [`ts_query_cursor_exec_with_options`] and pass in a callback instead, this will be removed in 0.26. + * + * Get the duration in microseconds that query execution is allowed to take. + * + * This is set via [`ts_query_cursor_set_timeout_micros`]. + */ +uint64_t ts_query_cursor_timeout_micros(const TSQueryCursor *self); + +/** + * Set the range of bytes in which the query will be executed. + * + * The query cursor will return matches that intersect with the given point range. + * This means that a match may be returned even if some of its captures fall + * outside the specified range, as long as at least part of the match + * overlaps with the range. + * + * For example, if a query pattern matches a node that spans a larger area + * than the specified range, but part of that node intersects with the range, + * the entire match will be returned. + * + * This will return `false` if the start byte is greater than the end byte, otherwise + * it will return `true`. + */ +bool ts_query_cursor_set_byte_range(TSQueryCursor *self, uint32_t start_byte, uint32_t end_byte); + +/** + * Set the range of (row, column) positions in which the query will be executed. + * + * The query cursor will return matches that intersect with the given point range. + * This means that a match may be returned even if some of its captures fall + * outside the specified range, as long as at least part of the match + * overlaps with the range. + * + * For example, if a query pattern matches a node that spans a larger area + * than the specified range, but part of that node intersects with the range, + * the entire match will be returned. + * + * This will return `false` if the start point is greater than the end point, otherwise + * it will return `true`. + */ +bool ts_query_cursor_set_point_range(TSQueryCursor *self, TSPoint start_point, TSPoint end_point); + +/** + * Advance to the next match of the currently running query. + * + * If there is a match, write it to `*match` and return `true`. + * Otherwise, return `false`. + */ +bool ts_query_cursor_next_match(TSQueryCursor *self, TSQueryMatch *match); +void ts_query_cursor_remove_match(TSQueryCursor *self, uint32_t match_id); + +/** + * Advance to the next capture of the currently running query. + * + * If there is a capture, write its match to `*match` and its index within + * the match's capture list to `*capture_index`. Otherwise, return `false`. + */ +bool ts_query_cursor_next_capture( + TSQueryCursor *self, + TSQueryMatch *match, + uint32_t *capture_index +); + +/** + * Set the maximum start depth for a query cursor. + * + * This prevents cursors from exploring children nodes at a certain depth. + * Note if a pattern includes many children, then they will still be checked. + * + * The zero max start depth value can be used as a special behavior and + * it helps to destructure a subtree by staying on a node and using captures + * for interested parts. Note that the zero max start depth only limit a search + * depth for a pattern's root node but other nodes that are parts of the pattern + * may be searched at any depth what defined by the pattern structure. + * + * Set to `UINT32_MAX` to remove the maximum start depth. + */ +void ts_query_cursor_set_max_start_depth(TSQueryCursor *self, uint32_t max_start_depth); + +/**********************/ +/* Section - Language */ +/**********************/ + +/** + * Get another reference to the given language. + */ +const TSLanguage *ts_language_copy(const TSLanguage *self); + +/** + * Free any dynamically-allocated resources for this language, if + * this is the last reference. + */ +void ts_language_delete(const TSLanguage *self); + +/** + * Get the number of distinct node types in the language. + */ +uint32_t ts_language_symbol_count(const TSLanguage *self); + +/** + * Get the number of valid states in this language. +*/ +uint32_t ts_language_state_count(const TSLanguage *self); + +/** + * Get the numerical id for the given node type string. + */ +TSSymbol ts_language_symbol_for_name( + const TSLanguage *self, + const char *string, + uint32_t length, + bool is_named +); + +/** + * Get the number of distinct field names in the language. + */ +uint32_t ts_language_field_count(const TSLanguage *self); + +/** + * Get the field name string for the given numerical id. + */ +const char *ts_language_field_name_for_id(const TSLanguage *self, TSFieldId id); + +/** + * Get the numerical id for the given field name string. + */ +TSFieldId ts_language_field_id_for_name(const TSLanguage *self, const char *name, uint32_t name_length); + +/** + * Get a list of all supertype symbols for the language. +*/ +const TSSymbol *ts_language_supertypes(const TSLanguage *self, uint32_t *length); + +/** + * Get a list of all subtype symbol ids for a given supertype symbol. + * + * See [`ts_language_supertypes`] for fetching all supertype symbols. + */ +const TSSymbol *ts_language_subtypes( + const TSLanguage *self, + TSSymbol supertype, + uint32_t *length +); + +/** + * Get a node type string for the given numerical id. + */ +const char *ts_language_symbol_name(const TSLanguage *self, TSSymbol symbol); + +/** + * Check whether the given node type id belongs to named nodes, anonymous nodes, + * or a hidden nodes. + * + * See also [`ts_node_is_named`]. Hidden nodes are never returned from the API. + */ +TSSymbolType ts_language_symbol_type(const TSLanguage *self, TSSymbol symbol); + +/** + * @deprecated use [`ts_language_abi_version`] instead, this will be removed in 0.26. + * + * Get the ABI version number for this language. This version number is used + * to ensure that languages were generated by a compatible version of + * Tree-sitter. + * + * See also [`ts_parser_set_language`]. + */ +uint32_t ts_language_version(const TSLanguage *self); + +/** + * Get the ABI version number for this language. This version number is used + * to ensure that languages were generated by a compatible version of + * Tree-sitter. + * + * See also [`ts_parser_set_language`]. + */ +uint32_t ts_language_abi_version(const TSLanguage *self); + +/** + * Get the metadata for this language. This information is generated by the + * CLI, and relies on the language author providing the correct metadata in + * the language's `tree-sitter.json` file. + * + * See also [`TSMetadata`]. + */ +const TSLanguageMetadata *ts_language_metadata(const TSLanguage *self); + +/** + * Get the next parse state. Combine this with lookahead iterators to generate + * completion suggestions or valid symbols in error nodes. Use + * [`ts_node_grammar_symbol`] for valid symbols. +*/ +TSStateId ts_language_next_state(const TSLanguage *self, TSStateId state, TSSymbol symbol); + +/** + * Get the name of this language. This returns `NULL` in older parsers. + */ +const char *ts_language_name(const TSLanguage *self); + +/********************************/ +/* Section - Lookahead Iterator */ +/********************************/ + +/** + * Create a new lookahead iterator for the given language and parse state. + * + * This returns `NULL` if state is invalid for the language. + * + * Repeatedly using [`ts_lookahead_iterator_next`] and + * [`ts_lookahead_iterator_current_symbol`] will generate valid symbols in the + * given parse state. Newly created lookahead iterators will contain the `ERROR` + * symbol. + * + * Lookahead iterators can be useful to generate suggestions and improve syntax + * error diagnostics. To get symbols valid in an ERROR node, use the lookahead + * iterator on its first leaf node state. For `MISSING` nodes, a lookahead + * iterator created on the previous non-extra leaf node may be appropriate. +*/ +TSLookaheadIterator *ts_lookahead_iterator_new(const TSLanguage *self, TSStateId state); + +/** + * Delete a lookahead iterator freeing all the memory used. +*/ +void ts_lookahead_iterator_delete(TSLookaheadIterator *self); + +/** + * Reset the lookahead iterator to another state. + * + * This returns `true` if the iterator was reset to the given state and `false` + * otherwise. +*/ +bool ts_lookahead_iterator_reset_state(TSLookaheadIterator *self, TSStateId state); + +/** + * Reset the lookahead iterator. + * + * This returns `true` if the language was set successfully and `false` + * otherwise. +*/ +bool ts_lookahead_iterator_reset(TSLookaheadIterator *self, const TSLanguage *language, TSStateId state); + +/** + * Get the current language of the lookahead iterator. +*/ +const TSLanguage *ts_lookahead_iterator_language(const TSLookaheadIterator *self); + +/** + * Advance the lookahead iterator to the next symbol. + * + * This returns `true` if there is a new symbol and `false` otherwise. +*/ +bool ts_lookahead_iterator_next(TSLookaheadIterator *self); + +/** + * Get the current symbol of the lookahead iterator; +*/ +TSSymbol ts_lookahead_iterator_current_symbol(const TSLookaheadIterator *self); + +/** + * Get the current symbol type of the lookahead iterator as a null terminated + * string. +*/ +const char *ts_lookahead_iterator_current_symbol_name(const TSLookaheadIterator *self); + +/*************************************/ +/* Section - WebAssembly Integration */ +/************************************/ + +typedef struct wasm_engine_t TSWasmEngine; +typedef struct TSWasmStore TSWasmStore; + +typedef enum { + TSWasmErrorKindNone = 0, + TSWasmErrorKindParse, + TSWasmErrorKindCompile, + TSWasmErrorKindInstantiate, + TSWasmErrorKindAllocate, +} TSWasmErrorKind; + +typedef struct { + TSWasmErrorKind kind; + char *message; +} TSWasmError; + +/** + * Create a Wasm store. + */ +TSWasmStore *ts_wasm_store_new( + TSWasmEngine *engine, + TSWasmError *error +); + +/** + * Free the memory associated with the given Wasm store. + */ +void ts_wasm_store_delete(TSWasmStore *); + +/** + * Create a language from a buffer of Wasm. The resulting language behaves + * like any other Tree-sitter language, except that in order to use it with + * a parser, that parser must have a Wasm store. Note that the language + * can be used with any Wasm store, it doesn't need to be the same store that + * was used to originally load it. + */ +const TSLanguage *ts_wasm_store_load_language( + TSWasmStore *, + const char *name, + const char *wasm, + uint32_t wasm_len, + TSWasmError *error +); + +/** + * Get the number of languages instantiated in the given wasm store. + */ +size_t ts_wasm_store_language_count(const TSWasmStore *); + +/** + * Check if the language came from a Wasm module. If so, then in order to use + * this language with a Parser, that parser must have a Wasm store assigned. + */ +bool ts_language_is_wasm(const TSLanguage *); + +/** + * Assign the given Wasm store to the parser. A parser must have a Wasm store + * in order to use Wasm languages. + */ +void ts_parser_set_wasm_store(TSParser *, TSWasmStore *); + +/** + * Remove the parser's current Wasm store and return it. This returns NULL if + * the parser doesn't have a Wasm store. + */ +TSWasmStore *ts_parser_take_wasm_store(TSParser *); + +/**********************************/ +/* Section - Global Configuration */ +/**********************************/ + +/** + * Set the allocation functions used by the library. + * + * By default, Tree-sitter uses the standard libc allocation functions, + * but aborts the process when an allocation fails. This function lets + * you supply alternative allocation functions at runtime. + * + * If you pass `NULL` for any parameter, Tree-sitter will switch back to + * its default implementation of that function. + * + * If you call this function after the library has already been used, then + * you must ensure that either: + * 1. All the existing objects have been freed. + * 2. The new allocator shares its state with the old one, so it is capable + * of freeing memory that was allocated by the old allocator. + */ +void ts_set_allocator( + void *(*new_malloc)(size_t), + void *(*new_calloc)(size_t, size_t), + void *(*new_realloc)(void *, size_t), + void (*new_free)(void *) +); + +#ifdef __cplusplus +} +#endif + +#ifndef TREE_SITTER_HIDE_SYMBOLS +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC visibility pop +#endif +#endif + +#endif // TREE_SITTER_API_H_ diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/alloc.c b/Externals/crystaledit/editlib/tree-sitter/lib/src/alloc.c new file mode 100644 index 00000000000..e5cb1d5eab4 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/alloc.c @@ -0,0 +1,48 @@ +#include "alloc.h" +#include "tree_sitter/api.h" +#include + +static void *ts_malloc_default(size_t size) { + void *result = malloc(size); + if (size > 0 && !result) { + fprintf(stderr, "tree-sitter failed to allocate %zu bytes", size); + abort(); + } + return result; +} + +static void *ts_calloc_default(size_t count, size_t size) { + void *result = calloc(count, size); + if (count > 0 && !result) { + fprintf(stderr, "tree-sitter failed to allocate %zu bytes", count * size); + abort(); + } + return result; +} + +static void *ts_realloc_default(void *buffer, size_t size) { + void *result = realloc(buffer, size); + if (size > 0 && !result) { + fprintf(stderr, "tree-sitter failed to reallocate %zu bytes", size); + abort(); + } + return result; +} + +// Allow clients to override allocation functions dynamically +TS_PUBLIC void *(*ts_current_malloc)(size_t) = ts_malloc_default; +TS_PUBLIC void *(*ts_current_calloc)(size_t, size_t) = ts_calloc_default; +TS_PUBLIC void *(*ts_current_realloc)(void *, size_t) = ts_realloc_default; +TS_PUBLIC void (*ts_current_free)(void *) = free; + +void ts_set_allocator( + void *(*new_malloc)(size_t size), + void *(*new_calloc)(size_t count, size_t size), + void *(*new_realloc)(void *ptr, size_t size), + void (*new_free)(void *ptr) +) { + ts_current_malloc = new_malloc ? new_malloc : ts_malloc_default; + ts_current_calloc = new_calloc ? new_calloc : ts_calloc_default; + ts_current_realloc = new_realloc ? new_realloc : ts_realloc_default; + ts_current_free = new_free ? new_free : free; +} diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/alloc.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/alloc.h new file mode 100644 index 00000000000..a27b8a63345 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/alloc.h @@ -0,0 +1,41 @@ +#ifndef TREE_SITTER_ALLOC_H_ +#define TREE_SITTER_ALLOC_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#if defined(TREE_SITTER_HIDDEN_SYMBOLS) || defined(_WIN32) +#define TS_PUBLIC +#else +#define TS_PUBLIC __attribute__((visibility("default"))) +#endif + +TS_PUBLIC extern void *(*ts_current_malloc)(size_t size); +TS_PUBLIC extern void *(*ts_current_calloc)(size_t count, size_t size); +TS_PUBLIC extern void *(*ts_current_realloc)(void *ptr, size_t size); +TS_PUBLIC extern void (*ts_current_free)(void *ptr); + +// Allow clients to override allocation functions +#ifndef ts_malloc +#define ts_malloc ts_current_malloc +#endif +#ifndef ts_calloc +#define ts_calloc ts_current_calloc +#endif +#ifndef ts_realloc +#define ts_realloc ts_current_realloc +#endif +#ifndef ts_free +#define ts_free ts_current_free +#endif + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_ALLOC_H_ diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/array.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/array.h new file mode 100644 index 00000000000..d965c6170ce --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/array.h @@ -0,0 +1,291 @@ +#ifndef TREE_SITTER_ARRAY_H_ +#define TREE_SITTER_ARRAY_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "./alloc.h" +#include "./ts_assert.h" + +#include +#include +#include +#include + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4101) +#elif defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" +#endif + +#define Array(T) \ + struct { \ + T *contents; \ + uint32_t size; \ + uint32_t capacity; \ + } + +/// Initialize an array. +#define array_init(self) \ + ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL) + +/// Create an empty array. +#define array_new() \ + { NULL, 0, 0 } + +/// Get a pointer to the element at a given `index` in the array. +#define array_get(self, _index) \ + (ts_assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index]) + +/// Get a pointer to the first element in the array. +#define array_front(self) array_get(self, 0) + +/// Get a pointer to the last element in the array. +#define array_back(self) array_get(self, (self)->size - 1) + +/// Clear the array, setting its size to zero. Note that this does not free any +/// memory allocated for the array's contents. +#define array_clear(self) ((self)->size = 0) + +/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is +/// less than the array's current capacity, this function has no effect. +#define array_reserve(self, new_capacity) \ + _array__reserve((Array *)(self), array_elem_size(self), new_capacity) + +/// Free any memory allocated for this array. Note that this does not free any +/// memory allocated for the array's contents. +#define array_delete(self) _array__delete((Array *)(self)) + +/// Push a new `element` onto the end of the array. +#define array_push(self, element) \ + (_array__grow((Array *)(self), 1, array_elem_size(self)), \ + (self)->contents[(self)->size++] = (element)) + +/// Increase the array's size by `count` elements. +/// New elements are zero-initialized. +#define array_grow_by(self, count) \ + do { \ + if ((count) == 0) break; \ + _array__grow((Array *)(self), count, array_elem_size(self)); \ + memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \ + (self)->size += (count); \ + } while (0) + +/// Append all elements from one array to the end of another. +#define array_push_all(self, other) \ + array_extend((self), (other)->size, (other)->contents) + +/// Append `count` elements to the end of the array, reading their values from the +/// `contents` pointer. +#define array_extend(self, count, contents) \ + _array__splice( \ + (Array *)(self), array_elem_size(self), (self)->size, \ + 0, count, contents \ + ) + +/// Remove `old_count` elements from the array starting at the given `index`. At +/// the same index, insert `new_count` new elements, reading their values from the +/// `new_contents` pointer. +#define array_splice(self, _index, old_count, new_count, new_contents) \ + _array__splice( \ + (Array *)(self), array_elem_size(self), _index, \ + old_count, new_count, new_contents \ + ) + +/// Insert one `element` into the array at the given `index`. +#define array_insert(self, _index, element) \ + _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element)) + +/// Remove one element from the array at the given `index`. +#define array_erase(self, _index) \ + _array__erase((Array *)(self), array_elem_size(self), _index) + +/// Pop the last element off the array, returning the element by value. +#define array_pop(self) ((self)->contents[--(self)->size]) + +/// Assign the contents of one array to another, reallocating if necessary. +#define array_assign(self, other) \ + _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self)) + +/// Swap one array with another +#define array_swap(self, other) \ + _array__swap((Array *)(self), (Array *)(other)) + +/// Get the size of the array contents +#define array_elem_size(self) (sizeof *(self)->contents) + +/// Search a sorted array for a given `needle` value, using the given `compare` +/// callback to determine the order. +/// +/// If an existing element is found to be equal to `needle`, then the `index` +/// out-parameter is set to the existing value's index, and the `exists` +/// out-parameter is set to true. Otherwise, `index` is set to an index where +/// `needle` should be inserted in order to preserve the sorting, and `exists` +/// is set to false. +#define array_search_sorted_with(self, compare, needle, _index, _exists) \ + _array__search_sorted(self, 0, compare, , needle, _index, _exists) + +/// Search a sorted array for a given `needle` value, using integer comparisons +/// of a given struct field (specified with a leading dot) to determine the order. +/// +/// See also `array_search_sorted_with`. +#define array_search_sorted_by(self, field, needle, _index, _exists) \ + _array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists) + +/// Insert a given `value` into a sorted array, using the given `compare` +/// callback to determine the order. +#define array_insert_sorted_with(self, compare, value) \ + do { \ + unsigned _index, _exists; \ + array_search_sorted_with(self, compare, &(value), &_index, &_exists); \ + if (!_exists) array_insert(self, _index, value); \ + } while (0) + +/// Insert a given `value` into a sorted array, using integer comparisons of +/// a given struct field (specified with a leading dot) to determine the order. +/// +/// See also `array_search_sorted_by`. +#define array_insert_sorted_by(self, field, value) \ + do { \ + unsigned _index, _exists; \ + array_search_sorted_by(self, field, (value) field, &_index, &_exists); \ + if (!_exists) array_insert(self, _index, value); \ + } while (0) + +// Private + +typedef Array(void) Array; + +/// This is not what you're looking for, see `array_delete`. +static inline void _array__delete(Array *self) { + if (self->contents) { + ts_free(self->contents); + self->contents = NULL; + self->size = 0; + self->capacity = 0; + } +} + +/// This is not what you're looking for, see `array_erase`. +static inline void _array__erase(Array *self, size_t element_size, + uint32_t index) { + ts_assert(index < self->size); + char *contents = (char *)self->contents; + memmove(contents + index * element_size, contents + (index + 1) * element_size, + (self->size - index - 1) * element_size); + self->size--; +} + +/// This is not what you're looking for, see `array_reserve`. +static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) { + if (new_capacity > self->capacity) { + if (self->contents) { + self->contents = ts_realloc(self->contents, new_capacity * element_size); + } else { + self->contents = ts_malloc(new_capacity * element_size); + } + self->capacity = new_capacity; + } +} + +/// This is not what you're looking for, see `array_assign`. +static inline void _array__assign(Array *self, const Array *other, size_t element_size) { + _array__reserve(self, element_size, other->size); + self->size = other->size; + memcpy(self->contents, other->contents, self->size * element_size); +} + +/// This is not what you're looking for, see `array_swap`. +static inline void _array__swap(Array *self, Array *other) { + Array swap = *other; + *other = *self; + *self = swap; +} + +/// This is not what you're looking for, see `array_push` or `array_grow_by`. +static inline void _array__grow(Array *self, uint32_t count, size_t element_size) { + uint32_t new_size = self->size + count; + if (new_size > self->capacity) { + uint32_t new_capacity = self->capacity * 2; + if (new_capacity < 8) new_capacity = 8; + if (new_capacity < new_size) new_capacity = new_size; + _array__reserve(self, element_size, new_capacity); + } +} + +/// This is not what you're looking for, see `array_splice`. +static inline void _array__splice(Array *self, size_t element_size, + uint32_t index, uint32_t old_count, + uint32_t new_count, const void *elements) { + uint32_t new_size = self->size + new_count - old_count; + uint32_t old_end = index + old_count; + uint32_t new_end = index + new_count; + ts_assert(old_end <= self->size); + + _array__reserve(self, element_size, new_size); + + char *contents = (char *)self->contents; + if (self->size > old_end) { + memmove( + contents + new_end * element_size, + contents + old_end * element_size, + (self->size - old_end) * element_size + ); + } + if (new_count > 0) { + if (elements) { + memcpy( + (contents + index * element_size), + elements, + new_count * element_size + ); + } else { + memset( + (contents + index * element_size), + 0, + new_count * element_size + ); + } + } + self->size += new_count - old_count; +} + +/// A binary search routine, based on Rust's `std::slice::binary_search_by`. +/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`. +#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \ + do { \ + *(_index) = start; \ + *(_exists) = false; \ + uint32_t size = (self)->size - *(_index); \ + if (size == 0) break; \ + int comparison; \ + while (size > 1) { \ + uint32_t half_size = size / 2; \ + uint32_t mid_index = *(_index) + half_size; \ + comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \ + if (comparison <= 0) *(_index) = mid_index; \ + size -= half_size; \ + } \ + comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \ + if (comparison == 0) *(_exists) = true; \ + else if (comparison < 0) *(_index) += 1; \ + } while (0) + +/// Helper macro for the `_sorted_by` routines below. This takes the left (existing) +/// parameter by reference in order to work with the generic sorting function above. +#define _compare_int(a, b) ((int)*(a) - (int)(b)) + +#ifdef _MSC_VER +#pragma warning(pop) +#elif defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_ARRAY_H_ diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/atomic.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/atomic.h new file mode 100644 index 00000000000..e680b60e521 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/atomic.h @@ -0,0 +1,68 @@ +#ifndef TREE_SITTER_ATOMIC_H_ +#define TREE_SITTER_ATOMIC_H_ + +#include +#include +#include + +#ifdef __TINYC__ + +static inline size_t atomic_load(const volatile size_t *p) { + return *p; +} + +static inline uint32_t atomic_inc(volatile uint32_t *p) { + *p += 1; + return *p; +} + +static inline uint32_t atomic_dec(volatile uint32_t *p) { + *p-= 1; + return *p; +} + +#elif defined(_WIN32) + +#include + +static inline size_t atomic_load(const volatile size_t *p) { + return *p; +} + +static inline uint32_t atomic_inc(volatile uint32_t *p) { + return InterlockedIncrement((long volatile *)p); +} + +static inline uint32_t atomic_dec(volatile uint32_t *p) { + return InterlockedDecrement((long volatile *)p); +} + +#else + +static inline size_t atomic_load(const volatile size_t *p) { +#ifdef __ATOMIC_RELAXED + return __atomic_load_n(p, __ATOMIC_RELAXED); +#else + return __sync_fetch_and_add((volatile size_t *)p, 0); +#endif +} + +static inline uint32_t atomic_inc(volatile uint32_t *p) { + #ifdef __ATOMIC_RELAXED + return __atomic_add_fetch(p, 1U, __ATOMIC_SEQ_CST); + #else + return __sync_add_and_fetch(p, 1U); + #endif +} + +static inline uint32_t atomic_dec(volatile uint32_t *p) { + #ifdef __ATOMIC_RELAXED + return __atomic_sub_fetch(p, 1U, __ATOMIC_SEQ_CST); + #else + return __sync_sub_and_fetch(p, 1U); + #endif +} + +#endif + +#endif // TREE_SITTER_ATOMIC_H_ diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/clock.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/clock.h new file mode 100644 index 00000000000..7a13185ec78 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/clock.h @@ -0,0 +1,146 @@ +#ifndef TREE_SITTER_CLOCK_H_ +#define TREE_SITTER_CLOCK_H_ + +#include +#include + +typedef uint64_t TSDuration; + +#ifdef _WIN32 + +// Windows: +// * Represent a time as a performance counter value. +// * Represent a duration as a number of performance counter ticks. + +#include +typedef uint64_t TSClock; + +static inline TSDuration duration_from_micros(uint64_t micros) { + LARGE_INTEGER frequency; + QueryPerformanceFrequency(&frequency); + return micros * (uint64_t)frequency.QuadPart / 1000000; +} + +static inline uint64_t duration_to_micros(TSDuration self) { + LARGE_INTEGER frequency; + QueryPerformanceFrequency(&frequency); + return self * 1000000 / (uint64_t)frequency.QuadPart; +} + +static inline TSClock clock_null(void) { + return 0; +} + +static inline TSClock clock_now(void) { + LARGE_INTEGER result; + QueryPerformanceCounter(&result); + return (uint64_t)result.QuadPart; +} + +static inline TSClock clock_after(TSClock base, TSDuration duration) { + return base + duration; +} + +static inline bool clock_is_null(TSClock self) { + return !self; +} + +static inline bool clock_is_gt(TSClock self, TSClock other) { + return self > other; +} + +#elif defined(CLOCK_MONOTONIC) + +// POSIX with monotonic clock support (Linux, macOS) +// * Represent a time as a monotonic (seconds, nanoseconds) pair. +// * Represent a duration as a number of microseconds. +// +// On these platforms, parse timeouts will correspond accurately to +// real time, regardless of what other processes are running. + +#include +typedef struct timespec TSClock; + +static inline TSDuration duration_from_micros(uint64_t micros) { + return micros; +} + +static inline uint64_t duration_to_micros(TSDuration self) { + return self; +} + +static inline TSClock clock_now(void) { + TSClock result; + clock_gettime(CLOCK_MONOTONIC, &result); + return result; +} + +static inline TSClock clock_null(void) { + return (TSClock) {0, 0}; +} + +static inline TSClock clock_after(TSClock base, TSDuration duration) { + TSClock result = base; + result.tv_sec += duration / 1000000; + result.tv_nsec += (duration % 1000000) * 1000; + if (result.tv_nsec >= 1000000000) { + result.tv_nsec -= 1000000000; + ++(result.tv_sec); + } + return result; +} + +static inline bool clock_is_null(TSClock self) { + return !self.tv_sec && !self.tv_nsec; +} + +static inline bool clock_is_gt(TSClock self, TSClock other) { + if (self.tv_sec > other.tv_sec) return true; + if (self.tv_sec < other.tv_sec) return false; + return self.tv_nsec > other.tv_nsec; +} + +#else + +// POSIX without monotonic clock support +// * Represent a time as a process clock value. +// * Represent a duration as a number of process clock ticks. +// +// On these platforms, parse timeouts may be affected by other processes, +// which is not ideal, but is better than using a non-monotonic time API +// like `gettimeofday`. + +#include +typedef uint64_t TSClock; + +static inline TSDuration duration_from_micros(uint64_t micros) { + return micros * (uint64_t)CLOCKS_PER_SEC / 1000000; +} + +static inline uint64_t duration_to_micros(TSDuration self) { + return self * 1000000 / (uint64_t)CLOCKS_PER_SEC; +} + +static inline TSClock clock_null(void) { + return 0; +} + +static inline TSClock clock_now(void) { + return (uint64_t)clock(); +} + +static inline TSClock clock_after(TSClock base, TSDuration duration) { + return base + duration; +} + +static inline bool clock_is_null(TSClock self) { + return !self; +} + +static inline bool clock_is_gt(TSClock self, TSClock other) { + return self > other; +} + +#endif + +#endif // TREE_SITTER_CLOCK_H_ diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/error_costs.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/error_costs.h new file mode 100644 index 00000000000..32d3666a661 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/error_costs.h @@ -0,0 +1,11 @@ +#ifndef TREE_SITTER_ERROR_COSTS_H_ +#define TREE_SITTER_ERROR_COSTS_H_ + +#define ERROR_STATE 0 +#define ERROR_COST_PER_RECOVERY 500 +#define ERROR_COST_PER_MISSING_TREE 110 +#define ERROR_COST_PER_SKIPPED_TREE 100 +#define ERROR_COST_PER_SKIPPED_LINE 30 +#define ERROR_COST_PER_SKIPPED_CHAR 1 + +#endif diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/get_changed_ranges.c b/Externals/crystaledit/editlib/tree-sitter/lib/src/get_changed_ranges.c new file mode 100644 index 00000000000..11084c33d7c --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/get_changed_ranges.c @@ -0,0 +1,523 @@ +#include "./get_changed_ranges.h" +#include "./subtree.h" +#include "./language.h" +#include "./error_costs.h" +#include "./tree_cursor.h" +#include "./ts_assert.h" + +// #define DEBUG_GET_CHANGED_RANGES + +static void ts_range_array_add( + TSRangeArray *self, + Length start, + Length end +) { + if (self->size > 0) { + TSRange *last_range = array_back(self); + if (start.bytes <= last_range->end_byte) { + last_range->end_byte = end.bytes; + last_range->end_point = end.extent; + return; + } + } + + if (start.bytes < end.bytes) { + TSRange range = { start.extent, end.extent, start.bytes, end.bytes }; + array_push(self, range); + } +} + +bool ts_range_array_intersects( + const TSRangeArray *self, + unsigned start_index, + uint32_t start_byte, + uint32_t end_byte +) { + for (unsigned i = start_index; i < self->size; i++) { + TSRange *range = array_get(self, i); + if (range->end_byte > start_byte) { + if (range->start_byte >= end_byte) break; + return true; + } + } + return false; +} + +void ts_range_array_get_changed_ranges( + const TSRange *old_ranges, unsigned old_range_count, + const TSRange *new_ranges, unsigned new_range_count, + TSRangeArray *differences +) { + unsigned new_index = 0; + unsigned old_index = 0; + Length current_position = length_zero(); + bool in_old_range = false; + bool in_new_range = false; + + while (old_index < old_range_count || new_index < new_range_count) { + const TSRange *old_range = &old_ranges[old_index]; + const TSRange *new_range = &new_ranges[new_index]; + + Length next_old_position; + if (in_old_range) { + next_old_position = (Length) {old_range->end_byte, old_range->end_point}; + } else if (old_index < old_range_count) { + next_old_position = (Length) {old_range->start_byte, old_range->start_point}; + } else { + next_old_position = LENGTH_MAX; + } + + Length next_new_position; + if (in_new_range) { + next_new_position = (Length) {new_range->end_byte, new_range->end_point}; + } else if (new_index < new_range_count) { + next_new_position = (Length) {new_range->start_byte, new_range->start_point}; + } else { + next_new_position = LENGTH_MAX; + } + + if (next_old_position.bytes < next_new_position.bytes) { + if (in_old_range != in_new_range) { + ts_range_array_add(differences, current_position, next_old_position); + } + if (in_old_range) old_index++; + current_position = next_old_position; + in_old_range = !in_old_range; + } else if (next_new_position.bytes < next_old_position.bytes) { + if (in_old_range != in_new_range) { + ts_range_array_add(differences, current_position, next_new_position); + } + if (in_new_range) new_index++; + current_position = next_new_position; + in_new_range = !in_new_range; + } else { + if (in_old_range != in_new_range) { + ts_range_array_add(differences, current_position, next_new_position); + } + if (in_old_range) old_index++; + if (in_new_range) new_index++; + in_old_range = !in_old_range; + in_new_range = !in_new_range; + current_position = next_new_position; + } + } +} + +typedef struct { + TreeCursor cursor; + const TSLanguage *language; + unsigned visible_depth; + bool in_padding; + Subtree prev_external_token; +} Iterator; + +static Iterator iterator_new( + TreeCursor *cursor, + const Subtree *tree, + const TSLanguage *language +) { + array_clear(&cursor->stack); + array_push(&cursor->stack, ((TreeCursorEntry) { + .subtree = tree, + .position = length_zero(), + .child_index = 0, + .structural_child_index = 0, + })); + return (Iterator) { + .cursor = *cursor, + .language = language, + .visible_depth = 1, + .in_padding = false, + .prev_external_token = NULL_SUBTREE, + }; +} + +static bool iterator_done(Iterator *self) { + return self->cursor.stack.size == 0; +} + +static Length iterator_start_position(Iterator *self) { + TreeCursorEntry entry = *array_back(&self->cursor.stack); + if (self->in_padding) { + return entry.position; + } else { + return length_add(entry.position, ts_subtree_padding(*entry.subtree)); + } +} + +static Length iterator_end_position(Iterator *self) { + TreeCursorEntry entry = *array_back(&self->cursor.stack); + Length result = length_add(entry.position, ts_subtree_padding(*entry.subtree)); + if (self->in_padding) { + return result; + } else { + return length_add(result, ts_subtree_size(*entry.subtree)); + } +} + +static bool iterator_tree_is_visible(const Iterator *self) { + TreeCursorEntry entry = *array_back(&self->cursor.stack); + if (ts_subtree_visible(*entry.subtree)) return true; + if (self->cursor.stack.size > 1) { + Subtree parent = *array_get(&self->cursor.stack, self->cursor.stack.size - 2)->subtree; + return ts_language_alias_at( + self->language, + parent.ptr->production_id, + entry.structural_child_index + ) != 0; + } + return false; +} + +static void iterator_get_visible_state( + const Iterator *self, + Subtree *tree, + TSSymbol *alias_symbol, + uint32_t *start_byte +) { + uint32_t i = self->cursor.stack.size - 1; + + if (self->in_padding) { + if (i == 0) return; + i--; + } + + for (; i + 1 > 0; i--) { + TreeCursorEntry entry = *array_get(&self->cursor.stack, i); + + if (i > 0) { + const Subtree *parent = array_get(&self->cursor.stack, i - 1)->subtree; + *alias_symbol = ts_language_alias_at( + self->language, + parent->ptr->production_id, + entry.structural_child_index + ); + } + + if (ts_subtree_visible(*entry.subtree) || *alias_symbol) { + *tree = *entry.subtree; + *start_byte = entry.position.bytes; + break; + } + } +} + +static void iterator_ascend(Iterator *self) { + if (iterator_done(self)) return; + if (iterator_tree_is_visible(self) && !self->in_padding) self->visible_depth--; + if (array_back(&self->cursor.stack)->child_index > 0) self->in_padding = false; + self->cursor.stack.size--; +} + +static bool iterator_descend(Iterator *self, uint32_t goal_position) { + if (self->in_padding) return false; + + bool did_descend = false; + do { + did_descend = false; + TreeCursorEntry entry = *array_back(&self->cursor.stack); + Length position = entry.position; + uint32_t structural_child_index = 0; + for (uint32_t i = 0, n = ts_subtree_child_count(*entry.subtree); i < n; i++) { + const Subtree *child = &ts_subtree_children(*entry.subtree)[i]; + Length child_left = length_add(position, ts_subtree_padding(*child)); + Length child_right = length_add(child_left, ts_subtree_size(*child)); + + if (child_right.bytes > goal_position) { + array_push(&self->cursor.stack, ((TreeCursorEntry) { + .subtree = child, + .position = position, + .child_index = i, + .structural_child_index = structural_child_index, + })); + + if (iterator_tree_is_visible(self)) { + if (child_left.bytes > goal_position) { + self->in_padding = true; + } else { + self->visible_depth++; + } + return true; + } + + did_descend = true; + break; + } + + position = child_right; + if (!ts_subtree_extra(*child)) structural_child_index++; + Subtree last_external_token = ts_subtree_last_external_token(*child); + if (last_external_token.ptr) { + self->prev_external_token = last_external_token; + } + } + } while (did_descend); + + return false; +} + +static void iterator_advance(Iterator *self) { + if (self->in_padding) { + self->in_padding = false; + if (iterator_tree_is_visible(self)) { + self->visible_depth++; + } else { + iterator_descend(self, 0); + } + return; + } + + for (;;) { + if (iterator_tree_is_visible(self)) self->visible_depth--; + TreeCursorEntry entry = array_pop(&self->cursor.stack); + if (iterator_done(self)) return; + + const Subtree *parent = array_back(&self->cursor.stack)->subtree; + uint32_t child_index = entry.child_index + 1; + Subtree last_external_token = ts_subtree_last_external_token(*entry.subtree); + if (last_external_token.ptr) { + self->prev_external_token = last_external_token; + } + if (ts_subtree_child_count(*parent) > child_index) { + Length position = length_add(entry.position, ts_subtree_total_size(*entry.subtree)); + uint32_t structural_child_index = entry.structural_child_index; + if (!ts_subtree_extra(*entry.subtree)) structural_child_index++; + const Subtree *next_child = &ts_subtree_children(*parent)[child_index]; + + array_push(&self->cursor.stack, ((TreeCursorEntry) { + .subtree = next_child, + .position = position, + .child_index = child_index, + .structural_child_index = structural_child_index, + })); + + if (iterator_tree_is_visible(self)) { + if (ts_subtree_padding(*next_child).bytes > 0) { + self->in_padding = true; + } else { + self->visible_depth++; + } + } else { + iterator_descend(self, 0); + } + break; + } + } +} + +typedef enum { + IteratorDiffers, + IteratorMayDiffer, + IteratorMatches, +} IteratorComparison; + +static IteratorComparison iterator_compare( + const Iterator *old_iter, + const Iterator *new_iter +) { + Subtree old_tree = NULL_SUBTREE; + Subtree new_tree = NULL_SUBTREE; + uint32_t old_start = 0; + uint32_t new_start = 0; + TSSymbol old_alias_symbol = 0; + TSSymbol new_alias_symbol = 0; + iterator_get_visible_state(old_iter, &old_tree, &old_alias_symbol, &old_start); + iterator_get_visible_state(new_iter, &new_tree, &new_alias_symbol, &new_start); + TSSymbol old_symbol = ts_subtree_symbol(old_tree); + TSSymbol new_symbol = ts_subtree_symbol(new_tree); + + if (!old_tree.ptr && !new_tree.ptr) return IteratorMatches; + if (!old_tree.ptr || !new_tree.ptr) return IteratorDiffers; + if (old_alias_symbol != new_alias_symbol || old_symbol != new_symbol) return IteratorDiffers; + + uint32_t old_size = ts_subtree_size(old_tree).bytes; + uint32_t new_size = ts_subtree_size(new_tree).bytes; + TSStateId old_state = ts_subtree_parse_state(old_tree); + TSStateId new_state = ts_subtree_parse_state(new_tree); + bool old_has_external_tokens = ts_subtree_has_external_tokens(old_tree); + bool new_has_external_tokens = ts_subtree_has_external_tokens(new_tree); + uint32_t old_error_cost = ts_subtree_error_cost(old_tree); + uint32_t new_error_cost = ts_subtree_error_cost(new_tree); + + if ( + old_start != new_start || + old_symbol == ts_builtin_sym_error || + old_size != new_size || + old_state == TS_TREE_STATE_NONE || + new_state == TS_TREE_STATE_NONE || + ((old_state == ERROR_STATE) != (new_state == ERROR_STATE)) || + old_error_cost != new_error_cost || + old_has_external_tokens != new_has_external_tokens || + ts_subtree_has_changes(old_tree) || + ( + old_has_external_tokens && + !ts_subtree_external_scanner_state_eq(old_iter->prev_external_token, new_iter->prev_external_token) + ) + ) { + return IteratorMayDiffer; + } + + return IteratorMatches; +} + +#ifdef DEBUG_GET_CHANGED_RANGES +static inline void iterator_print_state(Iterator *self) { + TreeCursorEntry entry = *array_back(&self->cursor.stack); + TSPoint start = iterator_start_position(self).extent; + TSPoint end = iterator_end_position(self).extent; + const char *name = ts_language_symbol_name(self->language, ts_subtree_symbol(*entry.subtree)); + printf( + "(%-25s %s\t depth:%u [%u, %u] - [%u, %u])", + name, self->in_padding ? "(p)" : " ", + self->visible_depth, + start.row, start.column, + end.row, end.column + ); +} +#endif + +unsigned ts_subtree_get_changed_ranges( + const Subtree *old_tree, const Subtree *new_tree, + TreeCursor *cursor1, TreeCursor *cursor2, + const TSLanguage *language, + const TSRangeArray *included_range_differences, + TSRange **ranges +) { + TSRangeArray results = array_new(); + + Iterator old_iter = iterator_new(cursor1, old_tree, language); + Iterator new_iter = iterator_new(cursor2, new_tree, language); + + unsigned included_range_difference_index = 0; + + Length position = iterator_start_position(&old_iter); + Length next_position = iterator_start_position(&new_iter); + if (position.bytes < next_position.bytes) { + ts_range_array_add(&results, position, next_position); + position = next_position; + } else if (position.bytes > next_position.bytes) { + ts_range_array_add(&results, next_position, position); + next_position = position; + } + + do { + #ifdef DEBUG_GET_CHANGED_RANGES + printf("At [%-2u, %-2u] Compare ", position.extent.row, position.extent.column); + iterator_print_state(&old_iter); + printf("\tvs\t"); + iterator_print_state(&new_iter); + puts(""); + #endif + + // Compare the old and new subtrees. + IteratorComparison comparison = iterator_compare(&old_iter, &new_iter); + + // Even if the two subtrees appear to be identical, they could differ + // internally if they contain a range of text that was previously + // excluded from the parse, and is now included, or vice-versa. + if (comparison == IteratorMatches && ts_range_array_intersects( + included_range_differences, + included_range_difference_index, + position.bytes, + iterator_end_position(&old_iter).bytes + )) { + comparison = IteratorMayDiffer; + } + + bool is_changed = false; + switch (comparison) { + // If the subtrees are definitely identical, move to the end + // of both subtrees. + case IteratorMatches: + next_position = iterator_end_position(&old_iter); + break; + + // If the subtrees might differ internally, descend into both + // subtrees, finding the first child that spans the current position. + case IteratorMayDiffer: + if (iterator_descend(&old_iter, position.bytes)) { + if (!iterator_descend(&new_iter, position.bytes)) { + is_changed = true; + next_position = iterator_end_position(&old_iter); + } + } else if (iterator_descend(&new_iter, position.bytes)) { + is_changed = true; + next_position = iterator_end_position(&new_iter); + } else { + next_position = length_min( + iterator_end_position(&old_iter), + iterator_end_position(&new_iter) + ); + } + break; + + // If the subtrees are different, record a change and then move + // to the end of both subtrees. + case IteratorDiffers: + is_changed = true; + next_position = length_min( + iterator_end_position(&old_iter), + iterator_end_position(&new_iter) + ); + break; + } + + // Ensure that both iterators are caught up to the current position. + while ( + !iterator_done(&old_iter) && + iterator_end_position(&old_iter).bytes <= next_position.bytes + ) iterator_advance(&old_iter); + while ( + !iterator_done(&new_iter) && + iterator_end_position(&new_iter).bytes <= next_position.bytes + ) iterator_advance(&new_iter); + + // Ensure that both iterators are at the same depth in the tree. + while (old_iter.visible_depth > new_iter.visible_depth) { + iterator_ascend(&old_iter); + } + while (new_iter.visible_depth > old_iter.visible_depth) { + iterator_ascend(&new_iter); + } + + if (is_changed) { + #ifdef DEBUG_GET_CHANGED_RANGES + printf( + " change: [[%u, %u] - [%u, %u]]\n", + position.extent.row + 1, position.extent.column, + next_position.extent.row + 1, next_position.extent.column + ); + #endif + + ts_range_array_add(&results, position, next_position); + } + + position = next_position; + + // Keep track of the current position in the included range differences + // array in order to avoid scanning the entire array on each iteration. + while (included_range_difference_index < included_range_differences->size) { + const TSRange *range = array_get(included_range_differences, + included_range_difference_index + ); + if (range->end_byte <= position.bytes) { + included_range_difference_index++; + } else { + break; + } + } + } while (!iterator_done(&old_iter) && !iterator_done(&new_iter)); + + Length old_size = ts_subtree_total_size(*old_tree); + Length new_size = ts_subtree_total_size(*new_tree); + if (old_size.bytes < new_size.bytes) { + ts_range_array_add(&results, old_size, new_size); + } else if (new_size.bytes < old_size.bytes) { + ts_range_array_add(&results, new_size, old_size); + } + + *cursor1 = old_iter.cursor; + *cursor2 = new_iter.cursor; + *ranges = results.contents; + return results.size; +} diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/get_changed_ranges.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/get_changed_ranges.h new file mode 100644 index 00000000000..a1f1dbb4304 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/get_changed_ranges.h @@ -0,0 +1,36 @@ +#ifndef TREE_SITTER_GET_CHANGED_RANGES_H_ +#define TREE_SITTER_GET_CHANGED_RANGES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "./tree_cursor.h" +#include "./subtree.h" + +typedef Array(TSRange) TSRangeArray; + +void ts_range_array_get_changed_ranges( + const TSRange *old_ranges, unsigned old_range_count, + const TSRange *new_ranges, unsigned new_range_count, + TSRangeArray *differences +); + +bool ts_range_array_intersects( + const TSRangeArray *self, unsigned start_index, + uint32_t start_byte, uint32_t end_byte +); + +unsigned ts_subtree_get_changed_ranges( + const Subtree *old_tree, const Subtree *new_tree, + TreeCursor *cursor1, TreeCursor *cursor2, + const TSLanguage *language, + const TSRangeArray *included_range_differences, + TSRange **ranges +); + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_GET_CHANGED_RANGES_H_ diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/host.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/host.h new file mode 100644 index 00000000000..a07e9f89495 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/host.h @@ -0,0 +1,21 @@ + +// Determine endian and pointer size based on known defines. +// TS_BIG_ENDIAN and TS_PTR_SIZE can be set as -D compiler arguments +// to override this. + +#if !defined(TS_BIG_ENDIAN) +#if (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) \ + || (defined( __APPLE_CC__) && (defined(__ppc__) || defined(__ppc64__))) +#define TS_BIG_ENDIAN 1 +#else +#define TS_BIG_ENDIAN 0 +#endif +#endif + +#if !defined(TS_PTR_SIZE) +#if UINTPTR_MAX == 0xFFFFFFFF +#define TS_PTR_SIZE 32 +#else +#define TS_PTR_SIZE 64 +#endif +#endif diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/language.c b/Externals/crystaledit/editlib/tree-sitter/lib/src/language.c new file mode 100644 index 00000000000..2dce6998391 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/language.c @@ -0,0 +1,293 @@ +#include "./language.h" +#include "./wasm_store.h" +#include "tree_sitter/api.h" +#include + +const TSLanguage *ts_language_copy(const TSLanguage *self) { + if (self && ts_language_is_wasm(self)) { + ts_wasm_language_retain(self); + } + return self; +} + +void ts_language_delete(const TSLanguage *self) { + if (self && ts_language_is_wasm(self)) { + ts_wasm_language_release(self); + } +} + +uint32_t ts_language_symbol_count(const TSLanguage *self) { + return self->symbol_count + self->alias_count; +} + +uint32_t ts_language_state_count(const TSLanguage *self) { + return self->state_count; +} + +const TSSymbol *ts_language_supertypes(const TSLanguage *self, uint32_t *length) { + if (self->abi_version >= LANGUAGE_VERSION_WITH_RESERVED_WORDS) { + *length = self->supertype_count; + return self->supertype_symbols; + } else { + *length = 0; + return NULL; + } +} + +const TSSymbol *ts_language_subtypes( + const TSLanguage *self, + TSSymbol supertype, + uint32_t *length +) { + if (self->abi_version < LANGUAGE_VERSION_WITH_RESERVED_WORDS || !ts_language_symbol_metadata(self, supertype).supertype) { + *length = 0; + return NULL; + } + + TSMapSlice slice = self->supertype_map_slices[supertype]; + *length = slice.length; + return &self->supertype_map_entries[slice.index]; +} + +uint32_t ts_language_version(const TSLanguage *self) { + return self->abi_version; +} + +uint32_t ts_language_abi_version(const TSLanguage *self) { + return self->abi_version; +} + +const TSLanguageMetadata *ts_language_metadata(const TSLanguage *self) { + return self->abi_version >= LANGUAGE_VERSION_WITH_RESERVED_WORDS ? &self->metadata : NULL; +} + +const char *ts_language_name(const TSLanguage *self) { + return self->abi_version >= LANGUAGE_VERSION_WITH_RESERVED_WORDS ? self->name : NULL; +} + +uint32_t ts_language_field_count(const TSLanguage *self) { + return self->field_count; +} + +void ts_language_table_entry( + const TSLanguage *self, + TSStateId state, + TSSymbol symbol, + TableEntry *result +) { + if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { + result->action_count = 0; + result->is_reusable = false; + result->actions = NULL; + } else { + ts_assert(symbol < self->token_count); + uint32_t action_index = ts_language_lookup(self, state, symbol); + const TSParseActionEntry *entry = &self->parse_actions[action_index]; + result->action_count = entry->entry.count; + result->is_reusable = entry->entry.reusable; + result->actions = (const TSParseAction *)(entry + 1); + } +} + +TSLexerMode ts_language_lex_mode_for_state( + const TSLanguage *self, + TSStateId state +) { + if (self->abi_version < 15) { + TSLexMode mode = ((const TSLexMode *)self->lex_modes)[state]; + return (TSLexerMode) { + .lex_state = mode.lex_state, + .external_lex_state = mode.external_lex_state, + .reserved_word_set_id = 0, + }; + } else { + return self->lex_modes[state]; + } +} + +bool ts_language_is_reserved_word( + const TSLanguage *self, + TSStateId state, + TSSymbol symbol +) { + TSLexerMode lex_mode = ts_language_lex_mode_for_state(self, state); + if (lex_mode.reserved_word_set_id > 0) { + unsigned start = lex_mode.reserved_word_set_id * self->max_reserved_word_set_size; + unsigned end = start + self->max_reserved_word_set_size; + for (unsigned i = start; i < end; i++) { + if (self->reserved_words[i] == symbol) return true; + if (self->reserved_words[i] == 0) break; + } + } + return false; +} + +TSSymbolMetadata ts_language_symbol_metadata( + const TSLanguage *self, + TSSymbol symbol +) { + if (symbol == ts_builtin_sym_error) { + return (TSSymbolMetadata) {.visible = true, .named = true}; + } else if (symbol == ts_builtin_sym_error_repeat) { + return (TSSymbolMetadata) {.visible = false, .named = false}; + } else { + return self->symbol_metadata[symbol]; + } +} + +TSSymbol ts_language_public_symbol( + const TSLanguage *self, + TSSymbol symbol +) { + if (symbol == ts_builtin_sym_error) return symbol; + return self->public_symbol_map[symbol]; +} + +TSStateId ts_language_next_state( + const TSLanguage *self, + TSStateId state, + TSSymbol symbol +) { + if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) { + return 0; + } else if (symbol < self->token_count) { + uint32_t count; + const TSParseAction *actions = ts_language_actions(self, state, symbol, &count); + if (count > 0) { + TSParseAction action = actions[count - 1]; + if (action.type == TSParseActionTypeShift) { + return action.shift.extra ? state : action.shift.state; + } + } + return 0; + } else { + return ts_language_lookup(self, state, symbol); + } +} + +const char *ts_language_symbol_name( + const TSLanguage *self, + TSSymbol symbol +) { + if (symbol == ts_builtin_sym_error) { + return "ERROR"; + } else if (symbol == ts_builtin_sym_error_repeat) { + return "_ERROR"; + } else if (symbol < ts_language_symbol_count(self)) { + return self->symbol_names[symbol]; + } else { + return NULL; + } +} + +TSSymbol ts_language_symbol_for_name( + const TSLanguage *self, + const char *string, + uint32_t length, + bool is_named +) { + if (is_named && !strncmp(string, "ERROR", length)) return ts_builtin_sym_error; + uint16_t count = (uint16_t)ts_language_symbol_count(self); + for (TSSymbol i = 0; i < count; i++) { + TSSymbolMetadata metadata = ts_language_symbol_metadata(self, i); + if ((!metadata.visible && !metadata.supertype) || metadata.named != is_named) continue; + const char *symbol_name = self->symbol_names[i]; + if (!strncmp(symbol_name, string, length) && !symbol_name[length]) { + return self->public_symbol_map[i]; + } + } + return 0; +} + +TSSymbolType ts_language_symbol_type( + const TSLanguage *self, + TSSymbol symbol +) { + TSSymbolMetadata metadata = ts_language_symbol_metadata(self, symbol); + if (metadata.named && metadata.visible) { + return TSSymbolTypeRegular; + } else if (metadata.visible) { + return TSSymbolTypeAnonymous; + } else if (metadata.supertype) { + return TSSymbolTypeSupertype; + } else { + return TSSymbolTypeAuxiliary; + } +} + +const char *ts_language_field_name_for_id( + const TSLanguage *self, + TSFieldId id +) { + uint32_t count = ts_language_field_count(self); + if (count && id <= count) { + return self->field_names[id]; + } else { + return NULL; + } +} + +TSFieldId ts_language_field_id_for_name( + const TSLanguage *self, + const char *name, + uint32_t name_length +) { + uint16_t count = (uint16_t)ts_language_field_count(self); + for (TSSymbol i = 1; i < count + 1; i++) { + switch (strncmp(name, self->field_names[i], name_length)) { + case 0: + if (self->field_names[i][name_length] == 0) return i; + break; + case -1: + return 0; + default: + break; + } + } + return 0; +} + +TSLookaheadIterator *ts_lookahead_iterator_new(const TSLanguage *self, TSStateId state) { + if (state >= self->state_count) return NULL; + LookaheadIterator *iterator = ts_malloc(sizeof(LookaheadIterator)); + *iterator = ts_language_lookaheads(self, state); + return (TSLookaheadIterator *)iterator; +} + +void ts_lookahead_iterator_delete(TSLookaheadIterator *self) { + ts_free(self); +} + +bool ts_lookahead_iterator_reset_state(TSLookaheadIterator * self, TSStateId state) { + LookaheadIterator *iterator = (LookaheadIterator *)self; + if (state >= iterator->language->state_count) return false; + *iterator = ts_language_lookaheads(iterator->language, state); + return true; +} + +const TSLanguage *ts_lookahead_iterator_language(const TSLookaheadIterator *self) { + const LookaheadIterator *iterator = (const LookaheadIterator *)self; + return iterator->language; +} + +bool ts_lookahead_iterator_reset(TSLookaheadIterator *self, const TSLanguage *language, TSStateId state) { + if (state >= language->state_count) return false; + LookaheadIterator *iterator = (LookaheadIterator *)self; + *iterator = ts_language_lookaheads(language, state); + return true; +} + +bool ts_lookahead_iterator_next(TSLookaheadIterator *self) { + LookaheadIterator *iterator = (LookaheadIterator *)self; + return ts_lookahead_iterator__next(iterator); +} + +TSSymbol ts_lookahead_iterator_current_symbol(const TSLookaheadIterator *self) { + const LookaheadIterator *iterator = (const LookaheadIterator *)self; + return iterator->symbol; +} + +const char *ts_lookahead_iterator_current_symbol_name(const TSLookaheadIterator *self) { + const LookaheadIterator *iterator = (const LookaheadIterator *)self; + return ts_language_symbol_name(iterator->language, iterator->symbol); +} diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/language.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/language.h new file mode 100644 index 00000000000..518c06bfc55 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/language.h @@ -0,0 +1,293 @@ +#ifndef TREE_SITTER_LANGUAGE_H_ +#define TREE_SITTER_LANGUAGE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "./subtree.h" +#include "./parser.h" + +#define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1) + +#define LANGUAGE_VERSION_WITH_RESERVED_WORDS 15 +#define LANGUAGE_VERSION_WITH_PRIMARY_STATES 14 + +typedef struct { + const TSParseAction *actions; + uint32_t action_count; + bool is_reusable; +} TableEntry; + +typedef struct { + const TSLanguage *language; + const uint16_t *data; + const uint16_t *group_end; + TSStateId state; + uint16_t table_value; + uint16_t section_index; + uint16_t group_count; + bool is_small_state; + + const TSParseAction *actions; + TSSymbol symbol; + TSStateId next_state; + uint16_t action_count; +} LookaheadIterator; + +void ts_language_table_entry(const TSLanguage *self, TSStateId state, TSSymbol symbol, TableEntry *result); +TSLexerMode ts_language_lex_mode_for_state(const TSLanguage *self, TSStateId state); +bool ts_language_is_reserved_word(const TSLanguage *self, TSStateId state, TSSymbol symbol); +TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *self, TSSymbol symbol); +TSSymbol ts_language_public_symbol(const TSLanguage *self, TSSymbol symbol); + +static inline const TSParseAction *ts_language_actions( + const TSLanguage *self, + TSStateId state, + TSSymbol symbol, + uint32_t *count +) { + TableEntry entry; + ts_language_table_entry(self, state, symbol, &entry); + *count = entry.action_count; + return entry.actions; +} + +static inline bool ts_language_has_reduce_action( + const TSLanguage *self, + TSStateId state, + TSSymbol symbol +) { + TableEntry entry; + ts_language_table_entry(self, state, symbol, &entry); + return entry.action_count > 0 && entry.actions[0].type == TSParseActionTypeReduce; +} + +// Lookup the table value for a given symbol and state. +// +// For non-terminal symbols, the table value represents a successor state. +// For terminal symbols, it represents an index in the actions table. +// For 'large' parse states, this is a direct lookup. For 'small' parse +// states, this requires searching through the symbol groups to find +// the given symbol. +static inline uint16_t ts_language_lookup( + const TSLanguage *self, + TSStateId state, + TSSymbol symbol +) { + if (state >= self->large_state_count) { + uint32_t index = self->small_parse_table_map[state - self->large_state_count]; + const uint16_t *data = &self->small_parse_table[index]; + uint16_t group_count = *(data++); + for (unsigned i = 0; i < group_count; i++) { + uint16_t section_value = *(data++); + uint16_t symbol_count = *(data++); + for (unsigned j = 0; j < symbol_count; j++) { + if (*(data++) == symbol) return section_value; + } + } + return 0; + } else { + return self->parse_table[state * self->symbol_count + symbol]; + } +} + +static inline bool ts_language_has_actions( + const TSLanguage *self, + TSStateId state, + TSSymbol symbol +) { + return ts_language_lookup(self, state, symbol) != 0; +} + +// Iterate over all of the symbols that are valid in the given state. +// +// For 'large' parse states, this just requires iterating through +// all possible symbols and checking the parse table for each one. +// For 'small' parse states, this exploits the structure of the +// table to only visit the valid symbols. +static inline LookaheadIterator ts_language_lookaheads( + const TSLanguage *self, + TSStateId state +) { + bool is_small_state = state >= self->large_state_count; + const uint16_t *data; + const uint16_t *group_end = NULL; + uint16_t group_count = 0; + if (is_small_state) { + uint32_t index = self->small_parse_table_map[state - self->large_state_count]; + data = &self->small_parse_table[index]; + group_end = data + 1; + group_count = *data; + } else { + data = &self->parse_table[state * self->symbol_count] - 1; + } + return (LookaheadIterator) { + .language = self, + .data = data, + .group_end = group_end, + .group_count = group_count, + .is_small_state = is_small_state, + .symbol = UINT16_MAX, + .next_state = 0, + }; +} + +static inline bool ts_lookahead_iterator__next(LookaheadIterator *self) { + // For small parse states, valid symbols are listed explicitly, + // grouped by their value. There's no need to look up the actions + // again until moving to the next group. + if (self->is_small_state) { + self->data++; + if (self->data == self->group_end) { + if (self->group_count == 0) return false; + self->group_count--; + self->table_value = *(self->data++); + unsigned symbol_count = *(self->data++); + self->group_end = self->data + symbol_count; + self->symbol = *self->data; + } else { + self->symbol = *self->data; + return true; + } + } + + // For large parse states, iterate through every symbol until one + // is found that has valid actions. + else { + do { + self->data++; + self->symbol++; + if (self->symbol >= self->language->symbol_count) return false; + self->table_value = *self->data; + } while (!self->table_value); + } + + // Depending on if the symbols is terminal or non-terminal, the table value either + // represents a list of actions or a successor state. + if (self->symbol < self->language->token_count) { + const TSParseActionEntry *entry = &self->language->parse_actions[self->table_value]; + self->action_count = entry->entry.count; + self->actions = (const TSParseAction *)(entry + 1); + self->next_state = 0; + } else { + self->action_count = 0; + self->next_state = self->table_value; + } + return true; +} + +// Whether the state is a "primary state". If this returns false, it indicates that there exists +// another state that behaves identically to this one with respect to query analysis. +static inline bool ts_language_state_is_primary( + const TSLanguage *self, + TSStateId state +) { + if (self->abi_version >= LANGUAGE_VERSION_WITH_PRIMARY_STATES) { + return state == self->primary_state_ids[state]; + } else { + return true; + } +} + +static inline const bool *ts_language_enabled_external_tokens( + const TSLanguage *self, + unsigned external_scanner_state +) { + if (external_scanner_state == 0) { + return NULL; + } else { + return self->external_scanner.states + self->external_token_count * external_scanner_state; + } +} + +static inline const TSSymbol *ts_language_alias_sequence( + const TSLanguage *self, + uint32_t production_id +) { + return production_id ? + &self->alias_sequences[production_id * self->max_alias_sequence_length] : + NULL; +} + +static inline TSSymbol ts_language_alias_at( + const TSLanguage *self, + uint32_t production_id, + uint32_t child_index +) { + return production_id ? + self->alias_sequences[production_id * self->max_alias_sequence_length + child_index] : + 0; +} + +static inline void ts_language_field_map( + const TSLanguage *self, + uint32_t production_id, + const TSFieldMapEntry **start, + const TSFieldMapEntry **end +) { + if (self->field_count == 0) { + *start = NULL; + *end = NULL; + return; + } + + TSMapSlice slice = self->field_map_slices[production_id]; + *start = &self->field_map_entries[slice.index]; + *end = &self->field_map_entries[slice.index] + slice.length; +} + +static inline void ts_language_aliases_for_symbol( + const TSLanguage *self, + TSSymbol original_symbol, + const TSSymbol **start, + const TSSymbol **end +) { + *start = &self->public_symbol_map[original_symbol]; + *end = *start + 1; + + unsigned idx = 0; + for (;;) { + TSSymbol symbol = self->alias_map[idx++]; + if (symbol == 0 || symbol > original_symbol) break; + uint16_t count = self->alias_map[idx++]; + if (symbol == original_symbol) { + *start = &self->alias_map[idx]; + *end = &self->alias_map[idx + count]; + break; + } + idx += count; + } +} + +static inline void ts_language_write_symbol_as_dot_string( + const TSLanguage *self, + FILE *f, + TSSymbol symbol +) { + const char *name = ts_language_symbol_name(self, symbol); + for (const char *chr = name; *chr; chr++) { + switch (*chr) { + case '"': + case '\\': + fputc('\\', f); + fputc(*chr, f); + break; + case '\n': + fputs("\\n", f); + break; + case '\t': + fputs("\\t", f); + break; + default: + fputc(*chr, f); + break; + } + } +} + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_LANGUAGE_H_ diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/length.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/length.h new file mode 100644 index 00000000000..ddf156ce098 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/length.h @@ -0,0 +1,52 @@ +#ifndef TREE_SITTER_LENGTH_H_ +#define TREE_SITTER_LENGTH_H_ + +#include +#include +#include "./point.h" +#include "tree_sitter/api.h" + +typedef struct { + uint32_t bytes; + TSPoint extent; +} Length; + +static const Length LENGTH_UNDEFINED = {0, {0, 1}}; +static const Length LENGTH_MAX = {UINT32_MAX, {UINT32_MAX, UINT32_MAX}}; + +static inline bool length_is_undefined(Length length) { + return length.bytes == 0 && length.extent.column != 0; +} + +static inline Length length_min(Length len1, Length len2) { + return (len1.bytes < len2.bytes) ? len1 : len2; +} + +static inline Length length_add(Length len1, Length len2) { + Length result; + result.bytes = len1.bytes + len2.bytes; + result.extent = point_add(len1.extent, len2.extent); + return result; +} + +static inline Length length_sub(Length len1, Length len2) { + Length result; + result.bytes = (len1.bytes >= len2.bytes) ? len1.bytes - len2.bytes : 0; + result.extent = point_sub(len1.extent, len2.extent); + return result; +} + +static inline Length length_zero(void) { + Length result = {0, {0, 0}}; + return result; +} + +static inline Length length_saturating_sub(Length len1, Length len2) { + if (len1.bytes > len2.bytes) { + return length_sub(len1, len2); + } else { + return length_zero(); + } +} + +#endif diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/lexer.c b/Externals/crystaledit/editlib/tree-sitter/lib/src/lexer.c new file mode 100644 index 00000000000..94124fd1d59 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/lexer.c @@ -0,0 +1,483 @@ +#include "./length.h" +#include "./lexer.h" +#include "./unicode.h" + +#include "tree_sitter/api.h" + +#include +#include + +#define LOG(message, character) \ + if (self->logger.log) { \ + snprintf( \ + self->debug_buffer, \ + TREE_SITTER_SERIALIZATION_BUFFER_SIZE, \ + 32 <= character && character < 127 ? \ + message " character:'%c'" : \ + message " character:%d", \ + character \ + ); \ + self->logger.log( \ + self->logger.payload, \ + TSLogTypeLex, \ + self->debug_buffer \ + ); \ + } + +static const int32_t BYTE_ORDER_MARK = 0xFEFF; + +static const TSRange DEFAULT_RANGE = { + .start_point = { + .row = 0, + .column = 0, + }, + .end_point = { + .row = UINT32_MAX, + .column = UINT32_MAX, + }, + .start_byte = 0, + .end_byte = UINT32_MAX +}; + +/** + * Sets the column data to the given value and marks it valid. + * @param self The lexer state. + * @param val The new value of the column data. + */ +static void ts_lexer__set_column_data(Lexer *self, uint32_t val) { + self->column_data.valid = true; + self->column_data.value = val; +} + +/** + * Increments the value of the column data; no-op if invalid. + * @param self The lexer state. + */ +static void ts_lexer__increment_column_data(Lexer *self) { + if (self->column_data.valid) { + self->column_data.value++; + } +} + +/** + * Marks the column data as invalid. + * @param self The lexer state. + */ +static void ts_lexer__invalidate_column_data(Lexer *self) { + self->column_data.valid = false; + self->column_data.value = 0; +} + +// Check if the lexer has reached EOF. This state is stored +// by setting the lexer's `current_included_range_index` such that +// it has consumed all of its available ranges. +static bool ts_lexer__eof(const TSLexer *_self) { + Lexer *self = (Lexer *)_self; + return self->current_included_range_index == self->included_range_count; +} + +// Clear the currently stored chunk of source code, because the lexer's +// position has changed. +static void ts_lexer__clear_chunk(Lexer *self) { + self->chunk = NULL; + self->chunk_size = 0; + self->chunk_start = 0; +} + +// Call the lexer's input callback to obtain a new chunk of source code +// for the current position. +static void ts_lexer__get_chunk(Lexer *self) { + self->chunk_start = self->current_position.bytes; + self->chunk = self->input.read( + self->input.payload, + self->current_position.bytes, + self->current_position.extent, + &self->chunk_size + ); + if (!self->chunk_size) { + self->current_included_range_index = self->included_range_count; + self->chunk = NULL; + } +} + +// Decode the next unicode character in the current chunk of source code. +// This assumes that the lexer has already retrieved a chunk of source +// code that spans the current position. +static void ts_lexer__get_lookahead(Lexer *self) { + uint32_t position_in_chunk = self->current_position.bytes - self->chunk_start; + uint32_t size = self->chunk_size - position_in_chunk; + + if (size == 0) { + self->lookahead_size = 1; + self->data.lookahead = '\0'; + return; + } + + const uint8_t *chunk = (const uint8_t *)self->chunk + position_in_chunk; + DecodeFunction decode = + self->input.encoding == TSInputEncodingUTF8 ? ts_decode_utf8 : + self->input.encoding == TSInputEncodingUTF16LE ? ts_decode_utf16_le : + self->input.encoding == TSInputEncodingUTF16BE ? ts_decode_utf16_be : self->input.decode; + + self->lookahead_size = decode(chunk, size, &self->data.lookahead); + + // If this chunk ended in the middle of a multi-byte character, + // try again with a fresh chunk. + if (self->data.lookahead == TS_DECODE_ERROR && size < 4) { + ts_lexer__get_chunk(self); + chunk = (const uint8_t *)self->chunk; + size = self->chunk_size; + self->lookahead_size = decode(chunk, size, &self->data.lookahead); + } + + if (self->data.lookahead == TS_DECODE_ERROR) { + self->lookahead_size = 1; + } +} + +static void ts_lexer_goto(Lexer *self, Length position) { + if (position.bytes != self->current_position.bytes) { + ts_lexer__invalidate_column_data(self); + } + + self->current_position = position; + + // Move to the first valid position at or after the given position. + bool found_included_range = false; + for (unsigned i = 0; i < self->included_range_count; i++) { + TSRange *included_range = &self->included_ranges[i]; + if ( + included_range->end_byte > self->current_position.bytes && + included_range->end_byte > included_range->start_byte + ) { + if (included_range->start_byte >= self->current_position.bytes) { + self->current_position = (Length) { + .bytes = included_range->start_byte, + .extent = included_range->start_point, + }; + } + + self->current_included_range_index = i; + found_included_range = true; + break; + } + } + + if (found_included_range) { + // If the current position is outside of the current chunk of text, + // then clear out the current chunk of text. + if (self->chunk && ( + self->current_position.bytes < self->chunk_start || + self->current_position.bytes >= self->chunk_start + self->chunk_size + )) { + ts_lexer__clear_chunk(self); + } + + self->lookahead_size = 0; + self->data.lookahead = '\0'; + } + + // If the given position is beyond any of included ranges, move to the EOF + // state - past the end of the included ranges. + else { + self->current_included_range_index = self->included_range_count; + TSRange *last_included_range = &self->included_ranges[self->included_range_count - 1]; + self->current_position = (Length) { + .bytes = last_included_range->end_byte, + .extent = last_included_range->end_point, + }; + ts_lexer__clear_chunk(self); + self->lookahead_size = 1; + self->data.lookahead = '\0'; + } +} + +/** + * Actually advances the lexer. Does not log anything. + * @param self The lexer state. + * @param skip Whether to mark the consumed codepoint as whitespace. + */ +static void ts_lexer__do_advance(Lexer *self, bool skip) { + if (self->lookahead_size) { + if (self->data.lookahead == '\n') { + self->current_position.extent.row++; + self->current_position.extent.column = 0; + ts_lexer__set_column_data(self, 0); + } else { + bool is_bom = self->current_position.bytes == 0 && + self->data.lookahead == BYTE_ORDER_MARK; + if (!is_bom) ts_lexer__increment_column_data(self); + self->current_position.extent.column += self->lookahead_size; + } + self->current_position.bytes += self->lookahead_size; + } + + const TSRange *current_range = &self->included_ranges[self->current_included_range_index]; + while ( + self->current_position.bytes >= current_range->end_byte || + current_range->end_byte == current_range->start_byte + ) { + if (self->current_included_range_index < self->included_range_count) { + self->current_included_range_index++; + } + if (self->current_included_range_index < self->included_range_count) { + current_range++; + self->current_position = (Length) { + current_range->start_byte, + current_range->start_point, + }; + } else { + current_range = NULL; + break; + } + } + + if (skip) self->token_start_position = self->current_position; + + if (current_range) { + if ( + self->current_position.bytes < self->chunk_start || + self->current_position.bytes >= self->chunk_start + self->chunk_size + ) { + ts_lexer__get_chunk(self); + } + ts_lexer__get_lookahead(self); + } else { + ts_lexer__clear_chunk(self); + self->data.lookahead = '\0'; + self->lookahead_size = 1; + } +} + +// Advance to the next character in the source code, retrieving a new +// chunk of source code if needed. +static void ts_lexer__advance(TSLexer *_self, bool skip) { + Lexer *self = (Lexer *)_self; + if (!self->chunk) return; + + if (skip) { + LOG("skip", self->data.lookahead) + } else { + LOG("consume", self->data.lookahead) + } + + ts_lexer__do_advance(self, skip); +} + +// Mark that a token match has completed. This can be called multiple +// times if a longer match is found later. +static void ts_lexer__mark_end(TSLexer *_self) { + Lexer *self = (Lexer *)_self; + if (!ts_lexer__eof(&self->data)) { + // If the lexer is right at the beginning of included range, + // then the token should be considered to end at the *end* of the + // previous included range, rather than here. + TSRange *current_included_range = &self->included_ranges[ + self->current_included_range_index + ]; + if ( + self->current_included_range_index > 0 && + self->current_position.bytes == current_included_range->start_byte + ) { + TSRange *previous_included_range = current_included_range - 1; + self->token_end_position = (Length) { + previous_included_range->end_byte, + previous_included_range->end_point, + }; + return; + } + } + self->token_end_position = self->current_position; +} + +static uint32_t ts_lexer__get_column(TSLexer *_self) { + Lexer *self = (Lexer *)_self; + + self->did_get_column = true; + + if (!self->column_data.valid) { + // Record current position + uint32_t goal_byte = self->current_position.bytes; + + // Back up to the beginning of the line + Length start_of_col = { + self->current_position.bytes - self->current_position.extent.column, + {self->current_position.extent.row, 0}, + }; + ts_lexer_goto(self, start_of_col); + ts_lexer__set_column_data(self, 0); + ts_lexer__get_chunk(self); + + if (!ts_lexer__eof(_self)) { + ts_lexer__get_lookahead(self); + + // Advance to the recorded position + while (self->current_position.bytes < goal_byte && !ts_lexer__eof(_self) && self->chunk) { + ts_lexer__do_advance(self, false); + if (ts_lexer__eof(_self)) break; + } + } + } + + return self->column_data.value; +} + +// Is the lexer at a boundary between two disjoint included ranges of +// source code? This is exposed as an API because some languages' external +// scanners need to perform custom actions at these boundaries. +static bool ts_lexer__is_at_included_range_start(const TSLexer *_self) { + const Lexer *self = (const Lexer *)_self; + if (self->current_included_range_index < self->included_range_count) { + TSRange *current_range = &self->included_ranges[self->current_included_range_index]; + return self->current_position.bytes == current_range->start_byte; + } else { + return false; + } +} + +static void ts_lexer__log(const TSLexer *_self, const char *fmt, ...) { + Lexer *self = (Lexer *)_self; + va_list args; + va_start(args, fmt); + if (self->logger.log) { + vsnprintf(self->debug_buffer, TREE_SITTER_SERIALIZATION_BUFFER_SIZE, fmt, args); + self->logger.log(self->logger.payload, TSLogTypeLex, self->debug_buffer); + } + va_end(args); +} + +void ts_lexer_init(Lexer *self) { + *self = (Lexer) { + .data = { + // The lexer's methods are stored as struct fields so that generated + // parsers can call them without needing to be linked against this + // library. + .advance = ts_lexer__advance, + .mark_end = ts_lexer__mark_end, + .get_column = ts_lexer__get_column, + .is_at_included_range_start = ts_lexer__is_at_included_range_start, + .eof = ts_lexer__eof, + .log = ts_lexer__log, + .lookahead = 0, + .result_symbol = 0, + }, + .chunk = NULL, + .chunk_size = 0, + .chunk_start = 0, + .current_position = {0, {0, 0}}, + .logger = { + .payload = NULL, + .log = NULL + }, + .included_ranges = NULL, + .included_range_count = 0, + .current_included_range_index = 0, + .did_get_column = false, + .column_data = { + .valid = false, + .value = 0 + } + }; + ts_lexer_set_included_ranges(self, NULL, 0); +} + +void ts_lexer_delete(Lexer *self) { + ts_free(self->included_ranges); +} + +void ts_lexer_set_input(Lexer *self, TSInput input) { + self->input = input; + ts_lexer__clear_chunk(self); + ts_lexer_goto(self, self->current_position); +} + +// Move the lexer to the given position. This doesn't do any work +// if the parser is already at the given position. +void ts_lexer_reset(Lexer *self, Length position) { + if (position.bytes != self->current_position.bytes) { + ts_lexer_goto(self, position); + } +} + +void ts_lexer_start(Lexer *self) { + self->token_start_position = self->current_position; + self->token_end_position = LENGTH_UNDEFINED; + self->data.result_symbol = 0; + self->did_get_column = false; + if (!ts_lexer__eof(&self->data)) { + if (!self->chunk_size) ts_lexer__get_chunk(self); + if (!self->lookahead_size) ts_lexer__get_lookahead(self); + if (self->current_position.bytes == 0) { + if (self->data.lookahead == BYTE_ORDER_MARK) { + ts_lexer__advance(&self->data, true); + } + ts_lexer__set_column_data(self, 0); + } + } +} + +void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte) { + if (length_is_undefined(self->token_end_position)) { + ts_lexer__mark_end(&self->data); + } + + // If the token ended at an included range boundary, then its end position + // will have been reset to the end of the preceding range. Reset the start + // position to match. + if (self->token_end_position.bytes < self->token_start_position.bytes) { + self->token_start_position = self->token_end_position; + } + + uint32_t current_lookahead_end_byte = self->current_position.bytes + 1; + + // In order to determine that a byte sequence is invalid UTF8 or UTF16, + // the character decoding algorithm may have looked at the following byte. + // Therefore, the next byte *after* the current (invalid) character + // affects the interpretation of the current character. + if (self->data.lookahead == TS_DECODE_ERROR) { + current_lookahead_end_byte += 4; // the maximum number of bytes read to identify an invalid code point + } + + if (current_lookahead_end_byte > *lookahead_end_byte) { + *lookahead_end_byte = current_lookahead_end_byte; + } +} + +void ts_lexer_mark_end(Lexer *self) { + ts_lexer__mark_end(&self->data); +} + +bool ts_lexer_set_included_ranges( + Lexer *self, + const TSRange *ranges, + uint32_t count +) { + if (count == 0 || !ranges) { + ranges = &DEFAULT_RANGE; + count = 1; + } else { + uint32_t previous_byte = 0; + for (unsigned i = 0; i < count; i++) { + const TSRange *range = &ranges[i]; + if ( + range->start_byte < previous_byte || + range->end_byte < range->start_byte + ) return false; + previous_byte = range->end_byte; + } + } + + size_t size = count * sizeof(TSRange); + self->included_ranges = ts_realloc(self->included_ranges, size); + memcpy(self->included_ranges, ranges, size); + self->included_range_count = count; + ts_lexer_goto(self, self->current_position); + return true; +} + +TSRange *ts_lexer_included_ranges(const Lexer *self, uint32_t *count) { + *count = self->included_range_count; + return self->included_ranges; +} + +#undef LOG diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/lexer.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/lexer.h new file mode 100644 index 00000000000..7f451e3f93a --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/lexer.h @@ -0,0 +1,54 @@ +#ifndef TREE_SITTER_LEXER_H_ +#define TREE_SITTER_LEXER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "./length.h" +#include "./subtree.h" +#include "tree_sitter/api.h" +#include "./parser.h" + +typedef struct { + uint32_t value; + bool valid; +} ColumnData; + +typedef struct { + TSLexer data; + Length current_position; + Length token_start_position; + Length token_end_position; + + TSRange *included_ranges; + const char *chunk; + TSInput input; + TSLogger logger; + + uint32_t included_range_count; + uint32_t current_included_range_index; + uint32_t chunk_start; + uint32_t chunk_size; + uint32_t lookahead_size; + bool did_get_column; + ColumnData column_data; + + char debug_buffer[TREE_SITTER_SERIALIZATION_BUFFER_SIZE]; +} Lexer; + +void ts_lexer_init(Lexer *self); +void ts_lexer_delete(Lexer *self); +void ts_lexer_set_input(Lexer *self, TSInput input); +void ts_lexer_reset(Lexer *self, Length position); +void ts_lexer_start(Lexer *self); +void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte); +void ts_lexer_mark_end(Lexer *self); +bool ts_lexer_set_included_ranges(Lexer *self, const TSRange *ranges, uint32_t count); +TSRange *ts_lexer_included_ranges(const Lexer *self, uint32_t *count); + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_LEXER_H_ diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/lib.c b/Externals/crystaledit/editlib/tree-sitter/lib/src/lib.c new file mode 100644 index 00000000000..9bfb69f00b7 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/lib.c @@ -0,0 +1,12 @@ +#include "./alloc.c" +#include "./get_changed_ranges.c" +#include "./language.c" +#include "./lexer.c" +#include "./node.c" +#include "./parser.c" +#include "./query.c" +#include "./stack.c" +#include "./subtree.c" +#include "./tree_cursor.c" +#include "./tree.c" +#include "./wasm_store.c" diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/node.c b/Externals/crystaledit/editlib/tree-sitter/lib/src/node.c new file mode 100644 index 00000000000..d83fa90b8b1 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/node.c @@ -0,0 +1,875 @@ +#include +#include "./point.h" +#include "./subtree.h" +#include "./tree.h" +#include "./language.h" + +typedef struct { + Subtree parent; + const TSTree *tree; + Length position; + uint32_t child_index; + uint32_t structural_child_index; + const TSSymbol *alias_sequence; +} NodeChildIterator; + +static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous); + +// TSNode - constructors + +TSNode ts_node_new( + const TSTree *tree, + const Subtree *subtree, + Length position, + TSSymbol alias +) { + return (TSNode) { + {position.bytes, position.extent.row, position.extent.column, alias}, + subtree, + tree, + }; +} + +static inline TSNode ts_node__null(void) { + return ts_node_new(NULL, NULL, length_zero(), 0); +} + +// TSNode - accessors + +uint32_t ts_node_start_byte(TSNode self) { + return self.context[0]; +} + +TSPoint ts_node_start_point(TSNode self) { + return (TSPoint) {self.context[1], self.context[2]}; +} + +static inline uint32_t ts_node__alias(const TSNode *self) { + return self->context[3]; +} + +static inline Subtree ts_node__subtree(TSNode self) { + return *(const Subtree *)self.id; +} + +// NodeChildIterator + +static inline NodeChildIterator ts_node_iterate_children(const TSNode *node) { + Subtree subtree = ts_node__subtree(*node); + if (ts_subtree_child_count(subtree) == 0) { + return (NodeChildIterator) {NULL_SUBTREE, node->tree, length_zero(), 0, 0, NULL}; + } + const TSSymbol *alias_sequence = ts_language_alias_sequence( + node->tree->language, + subtree.ptr->production_id + ); + return (NodeChildIterator) { + .tree = node->tree, + .parent = subtree, + .position = {ts_node_start_byte(*node), ts_node_start_point(*node)}, + .child_index = 0, + .structural_child_index = 0, + .alias_sequence = alias_sequence, + }; +} + +static inline bool ts_node_child_iterator_done(NodeChildIterator *self) { + return self->child_index == self->parent.ptr->child_count; +} + +static inline bool ts_node_child_iterator_next( + NodeChildIterator *self, + TSNode *result +) { + if (!self->parent.ptr || ts_node_child_iterator_done(self)) return false; + const Subtree *child = &ts_subtree_children(self->parent)[self->child_index]; + TSSymbol alias_symbol = 0; + if (!ts_subtree_extra(*child)) { + if (self->alias_sequence) { + alias_symbol = self->alias_sequence[self->structural_child_index]; + } + self->structural_child_index++; + } + if (self->child_index > 0) { + self->position = length_add(self->position, ts_subtree_padding(*child)); + } + *result = ts_node_new( + self->tree, + child, + self->position, + alias_symbol + ); + self->position = length_add(self->position, ts_subtree_size(*child)); + self->child_index++; + return true; +} + +// TSNode - private + +static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) { + Subtree tree = ts_node__subtree(self); + if (include_anonymous) { + return ts_subtree_visible(tree) || ts_node__alias(&self); + } else { + TSSymbol alias = ts_node__alias(&self); + if (alias) { + return ts_language_symbol_metadata(self.tree->language, alias).named; + } else { + return ts_subtree_visible(tree) && ts_subtree_named(tree); + } + } +} + +static inline uint32_t ts_node__relevant_child_count( + TSNode self, + bool include_anonymous +) { + Subtree tree = ts_node__subtree(self); + if (ts_subtree_child_count(tree) > 0) { + if (include_anonymous) { + return tree.ptr->visible_child_count; + } else { + return tree.ptr->named_child_count; + } + } else { + return 0; + } +} + +static inline TSNode ts_node__child( + TSNode self, + uint32_t child_index, + bool include_anonymous +) { + TSNode result = self; + bool did_descend = true; + + while (did_descend) { + did_descend = false; + + TSNode child; + uint32_t index = 0; + NodeChildIterator iterator = ts_node_iterate_children(&result); + while (ts_node_child_iterator_next(&iterator, &child)) { + if (ts_node__is_relevant(child, include_anonymous)) { + if (index == child_index) { + return child; + } + index++; + } else { + uint32_t grandchild_index = child_index - index; + uint32_t grandchild_count = ts_node__relevant_child_count(child, include_anonymous); + if (grandchild_index < grandchild_count) { + did_descend = true; + result = child; + child_index = grandchild_index; + break; + } + index += grandchild_count; + } + } + } + + return ts_node__null(); +} + +static bool ts_subtree_has_trailing_empty_descendant( + Subtree self, + Subtree other +) { + for (unsigned i = ts_subtree_child_count(self) - 1; i + 1 > 0; i--) { + Subtree child = ts_subtree_children(self)[i]; + if (ts_subtree_total_bytes(child) > 0) break; + if (child.ptr == other.ptr || ts_subtree_has_trailing_empty_descendant(child, other)) { + return true; + } + } + return false; +} + +static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) { + Subtree self_subtree = ts_node__subtree(self); + bool self_is_empty = ts_subtree_total_bytes(self_subtree) == 0; + uint32_t target_end_byte = ts_node_end_byte(self); + + TSNode node = ts_node_parent(self); + TSNode earlier_node = ts_node__null(); + bool earlier_node_is_relevant = false; + + while (!ts_node_is_null(node)) { + TSNode earlier_child = ts_node__null(); + bool earlier_child_is_relevant = false; + bool found_child_containing_target = false; + + TSNode child; + NodeChildIterator iterator = ts_node_iterate_children(&node); + while (ts_node_child_iterator_next(&iterator, &child)) { + if (child.id == self.id) break; + if (iterator.position.bytes > target_end_byte) { + found_child_containing_target = true; + break; + } + + if (iterator.position.bytes == target_end_byte && + (!self_is_empty || + ts_subtree_has_trailing_empty_descendant(ts_node__subtree(child), self_subtree))) { + found_child_containing_target = true; + break; + } + + if (ts_node__is_relevant(child, include_anonymous)) { + earlier_child = child; + earlier_child_is_relevant = true; + } else if (ts_node__relevant_child_count(child, include_anonymous) > 0) { + earlier_child = child; + earlier_child_is_relevant = false; + } + } + + if (found_child_containing_target) { + if (!ts_node_is_null(earlier_child)) { + earlier_node = earlier_child; + earlier_node_is_relevant = earlier_child_is_relevant; + } + node = child; + } else if (earlier_child_is_relevant) { + return earlier_child; + } else if (!ts_node_is_null(earlier_child)) { + node = earlier_child; + } else if (earlier_node_is_relevant) { + return earlier_node; + } else { + node = earlier_node; + earlier_node = ts_node__null(); + earlier_node_is_relevant = false; + } + } + + return ts_node__null(); +} + +static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) { + uint32_t target_end_byte = ts_node_end_byte(self); + + TSNode node = ts_node_parent(self); + TSNode later_node = ts_node__null(); + bool later_node_is_relevant = false; + + while (!ts_node_is_null(node)) { + TSNode later_child = ts_node__null(); + bool later_child_is_relevant = false; + TSNode child_containing_target = ts_node__null(); + + TSNode child; + NodeChildIterator iterator = ts_node_iterate_children(&node); + while (ts_node_child_iterator_next(&iterator, &child)) { + if (iterator.position.bytes <= target_end_byte) continue; + uint32_t start_byte = ts_node_start_byte(self); + uint32_t child_start_byte = ts_node_start_byte(child); + + bool is_empty = start_byte == target_end_byte; + bool contains_target = is_empty ? + child_start_byte < start_byte : + child_start_byte <= start_byte; + + if (contains_target) { + if (ts_node__subtree(child).ptr != ts_node__subtree(self).ptr) { + child_containing_target = child; + } + } else if (ts_node__is_relevant(child, include_anonymous)) { + later_child = child; + later_child_is_relevant = true; + break; + } else if (ts_node__relevant_child_count(child, include_anonymous) > 0) { + later_child = child; + later_child_is_relevant = false; + break; + } + } + + if (!ts_node_is_null(child_containing_target)) { + if (!ts_node_is_null(later_child)) { + later_node = later_child; + later_node_is_relevant = later_child_is_relevant; + } + node = child_containing_target; + } else if (later_child_is_relevant) { + return later_child; + } else if (!ts_node_is_null(later_child)) { + node = later_child; + } else if (later_node_is_relevant) { + return later_node; + } else { + node = later_node; + } + } + + return ts_node__null(); +} + +static inline TSNode ts_node__first_child_for_byte( + TSNode self, + uint32_t goal, + bool include_anonymous +) { + TSNode node = self; + bool did_descend = true; + + NodeChildIterator last_iterator; + bool has_last_iterator = false; + + while (did_descend) { + did_descend = false; + + TSNode child; + NodeChildIterator iterator = ts_node_iterate_children(&node); + loop: + while (ts_node_child_iterator_next(&iterator, &child)) { + if (ts_node_end_byte(child) > goal) { + if (ts_node__is_relevant(child, include_anonymous)) { + return child; + } else if (ts_node_child_count(child) > 0) { + if (iterator.child_index < ts_subtree_child_count(ts_node__subtree(child))) { + last_iterator = iterator; + has_last_iterator = true; + } + did_descend = true; + node = child; + break; + } + } + } + + if (!did_descend && has_last_iterator) { + iterator = last_iterator; + has_last_iterator = false; + goto loop; + } + } + + return ts_node__null(); +} + +static inline TSNode ts_node__descendant_for_byte_range( + TSNode self, + uint32_t range_start, + uint32_t range_end, + bool include_anonymous +) { + if (range_start > range_end) { + return ts_node__null(); + } + TSNode node = self; + TSNode last_visible_node = self; + + bool did_descend = true; + while (did_descend) { + did_descend = false; + + TSNode child; + NodeChildIterator iterator = ts_node_iterate_children(&node); + while (ts_node_child_iterator_next(&iterator, &child)) { + uint32_t node_end = iterator.position.bytes; + + // The end of this node must extend far enough forward to touch + // the end of the range + if (node_end < range_end) continue; + + // ...and exceed the start of the range, unless the node itself is + // empty, in which case it must at least be equal to the start of the range. + bool is_empty = ts_node_start_byte(child) == node_end; + if (is_empty ? node_end < range_start : node_end <= range_start) continue; + + // The start of this node must extend far enough backward to + // touch the start of the range. + if (range_start < ts_node_start_byte(child)) break; + + node = child; + if (ts_node__is_relevant(node, include_anonymous)) { + last_visible_node = node; + } + did_descend = true; + break; + } + } + + return last_visible_node; +} + +static inline TSNode ts_node__descendant_for_point_range( + TSNode self, + TSPoint range_start, + TSPoint range_end, + bool include_anonymous +) { + if (point_gt(range_start, range_end)) { + return ts_node__null(); + } + TSNode node = self; + TSNode last_visible_node = self; + + bool did_descend = true; + while (did_descend) { + did_descend = false; + + TSNode child; + NodeChildIterator iterator = ts_node_iterate_children(&node); + while (ts_node_child_iterator_next(&iterator, &child)) { + TSPoint node_end = iterator.position.extent; + + // The end of this node must extend far enough forward to touch + // the end of the range + if (point_lt(node_end, range_end)) continue; + + // ...and exceed the start of the range, unless the node itself is + // empty, in which case it must at least be equal to the start of the range. + bool is_empty = point_eq(ts_node_start_point(child), node_end); + if (is_empty ? point_lt(node_end, range_start) : point_lte(node_end, range_start)) { + continue; + } + + // The start of this node must extend far enough backward to + // touch the start of the range. + if (point_lt(range_start, ts_node_start_point(child))) break; + + node = child; + if (ts_node__is_relevant(node, include_anonymous)) { + last_visible_node = node; + } + did_descend = true; + break; + } + } + + return last_visible_node; +} + +// TSNode - public + +uint32_t ts_node_end_byte(TSNode self) { + return ts_node_start_byte(self) + ts_subtree_size(ts_node__subtree(self)).bytes; +} + +TSPoint ts_node_end_point(TSNode self) { + return point_add(ts_node_start_point(self), ts_subtree_size(ts_node__subtree(self)).extent); +} + +TSSymbol ts_node_symbol(TSNode self) { + TSSymbol symbol = ts_node__alias(&self); + if (!symbol) symbol = ts_subtree_symbol(ts_node__subtree(self)); + return ts_language_public_symbol(self.tree->language, symbol); +} + +const char *ts_node_type(TSNode self) { + TSSymbol symbol = ts_node__alias(&self); + if (!symbol) symbol = ts_subtree_symbol(ts_node__subtree(self)); + return ts_language_symbol_name(self.tree->language, symbol); +} + +const TSLanguage *ts_node_language(TSNode self) { + return self.tree->language; +} + +TSSymbol ts_node_grammar_symbol(TSNode self) { + return ts_subtree_symbol(ts_node__subtree(self)); +} + +const char *ts_node_grammar_type(TSNode self) { + TSSymbol symbol = ts_subtree_symbol(ts_node__subtree(self)); + return ts_language_symbol_name(self.tree->language, symbol); +} + +char *ts_node_string(TSNode self) { + TSSymbol alias_symbol = ts_node__alias(&self); + return ts_subtree_string( + ts_node__subtree(self), + alias_symbol, + ts_language_symbol_metadata(self.tree->language, alias_symbol).visible, + self.tree->language, + false + ); +} + +bool ts_node_eq(TSNode self, TSNode other) { + return self.tree == other.tree && self.id == other.id; +} + +bool ts_node_is_null(TSNode self) { + return self.id == 0; +} + +bool ts_node_is_extra(TSNode self) { + return ts_subtree_extra(ts_node__subtree(self)); +} + +bool ts_node_is_named(TSNode self) { + TSSymbol alias = ts_node__alias(&self); + return alias + ? ts_language_symbol_metadata(self.tree->language, alias).named + : ts_subtree_named(ts_node__subtree(self)); +} + +bool ts_node_is_missing(TSNode self) { + return ts_subtree_missing(ts_node__subtree(self)); +} + +bool ts_node_has_changes(TSNode self) { + return ts_subtree_has_changes(ts_node__subtree(self)); +} + +bool ts_node_has_error(TSNode self) { + return ts_subtree_error_cost(ts_node__subtree(self)) > 0; +} + +bool ts_node_is_error(TSNode self) { + TSSymbol symbol = ts_node_symbol(self); + return symbol == ts_builtin_sym_error; +} + +uint32_t ts_node_descendant_count(TSNode self) { + return ts_subtree_visible_descendant_count(ts_node__subtree(self)) + 1; +} + +TSStateId ts_node_parse_state(TSNode self) { + return ts_subtree_parse_state(ts_node__subtree(self)); +} + +TSStateId ts_node_next_parse_state(TSNode self) { + const TSLanguage *language = self.tree->language; + uint16_t state = ts_node_parse_state(self); + if (state == TS_TREE_STATE_NONE) { + return TS_TREE_STATE_NONE; + } + uint16_t symbol = ts_node_grammar_symbol(self); + return ts_language_next_state(language, state, symbol); +} + +TSNode ts_node_parent(TSNode self) { + TSNode node = ts_tree_root_node(self.tree); + if (node.id == self.id) return ts_node__null(); + + while (true) { + TSNode next_node = ts_node_child_with_descendant(node, self); + if (next_node.id == self.id || ts_node_is_null(next_node)) break; + node = next_node; + } + + return node; +} + +TSNode ts_node_child_with_descendant(TSNode self, TSNode descendant) { + uint32_t start_byte = ts_node_start_byte(descendant); + uint32_t end_byte = ts_node_end_byte(descendant); + bool is_empty = start_byte == end_byte; + + do { + NodeChildIterator iter = ts_node_iterate_children(&self); + do { + if ( + !ts_node_child_iterator_next(&iter, &self) + || ts_node_start_byte(self) > start_byte + ) { + return ts_node__null(); + } + if (self.id == descendant.id) { + return self; + } + + // If the descendant is empty, and the end byte is within `self`, + // we check whether `self` contains it or not. + if (is_empty && iter.position.bytes >= end_byte && ts_node_child_count(self) > 0) { + TSNode child = ts_node_child_with_descendant(self, descendant); + // If the child is not null, return self if it's relevant, else return the child + if (!ts_node_is_null(child)) { + return ts_node__is_relevant(self, true) ? self : child; + } + } + } while ((is_empty ? iter.position.bytes <= end_byte : iter.position.bytes < end_byte) || ts_node_child_count(self) == 0); + } while (!ts_node__is_relevant(self, true)); + + return self; +} + +TSNode ts_node_child(TSNode self, uint32_t child_index) { + return ts_node__child(self, child_index, true); +} + +TSNode ts_node_named_child(TSNode self, uint32_t child_index) { + return ts_node__child(self, child_index, false); +} + +TSNode ts_node_child_by_field_id(TSNode self, TSFieldId field_id) { +recur: + if (!field_id || ts_node_child_count(self) == 0) return ts_node__null(); + + const TSFieldMapEntry *field_map, *field_map_end; + ts_language_field_map( + self.tree->language, + ts_node__subtree(self).ptr->production_id, + &field_map, + &field_map_end + ); + if (field_map == field_map_end) return ts_node__null(); + + // The field mappings are sorted by their field id. Scan all + // the mappings to find the ones for the given field id. + while (field_map->field_id < field_id) { + field_map++; + if (field_map == field_map_end) return ts_node__null(); + } + while (field_map_end[-1].field_id > field_id) { + field_map_end--; + if (field_map == field_map_end) return ts_node__null(); + } + + TSNode child; + NodeChildIterator iterator = ts_node_iterate_children(&self); + while (ts_node_child_iterator_next(&iterator, &child)) { + if (!ts_subtree_extra(ts_node__subtree(child))) { + uint32_t index = iterator.structural_child_index - 1; + if (index < field_map->child_index) continue; + + // Hidden nodes' fields are "inherited" by their visible parent. + if (field_map->inherited) { + + // If this is the *last* possible child node for this field, + // then perform a tail call to avoid recursion. + if (field_map + 1 == field_map_end) { + self = child; + goto recur; + } + + // Otherwise, descend into this child, but if it doesn't contain + // the field, continue searching subsequent children. + else { + TSNode result = ts_node_child_by_field_id(child, field_id); + if (result.id) return result; + field_map++; + if (field_map == field_map_end) return ts_node__null(); + } + } + + else if (ts_node__is_relevant(child, true)) { + return child; + } + + // If the field refers to a hidden node with visible children, + // return the first visible child. + else if (ts_node_child_count(child) > 0 ) { + return ts_node_child(child, 0); + } + + // Otherwise, continue searching subsequent children. + else { + field_map++; + if (field_map == field_map_end) return ts_node__null(); + } + } + } + + return ts_node__null(); +} + +static inline const char *ts_node__field_name_from_language(TSNode self, uint32_t structural_child_index) { + const TSFieldMapEntry *field_map, *field_map_end; + ts_language_field_map( + self.tree->language, + ts_node__subtree(self).ptr->production_id, + &field_map, + &field_map_end + ); + for (; field_map != field_map_end; field_map++) { + if (!field_map->inherited && field_map->child_index == structural_child_index) { + return self.tree->language->field_names[field_map->field_id]; + } + } + return NULL; +} + +const char *ts_node_field_name_for_child(TSNode self, uint32_t child_index) { + TSNode result = self; + bool did_descend = true; + const char *inherited_field_name = NULL; + + while (did_descend) { + did_descend = false; + + TSNode child; + uint32_t index = 0; + NodeChildIterator iterator = ts_node_iterate_children(&result); + while (ts_node_child_iterator_next(&iterator, &child)) { + if (ts_node__is_relevant(child, true)) { + if (index == child_index) { + if (ts_node_is_extra(child)) { + return NULL; + } + const char *field_name = ts_node__field_name_from_language(result, iterator.structural_child_index - 1); + if (field_name) return field_name; + return inherited_field_name; + } + index++; + } else { + uint32_t grandchild_index = child_index - index; + uint32_t grandchild_count = ts_node__relevant_child_count(child, true); + if (grandchild_index < grandchild_count) { + const char *field_name = ts_node__field_name_from_language(result, iterator.structural_child_index - 1); + if (field_name) inherited_field_name = field_name; + + did_descend = true; + result = child; + child_index = grandchild_index; + break; + } + index += grandchild_count; + } + } + } + + return NULL; +} + +const char *ts_node_field_name_for_named_child(TSNode self, uint32_t named_child_index) { + TSNode result = self; + bool did_descend = true; + const char *inherited_field_name = NULL; + + while (did_descend) { + did_descend = false; + + TSNode child; + uint32_t index = 0; + NodeChildIterator iterator = ts_node_iterate_children(&result); + while (ts_node_child_iterator_next(&iterator, &child)) { + if (ts_node__is_relevant(child, false)) { + if (index == named_child_index) { + if (ts_node_is_extra(child)) { + return NULL; + } + const char *field_name = ts_node__field_name_from_language(result, iterator.structural_child_index - 1); + if (field_name) return field_name; + return inherited_field_name; + } + index++; + } else { + uint32_t named_grandchild_index = named_child_index - index; + uint32_t grandchild_count = ts_node__relevant_child_count(child, false); + if (named_grandchild_index < grandchild_count) { + const char *field_name = ts_node__field_name_from_language(result, iterator.structural_child_index - 1); + if (field_name) inherited_field_name = field_name; + + did_descend = true; + result = child; + named_child_index = named_grandchild_index; + break; + } + index += grandchild_count; + } + } + } + + return NULL; +} + +TSNode ts_node_child_by_field_name( + TSNode self, + const char *name, + uint32_t name_length +) { + TSFieldId field_id = ts_language_field_id_for_name( + self.tree->language, + name, + name_length + ); + return ts_node_child_by_field_id(self, field_id); +} + +uint32_t ts_node_child_count(TSNode self) { + Subtree tree = ts_node__subtree(self); + if (ts_subtree_child_count(tree) > 0) { + return tree.ptr->visible_child_count; + } else { + return 0; + } +} + +uint32_t ts_node_named_child_count(TSNode self) { + Subtree tree = ts_node__subtree(self); + if (ts_subtree_child_count(tree) > 0) { + return tree.ptr->named_child_count; + } else { + return 0; + } +} + +TSNode ts_node_next_sibling(TSNode self) { + return ts_node__next_sibling(self, true); +} + +TSNode ts_node_next_named_sibling(TSNode self) { + return ts_node__next_sibling(self, false); +} + +TSNode ts_node_prev_sibling(TSNode self) { + return ts_node__prev_sibling(self, true); +} + +TSNode ts_node_prev_named_sibling(TSNode self) { + return ts_node__prev_sibling(self, false); +} + +TSNode ts_node_first_child_for_byte(TSNode self, uint32_t byte) { + return ts_node__first_child_for_byte(self, byte, true); +} + +TSNode ts_node_first_named_child_for_byte(TSNode self, uint32_t byte) { + return ts_node__first_child_for_byte(self, byte, false); +} + +TSNode ts_node_descendant_for_byte_range( + TSNode self, + uint32_t start, + uint32_t end +) { + return ts_node__descendant_for_byte_range(self, start, end, true); +} + +TSNode ts_node_named_descendant_for_byte_range( + TSNode self, + uint32_t start, + uint32_t end +) { + return ts_node__descendant_for_byte_range(self, start, end, false); +} + +TSNode ts_node_descendant_for_point_range( + TSNode self, + TSPoint start, + TSPoint end +) { + return ts_node__descendant_for_point_range(self, start, end, true); +} + +TSNode ts_node_named_descendant_for_point_range( + TSNode self, + TSPoint start, + TSPoint end +) { + return ts_node__descendant_for_point_range(self, start, end, false); +} + +void ts_node_edit(TSNode *self, const TSInputEdit *edit) { + uint32_t start_byte = ts_node_start_byte(*self); + TSPoint start_point = ts_node_start_point(*self); + + if (start_byte >= edit->old_end_byte) { + start_byte = edit->new_end_byte + (start_byte - edit->old_end_byte); + start_point = point_add(edit->new_end_point, point_sub(start_point, edit->old_end_point)); + } else if (start_byte > edit->start_byte) { + start_byte = edit->new_end_byte; + start_point = edit->new_end_point; + } + + self->context[0] = start_byte; + self->context[1] = start_point.row; + self->context[2] = start_point.column; +} diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/parser.c b/Externals/crystaledit/editlib/tree-sitter/lib/src/parser.c new file mode 100644 index 00000000000..39e6b89ff37 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/parser.c @@ -0,0 +1,2297 @@ +#include +#include +#include +#include +#include +#include "tree_sitter/api.h" +#include "./alloc.h" +#include "./array.h" +#include "./atomic.h" +#include "./clock.h" +#include "./error_costs.h" +#include "./get_changed_ranges.h" +#include "./language.h" +#include "./length.h" +#include "./lexer.h" +#include "./reduce_action.h" +#include "./reusable_node.h" +#include "./stack.h" +#include "./subtree.h" +#include "./tree.h" +#include "./ts_assert.h" +#include "./wasm_store.h" + +#define LOG(...) \ + if (self->lexer.logger.log || self->dot_graph_file) { \ + snprintf(self->lexer.debug_buffer, TREE_SITTER_SERIALIZATION_BUFFER_SIZE, __VA_ARGS__); \ + ts_parser__log(self); \ + } + +#define LOG_LOOKAHEAD(symbol_name, size) \ + if (self->lexer.logger.log || self->dot_graph_file) { \ + char *buf = self->lexer.debug_buffer; \ + const char *symbol = symbol_name; \ + int off = snprintf( \ + buf, \ + TREE_SITTER_SERIALIZATION_BUFFER_SIZE, \ + "lexed_lookahead sym:" \ + ); \ + for ( \ + int i = 0; \ + symbol[i] != '\0' \ + && off < TREE_SITTER_SERIALIZATION_BUFFER_SIZE; \ + i++ \ + ) { \ + switch (symbol[i]) { \ + case '\t': buf[off++] = '\\'; buf[off++] = 't'; break; \ + case '\n': buf[off++] = '\\'; buf[off++] = 'n'; break; \ + case '\v': buf[off++] = '\\'; buf[off++] = 'v'; break; \ + case '\f': buf[off++] = '\\'; buf[off++] = 'f'; break; \ + case '\r': buf[off++] = '\\'; buf[off++] = 'r'; break; \ + case '\\': buf[off++] = '\\'; buf[off++] = '\\'; break; \ + default: buf[off++] = symbol[i]; break; \ + } \ + } \ + snprintf( \ + buf + off, \ + TREE_SITTER_SERIALIZATION_BUFFER_SIZE - off, \ + ", size:%u", \ + size \ + ); \ + ts_parser__log(self); \ + } + +#define LOG_STACK() \ + if (self->dot_graph_file) { \ + ts_stack_print_dot_graph(self->stack, self->language, self->dot_graph_file); \ + fputs("\n\n", self->dot_graph_file); \ + } + +#define LOG_TREE(tree) \ + if (self->dot_graph_file) { \ + ts_subtree_print_dot_graph(tree, self->language, self->dot_graph_file); \ + fputs("\n", self->dot_graph_file); \ + } + +#define SYM_NAME(symbol) ts_language_symbol_name(self->language, symbol) + +#define TREE_NAME(tree) SYM_NAME(ts_subtree_symbol(tree)) + +static const unsigned MAX_VERSION_COUNT = 6; +static const unsigned MAX_VERSION_COUNT_OVERFLOW = 4; +static const unsigned MAX_SUMMARY_DEPTH = 16; +static const unsigned MAX_COST_DIFFERENCE = 18 * ERROR_COST_PER_SKIPPED_TREE; +static const unsigned OP_COUNT_PER_PARSER_TIMEOUT_CHECK = 100; + +typedef struct { + Subtree token; + Subtree last_external_token; + uint32_t byte_index; +} TokenCache; + +struct TSParser { + Lexer lexer; + Stack *stack; + SubtreePool tree_pool; + const TSLanguage *language; + TSWasmStore *wasm_store; + ReduceActionSet reduce_actions; + Subtree finished_tree; + SubtreeArray trailing_extras; + SubtreeArray trailing_extras2; + SubtreeArray scratch_trees; + TokenCache token_cache; + ReusableNode reusable_node; + void *external_scanner_payload; + FILE *dot_graph_file; + TSClock end_clock; + TSDuration timeout_duration; + unsigned accept_count; + unsigned operation_count; + const volatile size_t *cancellation_flag; + Subtree old_tree; + TSRangeArray included_range_differences; + TSParseOptions parse_options; + TSParseState parse_state; + unsigned included_range_difference_index; + bool has_scanner_error; + bool canceled_balancing; + bool has_error; +}; + +typedef struct { + unsigned cost; + unsigned node_count; + int dynamic_precedence; + bool is_in_error; +} ErrorStatus; + +typedef enum { + ErrorComparisonTakeLeft, + ErrorComparisonPreferLeft, + ErrorComparisonNone, + ErrorComparisonPreferRight, + ErrorComparisonTakeRight, +} ErrorComparison; + +typedef struct { + const char *string; + uint32_t length; +} TSStringInput; + +// StringInput + +static const char *ts_string_input_read( + void *_self, + uint32_t byte, + TSPoint point, + uint32_t *length +) { + (void)point; + TSStringInput *self = (TSStringInput *)_self; + if (byte >= self->length) { + *length = 0; + return ""; + } else { + *length = self->length - byte; + return self->string + byte; + } +} + +// Parser - Private + +static void ts_parser__log(TSParser *self) { + if (self->lexer.logger.log) { + self->lexer.logger.log( + self->lexer.logger.payload, + TSLogTypeParse, + self->lexer.debug_buffer + ); + } + + if (self->dot_graph_file) { + fprintf(self->dot_graph_file, "graph {\nlabel=\""); + for (char *chr = &self->lexer.debug_buffer[0]; *chr != 0; chr++) { + if (*chr == '"' || *chr == '\\') fputc('\\', self->dot_graph_file); + fputc(*chr, self->dot_graph_file); + } + fprintf(self->dot_graph_file, "\"\n}\n\n"); + } +} + +static bool ts_parser__breakdown_top_of_stack( + TSParser *self, + StackVersion version +) { + bool did_break_down = false; + bool pending = false; + + do { + StackSliceArray pop = ts_stack_pop_pending(self->stack, version); + if (!pop.size) break; + + did_break_down = true; + pending = false; + for (uint32_t i = 0; i < pop.size; i++) { + StackSlice slice = *array_get(&pop, i); + TSStateId state = ts_stack_state(self->stack, slice.version); + Subtree parent = *array_front(&slice.subtrees); + + for (uint32_t j = 0, n = ts_subtree_child_count(parent); j < n; j++) { + Subtree child = ts_subtree_children(parent)[j]; + pending = ts_subtree_child_count(child) > 0; + + if (ts_subtree_is_error(child)) { + state = ERROR_STATE; + } else if (!ts_subtree_extra(child)) { + state = ts_language_next_state(self->language, state, ts_subtree_symbol(child)); + } + + ts_subtree_retain(child); + ts_stack_push(self->stack, slice.version, child, pending, state); + } + + for (uint32_t j = 1; j < slice.subtrees.size; j++) { + Subtree tree = *array_get(&slice.subtrees, j); + ts_stack_push(self->stack, slice.version, tree, false, state); + } + + ts_subtree_release(&self->tree_pool, parent); + array_delete(&slice.subtrees); + + LOG("breakdown_top_of_stack tree:%s", TREE_NAME(parent)); + LOG_STACK(); + } + } while (pending); + + return did_break_down; +} + +static void ts_parser__breakdown_lookahead( + TSParser *self, + Subtree *lookahead, + TSStateId state, + ReusableNode *reusable_node +) { + bool did_descend = false; + Subtree tree = reusable_node_tree(reusable_node); + while (ts_subtree_child_count(tree) > 0 && ts_subtree_parse_state(tree) != state) { + LOG("state_mismatch sym:%s", TREE_NAME(tree)); + reusable_node_descend(reusable_node); + tree = reusable_node_tree(reusable_node); + did_descend = true; + } + + if (did_descend) { + ts_subtree_release(&self->tree_pool, *lookahead); + *lookahead = tree; + ts_subtree_retain(*lookahead); + } +} + +static ErrorComparison ts_parser__compare_versions( + TSParser *self, + ErrorStatus a, + ErrorStatus b +) { + (void)self; + if (!a.is_in_error && b.is_in_error) { + if (a.cost < b.cost) { + return ErrorComparisonTakeLeft; + } else { + return ErrorComparisonPreferLeft; + } + } + + if (a.is_in_error && !b.is_in_error) { + if (b.cost < a.cost) { + return ErrorComparisonTakeRight; + } else { + return ErrorComparisonPreferRight; + } + } + + if (a.cost < b.cost) { + if ((b.cost - a.cost) * (1 + a.node_count) > MAX_COST_DIFFERENCE) { + return ErrorComparisonTakeLeft; + } else { + return ErrorComparisonPreferLeft; + } + } + + if (b.cost < a.cost) { + if ((a.cost - b.cost) * (1 + b.node_count) > MAX_COST_DIFFERENCE) { + return ErrorComparisonTakeRight; + } else { + return ErrorComparisonPreferRight; + } + } + + if (a.dynamic_precedence > b.dynamic_precedence) return ErrorComparisonPreferLeft; + if (b.dynamic_precedence > a.dynamic_precedence) return ErrorComparisonPreferRight; + return ErrorComparisonNone; +} + +static ErrorStatus ts_parser__version_status( + TSParser *self, + StackVersion version +) { + unsigned cost = ts_stack_error_cost(self->stack, version); + bool is_paused = ts_stack_is_paused(self->stack, version); + if (is_paused) cost += ERROR_COST_PER_SKIPPED_TREE; + return (ErrorStatus) { + .cost = cost, + .node_count = ts_stack_node_count_since_error(self->stack, version), + .dynamic_precedence = ts_stack_dynamic_precedence(self->stack, version), + .is_in_error = is_paused || ts_stack_state(self->stack, version) == ERROR_STATE + }; +} + +static bool ts_parser__better_version_exists( + TSParser *self, + StackVersion version, + bool is_in_error, + unsigned cost +) { + if (self->finished_tree.ptr && ts_subtree_error_cost(self->finished_tree) <= cost) { + return true; + } + + Length position = ts_stack_position(self->stack, version); + ErrorStatus status = { + .cost = cost, + .is_in_error = is_in_error, + .dynamic_precedence = ts_stack_dynamic_precedence(self->stack, version), + .node_count = ts_stack_node_count_since_error(self->stack, version), + }; + + for (StackVersion i = 0, n = ts_stack_version_count(self->stack); i < n; i++) { + if (i == version || + !ts_stack_is_active(self->stack, i) || + ts_stack_position(self->stack, i).bytes < position.bytes) continue; + ErrorStatus status_i = ts_parser__version_status(self, i); + switch (ts_parser__compare_versions(self, status, status_i)) { + case ErrorComparisonTakeRight: + return true; + case ErrorComparisonPreferRight: + if (ts_stack_can_merge(self->stack, i, version)) return true; + break; + default: + break; + } + } + + return false; +} + +static bool ts_parser__call_main_lex_fn(TSParser *self, TSLexerMode lex_mode) { + if (ts_language_is_wasm(self->language)) { + return ts_wasm_store_call_lex_main(self->wasm_store, lex_mode.lex_state); + } else { + return self->language->lex_fn(&self->lexer.data, lex_mode.lex_state); + } +} + +static bool ts_parser__call_keyword_lex_fn(TSParser *self) { + if (ts_language_is_wasm(self->language)) { + return ts_wasm_store_call_lex_keyword(self->wasm_store, 0); + } else { + return self->language->keyword_lex_fn(&self->lexer.data, 0); + } +} + +static void ts_parser__external_scanner_create( + TSParser *self +) { + if (self->language && self->language->external_scanner.states) { + if (ts_language_is_wasm(self->language)) { + self->external_scanner_payload = (void *)(uintptr_t)ts_wasm_store_call_scanner_create( + self->wasm_store + ); + if (ts_wasm_store_has_error(self->wasm_store)) { + self->has_scanner_error = true; + } + } else if (self->language->external_scanner.create) { + self->external_scanner_payload = self->language->external_scanner.create(); + } + } +} + +static void ts_parser__external_scanner_destroy( + TSParser *self +) { + if ( + self->language && + self->external_scanner_payload && + self->language->external_scanner.destroy && + !ts_language_is_wasm(self->language) + ) { + self->language->external_scanner.destroy( + self->external_scanner_payload + ); + } + self->external_scanner_payload = NULL; +} + +static unsigned ts_parser__external_scanner_serialize( + TSParser *self +) { + uint32_t length; + if (ts_language_is_wasm(self->language)) { + length = ts_wasm_store_call_scanner_serialize( + self->wasm_store, + (uintptr_t)self->external_scanner_payload, + self->lexer.debug_buffer + ); + if (ts_wasm_store_has_error(self->wasm_store)) { + self->has_scanner_error = true; + } + } else { + length = self->language->external_scanner.serialize( + self->external_scanner_payload, + self->lexer.debug_buffer + ); + } + ts_assert(length <= TREE_SITTER_SERIALIZATION_BUFFER_SIZE); + return length; +} + +static void ts_parser__external_scanner_deserialize( + TSParser *self, + Subtree external_token +) { + const char *data = NULL; + uint32_t length = 0; + if (external_token.ptr) { + data = ts_external_scanner_state_data(&external_token.ptr->external_scanner_state); + length = external_token.ptr->external_scanner_state.length; + } + + if (ts_language_is_wasm(self->language)) { + ts_wasm_store_call_scanner_deserialize( + self->wasm_store, + (uintptr_t)self->external_scanner_payload, + data, + length + ); + if (ts_wasm_store_has_error(self->wasm_store)) { + self->has_scanner_error = true; + } + } else { + self->language->external_scanner.deserialize( + self->external_scanner_payload, + data, + length + ); + } +} + +static bool ts_parser__external_scanner_scan( + TSParser *self, + TSStateId external_lex_state +) { + if (ts_language_is_wasm(self->language)) { + bool result = ts_wasm_store_call_scanner_scan( + self->wasm_store, + (uintptr_t)self->external_scanner_payload, + external_lex_state * self->language->external_token_count + ); + if (ts_wasm_store_has_error(self->wasm_store)) { + self->has_scanner_error = true; + } + return result; + } else { + const bool *valid_external_tokens = ts_language_enabled_external_tokens( + self->language, + external_lex_state + ); + return self->language->external_scanner.scan( + self->external_scanner_payload, + &self->lexer.data, + valid_external_tokens + ); + } +} + +static bool ts_parser__can_reuse_first_leaf( + TSParser *self, + TSStateId state, + Subtree tree, + TableEntry *table_entry +) { + TSSymbol leaf_symbol = ts_subtree_leaf_symbol(tree); + TSStateId leaf_state = ts_subtree_leaf_parse_state(tree); + TSLexerMode current_lex_mode = ts_language_lex_mode_for_state(self->language, state); + TSLexerMode leaf_lex_mode = ts_language_lex_mode_for_state(self->language, leaf_state); + + // At the end of a non-terminal extra node, the lexer normally returns + // NULL, which indicates that the parser should look for a reduce action + // at symbol `0`. Avoid reusing tokens in this situation to ensure that + // the same thing happens when incrementally reparsing. + if (current_lex_mode.lex_state == (uint16_t)(-1)) return false; + + // If the token was created in a state with the same set of lookaheads, it is reusable. + if ( + table_entry->action_count > 0 && + memcmp(&leaf_lex_mode, ¤t_lex_mode, sizeof(TSLexerMode)) == 0 && + ( + leaf_symbol != self->language->keyword_capture_token || + (!ts_subtree_is_keyword(tree) && ts_subtree_parse_state(tree) == state) + ) + ) return true; + + // Empty tokens are not reusable in states with different lookaheads. + if (ts_subtree_size(tree).bytes == 0 && leaf_symbol != ts_builtin_sym_end) return false; + + // If the current state allows external tokens or other tokens that conflict with this + // token, this token is not reusable. + return current_lex_mode.external_lex_state == 0 && table_entry->is_reusable; +} + +static Subtree ts_parser__lex( + TSParser *self, + StackVersion version, + TSStateId parse_state +) { + TSLexerMode lex_mode = ts_language_lex_mode_for_state(self->language, parse_state); + if (lex_mode.lex_state == (uint16_t)-1) { + LOG("no_lookahead_after_non_terminal_extra"); + return NULL_SUBTREE; + } + + const Length start_position = ts_stack_position(self->stack, version); + const Subtree external_token = ts_stack_last_external_token(self->stack, version); + + bool found_external_token = false; + bool error_mode = parse_state == ERROR_STATE; + bool skipped_error = false; + bool called_get_column = false; + int32_t first_error_character = 0; + Length error_start_position = length_zero(); + Length error_end_position = length_zero(); + uint32_t lookahead_end_byte = 0; + uint32_t external_scanner_state_len = 0; + bool external_scanner_state_changed = false; + ts_lexer_reset(&self->lexer, start_position); + + for (;;) { + bool found_token = false; + Length current_position = self->lexer.current_position; + ColumnData column_data = self->lexer.column_data; + + if (lex_mode.external_lex_state != 0) { + LOG( + "lex_external state:%d, row:%u, column:%u", + lex_mode.external_lex_state, + current_position.extent.row, + current_position.extent.column + ); + ts_lexer_start(&self->lexer); + ts_parser__external_scanner_deserialize(self, external_token); + found_token = ts_parser__external_scanner_scan(self, lex_mode.external_lex_state); + if (self->has_scanner_error) return NULL_SUBTREE; + ts_lexer_finish(&self->lexer, &lookahead_end_byte); + + if (found_token) { + external_scanner_state_len = ts_parser__external_scanner_serialize(self); + external_scanner_state_changed = !ts_external_scanner_state_eq( + ts_subtree_external_scanner_state(external_token), + self->lexer.debug_buffer, + external_scanner_state_len + ); + + // Avoid infinite loops caused by the external scanner returning empty tokens. + // Empty tokens are needed in some circumstances, e.g. indent/dedent tokens + // in Python. Ignore the following classes of empty tokens: + // + // * Tokens produced during error recovery. When recovering from an error, + // all tokens are allowed, so it's easy to accidentally return unwanted + // empty tokens. + // * Tokens that are marked as 'extra' in the grammar. These don't change + // the parse state, so they would definitely cause an infinite loop. + if ( + self->lexer.token_end_position.bytes <= current_position.bytes && + !external_scanner_state_changed + ) { + TSSymbol symbol = self->language->external_scanner.symbol_map[self->lexer.data.result_symbol]; + TSStateId next_parse_state = ts_language_next_state(self->language, parse_state, symbol); + bool token_is_extra = (next_parse_state == parse_state); + if (error_mode || !ts_stack_has_advanced_since_error(self->stack, version) || token_is_extra) { + LOG( + "ignore_empty_external_token symbol:%s", + SYM_NAME(self->language->external_scanner.symbol_map[self->lexer.data.result_symbol]) + ); + found_token = false; + } + } + } + + if (found_token) { + found_external_token = true; + called_get_column = self->lexer.did_get_column; + break; + } + + ts_lexer_reset(&self->lexer, current_position); + self->lexer.column_data = column_data; + } + + LOG( + "lex_internal state:%d, row:%u, column:%u", + lex_mode.lex_state, + current_position.extent.row, + current_position.extent.column + ); + ts_lexer_start(&self->lexer); + found_token = ts_parser__call_main_lex_fn(self, lex_mode); + ts_lexer_finish(&self->lexer, &lookahead_end_byte); + if (found_token) break; + + if (!error_mode) { + error_mode = true; + lex_mode = ts_language_lex_mode_for_state(self->language, ERROR_STATE); + ts_lexer_reset(&self->lexer, start_position); + continue; + } + + if (!skipped_error) { + LOG("skip_unrecognized_character"); + skipped_error = true; + error_start_position = self->lexer.token_start_position; + error_end_position = self->lexer.token_start_position; + first_error_character = self->lexer.data.lookahead; + } + + if (self->lexer.current_position.bytes == error_end_position.bytes) { + if (self->lexer.data.eof(&self->lexer.data)) { + self->lexer.data.result_symbol = ts_builtin_sym_error; + break; + } + self->lexer.data.advance(&self->lexer.data, false); + } + + error_end_position = self->lexer.current_position; + } + + Subtree result; + if (skipped_error) { + Length padding = length_sub(error_start_position, start_position); + Length size = length_sub(error_end_position, error_start_position); + uint32_t lookahead_bytes = lookahead_end_byte - error_end_position.bytes; + result = ts_subtree_new_error( + &self->tree_pool, + first_error_character, + padding, + size, + lookahead_bytes, + parse_state, + self->language + ); + } else { + bool is_keyword = false; + TSSymbol symbol = self->lexer.data.result_symbol; + Length padding = length_sub(self->lexer.token_start_position, start_position); + Length size = length_sub(self->lexer.token_end_position, self->lexer.token_start_position); + uint32_t lookahead_bytes = lookahead_end_byte - self->lexer.token_end_position.bytes; + + if (found_external_token) { + symbol = self->language->external_scanner.symbol_map[symbol]; + } else if (symbol == self->language->keyword_capture_token && symbol != 0) { + uint32_t end_byte = self->lexer.token_end_position.bytes; + ts_lexer_reset(&self->lexer, self->lexer.token_start_position); + ts_lexer_start(&self->lexer); + + is_keyword = ts_parser__call_keyword_lex_fn(self); + + if ( + is_keyword && + self->lexer.token_end_position.bytes == end_byte && + ( + ts_language_has_actions(self->language, parse_state, self->lexer.data.result_symbol) || + ts_language_is_reserved_word(self->language, parse_state, self->lexer.data.result_symbol) + ) + ) { + symbol = self->lexer.data.result_symbol; + } + } + + result = ts_subtree_new_leaf( + &self->tree_pool, + symbol, + padding, + size, + lookahead_bytes, + parse_state, + found_external_token, + called_get_column, + is_keyword, + self->language + ); + + if (found_external_token) { + MutableSubtree mut_result = ts_subtree_to_mut_unsafe(result); + ts_external_scanner_state_init( + &mut_result.ptr->external_scanner_state, + self->lexer.debug_buffer, + external_scanner_state_len + ); + mut_result.ptr->has_external_scanner_state_change = external_scanner_state_changed; + } + } + + LOG_LOOKAHEAD( + SYM_NAME(ts_subtree_symbol(result)), + ts_subtree_total_size(result).bytes + ); + return result; +} + +static Subtree ts_parser__get_cached_token( + TSParser *self, + TSStateId state, + size_t position, + Subtree last_external_token, + TableEntry *table_entry +) { + TokenCache *cache = &self->token_cache; + if ( + cache->token.ptr && cache->byte_index == position && + ts_subtree_external_scanner_state_eq(cache->last_external_token, last_external_token) + ) { + ts_language_table_entry(self->language, state, ts_subtree_symbol(cache->token), table_entry); + if (ts_parser__can_reuse_first_leaf(self, state, cache->token, table_entry)) { + ts_subtree_retain(cache->token); + return cache->token; + } + } + return NULL_SUBTREE; +} + +static void ts_parser__set_cached_token( + TSParser *self, + uint32_t byte_index, + Subtree last_external_token, + Subtree token +) { + TokenCache *cache = &self->token_cache; + if (token.ptr) ts_subtree_retain(token); + if (last_external_token.ptr) ts_subtree_retain(last_external_token); + if (cache->token.ptr) ts_subtree_release(&self->tree_pool, cache->token); + if (cache->last_external_token.ptr) ts_subtree_release(&self->tree_pool, cache->last_external_token); + cache->token = token; + cache->byte_index = byte_index; + cache->last_external_token = last_external_token; +} + +static bool ts_parser__has_included_range_difference( + const TSParser *self, + uint32_t start_position, + uint32_t end_position +) { + return ts_range_array_intersects( + &self->included_range_differences, + self->included_range_difference_index, + start_position, + end_position + ); +} + +static Subtree ts_parser__reuse_node( + TSParser *self, + StackVersion version, + TSStateId *state, + uint32_t position, + Subtree last_external_token, + TableEntry *table_entry +) { + Subtree result; + while ((result = reusable_node_tree(&self->reusable_node)).ptr) { + uint32_t byte_offset = reusable_node_byte_offset(&self->reusable_node); + uint32_t end_byte_offset = byte_offset + ts_subtree_total_bytes(result); + + // Do not reuse an EOF node if the included ranges array has changes + // later on in the file. + if (ts_subtree_is_eof(result)) end_byte_offset = UINT32_MAX; + + if (byte_offset > position) { + LOG("before_reusable_node symbol:%s", TREE_NAME(result)); + break; + } + + if (byte_offset < position) { + LOG("past_reusable_node symbol:%s", TREE_NAME(result)); + if (end_byte_offset <= position || !reusable_node_descend(&self->reusable_node)) { + reusable_node_advance(&self->reusable_node); + } + continue; + } + + if (!ts_subtree_external_scanner_state_eq(self->reusable_node.last_external_token, last_external_token)) { + LOG("reusable_node_has_different_external_scanner_state symbol:%s", TREE_NAME(result)); + reusable_node_advance(&self->reusable_node); + continue; + } + + const char *reason = NULL; + if (ts_subtree_has_changes(result)) { + reason = "has_changes"; + } else if (ts_subtree_is_error(result)) { + reason = "is_error"; + } else if (ts_subtree_missing(result)) { + reason = "is_missing"; + } else if (ts_subtree_is_fragile(result)) { + reason = "is_fragile"; + } else if (ts_parser__has_included_range_difference(self, byte_offset, end_byte_offset)) { + reason = "contains_different_included_range"; + } + + if (reason) { + LOG("cant_reuse_node_%s tree:%s", reason, TREE_NAME(result)); + if (!reusable_node_descend(&self->reusable_node)) { + reusable_node_advance(&self->reusable_node); + ts_parser__breakdown_top_of_stack(self, version); + *state = ts_stack_state(self->stack, version); + } + continue; + } + + TSSymbol leaf_symbol = ts_subtree_leaf_symbol(result); + ts_language_table_entry(self->language, *state, leaf_symbol, table_entry); + if (!ts_parser__can_reuse_first_leaf(self, *state, result, table_entry)) { + LOG( + "cant_reuse_node symbol:%s, first_leaf_symbol:%s", + TREE_NAME(result), + SYM_NAME(leaf_symbol) + ); + reusable_node_advance_past_leaf(&self->reusable_node); + break; + } + + LOG("reuse_node symbol:%s", TREE_NAME(result)); + ts_subtree_retain(result); + return result; + } + + return NULL_SUBTREE; +} + +// Determine if a given tree should be replaced by an alternative tree. +// +// The decision is based on the trees' error costs (if any), their dynamic precedence, +// and finally, as a default, by a recursive comparison of the trees' symbols. +static bool ts_parser__select_tree(TSParser *self, Subtree left, Subtree right) { + if (!left.ptr) return true; + if (!right.ptr) return false; + + if (ts_subtree_error_cost(right) < ts_subtree_error_cost(left)) { + LOG("select_smaller_error symbol:%s, over_symbol:%s", TREE_NAME(right), TREE_NAME(left)); + return true; + } + + if (ts_subtree_error_cost(left) < ts_subtree_error_cost(right)) { + LOG("select_smaller_error symbol:%s, over_symbol:%s", TREE_NAME(left), TREE_NAME(right)); + return false; + } + + if (ts_subtree_dynamic_precedence(right) > ts_subtree_dynamic_precedence(left)) { + LOG("select_higher_precedence symbol:%s, prec:%" PRId32 ", over_symbol:%s, other_prec:%" PRId32, + TREE_NAME(right), ts_subtree_dynamic_precedence(right), TREE_NAME(left), + ts_subtree_dynamic_precedence(left)); + return true; + } + + if (ts_subtree_dynamic_precedence(left) > ts_subtree_dynamic_precedence(right)) { + LOG("select_higher_precedence symbol:%s, prec:%" PRId32 ", over_symbol:%s, other_prec:%" PRId32, + TREE_NAME(left), ts_subtree_dynamic_precedence(left), TREE_NAME(right), + ts_subtree_dynamic_precedence(right)); + return false; + } + + if (ts_subtree_error_cost(left) > 0) return true; + + int comparison = ts_subtree_compare(left, right, &self->tree_pool); + switch (comparison) { + case -1: + LOG("select_earlier symbol:%s, over_symbol:%s", TREE_NAME(left), TREE_NAME(right)); + return false; + break; + case 1: + LOG("select_earlier symbol:%s, over_symbol:%s", TREE_NAME(right), TREE_NAME(left)); + return true; + default: + LOG("select_existing symbol:%s, over_symbol:%s", TREE_NAME(left), TREE_NAME(right)); + return false; + } +} + +// Determine if a given tree's children should be replaced by an alternative +// array of children. +static bool ts_parser__select_children( + TSParser *self, + Subtree left, + const SubtreeArray *children +) { + array_assign(&self->scratch_trees, children); + + // Create a temporary subtree using the scratch trees array. This node does + // not perform any allocation except for possibly growing the array to make + // room for its own heap data. The scratch tree is never explicitly released, + // so the same 'scratch trees' array can be reused again later. + MutableSubtree scratch_tree = ts_subtree_new_node( + ts_subtree_symbol(left), + &self->scratch_trees, + 0, + self->language + ); + + return ts_parser__select_tree( + self, + left, + ts_subtree_from_mut(scratch_tree) + ); +} + +static void ts_parser__shift( + TSParser *self, + StackVersion version, + TSStateId state, + Subtree lookahead, + bool extra +) { + bool is_leaf = ts_subtree_child_count(lookahead) == 0; + Subtree subtree_to_push = lookahead; + if (extra != ts_subtree_extra(lookahead) && is_leaf) { + MutableSubtree result = ts_subtree_make_mut(&self->tree_pool, lookahead); + ts_subtree_set_extra(&result, extra); + subtree_to_push = ts_subtree_from_mut(result); + } + + ts_stack_push(self->stack, version, subtree_to_push, !is_leaf, state); + if (ts_subtree_has_external_tokens(subtree_to_push)) { + ts_stack_set_last_external_token( + self->stack, version, ts_subtree_last_external_token(subtree_to_push) + ); + } +} + +static StackVersion ts_parser__reduce( + TSParser *self, + StackVersion version, + TSSymbol symbol, + uint32_t count, + int dynamic_precedence, + uint16_t production_id, + bool is_fragile, + bool end_of_non_terminal_extra +) { + uint32_t initial_version_count = ts_stack_version_count(self->stack); + + // Pop the given number of nodes from the given version of the parse stack. + // If stack versions have previously merged, then there may be more than one + // path back through the stack. For each path, create a new parent node to + // contain the popped children, and push it onto the stack in place of the + // children. + StackSliceArray pop = ts_stack_pop_count(self->stack, version, count); + uint32_t removed_version_count = 0; + uint32_t halted_version_count = ts_stack_halted_version_count(self->stack); + for (uint32_t i = 0; i < pop.size; i++) { + StackSlice slice = *array_get(&pop, i); + StackVersion slice_version = slice.version - removed_version_count; + + // This is where new versions are added to the parse stack. The versions + // will all be sorted and truncated at the end of the outer parsing loop. + // Allow the maximum version count to be temporarily exceeded, but only + // by a limited threshold. + if (slice_version > MAX_VERSION_COUNT + MAX_VERSION_COUNT_OVERFLOW + halted_version_count) { + ts_stack_remove_version(self->stack, slice_version); + ts_subtree_array_delete(&self->tree_pool, &slice.subtrees); + removed_version_count++; + while (i + 1 < pop.size) { + LOG("aborting reduce with too many versions") + StackSlice next_slice = *array_get(&pop, i + 1); + if (next_slice.version != slice.version) break; + ts_subtree_array_delete(&self->tree_pool, &next_slice.subtrees); + i++; + } + continue; + } + + // Extra tokens on top of the stack should not be included in this new parent + // node. They will be re-pushed onto the stack after the parent node is + // created and pushed. + SubtreeArray children = slice.subtrees; + ts_subtree_array_remove_trailing_extras(&children, &self->trailing_extras); + + MutableSubtree parent = ts_subtree_new_node( + symbol, &children, production_id, self->language + ); + + // This pop operation may have caused multiple stack versions to collapse + // into one, because they all diverged from a common state. In that case, + // choose one of the arrays of trees to be the parent node's children, and + // delete the rest of the tree arrays. + while (i + 1 < pop.size) { + StackSlice next_slice = *array_get(&pop, i + 1); + if (next_slice.version != slice.version) break; + i++; + + SubtreeArray next_slice_children = next_slice.subtrees; + ts_subtree_array_remove_trailing_extras(&next_slice_children, &self->trailing_extras2); + + if (ts_parser__select_children( + self, + ts_subtree_from_mut(parent), + &next_slice_children + )) { + ts_subtree_array_clear(&self->tree_pool, &self->trailing_extras); + ts_subtree_release(&self->tree_pool, ts_subtree_from_mut(parent)); + array_swap(&self->trailing_extras, &self->trailing_extras2); + parent = ts_subtree_new_node( + symbol, &next_slice_children, production_id, self->language + ); + } else { + array_clear(&self->trailing_extras2); + ts_subtree_array_delete(&self->tree_pool, &next_slice.subtrees); + } + } + + TSStateId state = ts_stack_state(self->stack, slice_version); + TSStateId next_state = ts_language_next_state(self->language, state, symbol); + if (end_of_non_terminal_extra && next_state == state) { + parent.ptr->extra = true; + } + if (is_fragile || pop.size > 1 || initial_version_count > 1) { + parent.ptr->fragile_left = true; + parent.ptr->fragile_right = true; + parent.ptr->parse_state = TS_TREE_STATE_NONE; + } else { + parent.ptr->parse_state = state; + } + parent.ptr->dynamic_precedence += dynamic_precedence; + + // Push the parent node onto the stack, along with any extra tokens that + // were previously on top of the stack. + ts_stack_push(self->stack, slice_version, ts_subtree_from_mut(parent), false, next_state); + for (uint32_t j = 0; j < self->trailing_extras.size; j++) { + ts_stack_push(self->stack, slice_version, *array_get(&self->trailing_extras, j), false, next_state); + } + + for (StackVersion j = 0; j < slice_version; j++) { + if (j == version) continue; + if (ts_stack_merge(self->stack, j, slice_version)) { + removed_version_count++; + break; + } + } + } + + // Return the first new stack version that was created. + return ts_stack_version_count(self->stack) > initial_version_count + ? initial_version_count + : STACK_VERSION_NONE; +} + +static void ts_parser__accept( + TSParser *self, + StackVersion version, + Subtree lookahead +) { + ts_assert(ts_subtree_is_eof(lookahead)); + ts_stack_push(self->stack, version, lookahead, false, 1); + + StackSliceArray pop = ts_stack_pop_all(self->stack, version); + for (uint32_t i = 0; i < pop.size; i++) { + SubtreeArray trees = array_get(&pop, i)->subtrees; + + Subtree root = NULL_SUBTREE; + for (uint32_t j = trees.size - 1; j + 1 > 0; j--) { + Subtree tree = *array_get(&trees, j); + if (!ts_subtree_extra(tree)) { + ts_assert(!tree.data.is_inline); + uint32_t child_count = ts_subtree_child_count(tree); + const Subtree *children = ts_subtree_children(tree); + for (uint32_t k = 0; k < child_count; k++) { + ts_subtree_retain(children[k]); + } + array_splice(&trees, j, 1, child_count, children); + root = ts_subtree_from_mut(ts_subtree_new_node( + ts_subtree_symbol(tree), + &trees, + tree.ptr->production_id, + self->language + )); + ts_subtree_release(&self->tree_pool, tree); + break; + } + } + + ts_assert(root.ptr); + self->accept_count++; + + if (self->finished_tree.ptr) { + if (ts_parser__select_tree(self, self->finished_tree, root)) { + ts_subtree_release(&self->tree_pool, self->finished_tree); + self->finished_tree = root; + } else { + ts_subtree_release(&self->tree_pool, root); + } + } else { + self->finished_tree = root; + } + } + + ts_stack_remove_version(self->stack, array_get(&pop, 0)->version); + ts_stack_halt(self->stack, version); +} + +static bool ts_parser__do_all_potential_reductions( + TSParser *self, + StackVersion starting_version, + TSSymbol lookahead_symbol +) { + uint32_t initial_version_count = ts_stack_version_count(self->stack); + + bool can_shift_lookahead_symbol = false; + StackVersion version = starting_version; + for (unsigned i = 0; true; i++) { + uint32_t version_count = ts_stack_version_count(self->stack); + if (version >= version_count) break; + + bool merged = false; + for (StackVersion j = initial_version_count; j < version; j++) { + if (ts_stack_merge(self->stack, j, version)) { + merged = true; + break; + } + } + if (merged) continue; + + TSStateId state = ts_stack_state(self->stack, version); + bool has_shift_action = false; + array_clear(&self->reduce_actions); + + TSSymbol first_symbol, end_symbol; + if (lookahead_symbol != 0) { + first_symbol = lookahead_symbol; + end_symbol = lookahead_symbol + 1; + } else { + first_symbol = 1; + end_symbol = self->language->token_count; + } + + for (TSSymbol symbol = first_symbol; symbol < end_symbol; symbol++) { + TableEntry entry; + ts_language_table_entry(self->language, state, symbol, &entry); + for (uint32_t j = 0; j < entry.action_count; j++) { + TSParseAction action = entry.actions[j]; + switch (action.type) { + case TSParseActionTypeShift: + case TSParseActionTypeRecover: + if (!action.shift.extra && !action.shift.repetition) has_shift_action = true; + break; + case TSParseActionTypeReduce: + if (action.reduce.child_count > 0) + ts_reduce_action_set_add(&self->reduce_actions, (ReduceAction) { + .symbol = action.reduce.symbol, + .count = action.reduce.child_count, + .dynamic_precedence = action.reduce.dynamic_precedence, + .production_id = action.reduce.production_id, + }); + break; + default: + break; + } + } + } + + StackVersion reduction_version = STACK_VERSION_NONE; + for (uint32_t j = 0; j < self->reduce_actions.size; j++) { + ReduceAction action = *array_get(&self->reduce_actions, j); + + reduction_version = ts_parser__reduce( + self, version, action.symbol, action.count, + action.dynamic_precedence, action.production_id, + true, false + ); + } + + if (has_shift_action) { + can_shift_lookahead_symbol = true; + } else if (reduction_version != STACK_VERSION_NONE && i < MAX_VERSION_COUNT) { + ts_stack_renumber_version(self->stack, reduction_version, version); + continue; + } else if (lookahead_symbol != 0) { + ts_stack_remove_version(self->stack, version); + } + + if (version == starting_version) { + version = version_count; + } else { + version++; + } + } + + return can_shift_lookahead_symbol; +} + +static bool ts_parser__recover_to_state( + TSParser *self, + StackVersion version, + unsigned depth, + TSStateId goal_state +) { + StackSliceArray pop = ts_stack_pop_count(self->stack, version, depth); + StackVersion previous_version = STACK_VERSION_NONE; + + for (unsigned i = 0; i < pop.size; i++) { + StackSlice slice = *array_get(&pop, i); + + if (slice.version == previous_version) { + ts_subtree_array_delete(&self->tree_pool, &slice.subtrees); + array_erase(&pop, i--); + continue; + } + + if (ts_stack_state(self->stack, slice.version) != goal_state) { + ts_stack_halt(self->stack, slice.version); + ts_subtree_array_delete(&self->tree_pool, &slice.subtrees); + array_erase(&pop, i--); + continue; + } + + SubtreeArray error_trees = ts_stack_pop_error(self->stack, slice.version); + if (error_trees.size > 0) { + ts_assert(error_trees.size == 1); + Subtree error_tree = *array_get(&error_trees, 0); + uint32_t error_child_count = ts_subtree_child_count(error_tree); + if (error_child_count > 0) { + array_splice(&slice.subtrees, 0, 0, error_child_count, ts_subtree_children(error_tree)); + for (unsigned j = 0; j < error_child_count; j++) { + ts_subtree_retain(*array_get(&slice.subtrees, j)); + } + } + ts_subtree_array_delete(&self->tree_pool, &error_trees); + } + + ts_subtree_array_remove_trailing_extras(&slice.subtrees, &self->trailing_extras); + + if (slice.subtrees.size > 0) { + Subtree error = ts_subtree_new_error_node(&slice.subtrees, true, self->language); + ts_stack_push(self->stack, slice.version, error, false, goal_state); + } else { + array_delete(&slice.subtrees); + } + + for (unsigned j = 0; j < self->trailing_extras.size; j++) { + Subtree tree = *array_get(&self->trailing_extras, j); + ts_stack_push(self->stack, slice.version, tree, false, goal_state); + } + + previous_version = slice.version; + } + + return previous_version != STACK_VERSION_NONE; +} + +static void ts_parser__recover( + TSParser *self, + StackVersion version, + Subtree lookahead +) { + bool did_recover = false; + unsigned previous_version_count = ts_stack_version_count(self->stack); + Length position = ts_stack_position(self->stack, version); + StackSummary *summary = ts_stack_get_summary(self->stack, version); + unsigned node_count_since_error = ts_stack_node_count_since_error(self->stack, version); + unsigned current_error_cost = ts_stack_error_cost(self->stack, version); + + // When the parser is in the error state, there are two strategies for recovering with a + // given lookahead token: + // 1. Find a previous state on the stack in which that lookahead token would be valid. Then, + // create a new stack version that is in that state again. This entails popping all of the + // subtrees that have been pushed onto the stack since that previous state, and wrapping + // them in an ERROR node. + // 2. Wrap the lookahead token in an ERROR node, push that ERROR node onto the stack, and + // move on to the next lookahead token, remaining in the error state. + // + // First, try the strategy 1. Upon entering the error state, the parser recorded a summary + // of the previous parse states and their depths. Look at each state in the summary, to see + // if the current lookahead token would be valid in that state. + if (summary && !ts_subtree_is_error(lookahead)) { + for (unsigned i = 0; i < summary->size; i++) { + StackSummaryEntry entry = *array_get(summary, i); + + if (entry.state == ERROR_STATE) continue; + if (entry.position.bytes == position.bytes) continue; + unsigned depth = entry.depth; + if (node_count_since_error > 0) depth++; + + // Do not recover in ways that create redundant stack versions. + bool would_merge = false; + for (unsigned j = 0; j < previous_version_count; j++) { + if ( + ts_stack_state(self->stack, j) == entry.state && + ts_stack_position(self->stack, j).bytes == position.bytes + ) { + would_merge = true; + break; + } + } + if (would_merge) continue; + + // Do not recover if the result would clearly be worse than some existing stack version. + unsigned new_cost = + current_error_cost + + entry.depth * ERROR_COST_PER_SKIPPED_TREE + + (position.bytes - entry.position.bytes) * ERROR_COST_PER_SKIPPED_CHAR + + (position.extent.row - entry.position.extent.row) * ERROR_COST_PER_SKIPPED_LINE; + if (ts_parser__better_version_exists(self, version, false, new_cost)) break; + + // If the current lookahead token is valid in some previous state, recover to that state. + // Then stop looking for further recoveries. + if (ts_language_has_actions(self->language, entry.state, ts_subtree_symbol(lookahead))) { + if (ts_parser__recover_to_state(self, version, depth, entry.state)) { + did_recover = true; + LOG("recover_to_previous state:%u, depth:%u", entry.state, depth); + LOG_STACK(); + break; + } + } + } + } + + // In the process of attempting to recover, some stack versions may have been created + // and subsequently halted. Remove those versions. + for (unsigned i = previous_version_count; i < ts_stack_version_count(self->stack); i++) { + if (!ts_stack_is_active(self->stack, i)) { + LOG("removed paused version:%u", i); + ts_stack_remove_version(self->stack, i--); + LOG_STACK(); + } + } + + // If the parser is still in the error state at the end of the file, just wrap everything + // in an ERROR node and terminate. + if (ts_subtree_is_eof(lookahead)) { + LOG("recover_eof"); + SubtreeArray children = array_new(); + Subtree parent = ts_subtree_new_error_node(&children, false, self->language); + ts_stack_push(self->stack, version, parent, false, 1); + ts_parser__accept(self, version, lookahead); + return; + } + + // If strategy 1 succeeded, a new stack version will have been created which is able to handle + // the current lookahead token. Now, in addition, try strategy 2 described above: skip the + // current lookahead token by wrapping it in an ERROR node. + + // Don't pursue this additional strategy if there are already too many stack versions. + if (did_recover && ts_stack_version_count(self->stack) > MAX_VERSION_COUNT) { + ts_stack_halt(self->stack, version); + ts_subtree_release(&self->tree_pool, lookahead); + return; + } + + if ( + did_recover && + ts_subtree_has_external_scanner_state_change(lookahead) + ) { + ts_stack_halt(self->stack, version); + ts_subtree_release(&self->tree_pool, lookahead); + return; + } + + // Do not recover if the result would clearly be worse than some existing stack version. + unsigned new_cost = + current_error_cost + ERROR_COST_PER_SKIPPED_TREE + + ts_subtree_total_bytes(lookahead) * ERROR_COST_PER_SKIPPED_CHAR + + ts_subtree_total_size(lookahead).extent.row * ERROR_COST_PER_SKIPPED_LINE; + if (ts_parser__better_version_exists(self, version, false, new_cost)) { + ts_stack_halt(self->stack, version); + ts_subtree_release(&self->tree_pool, lookahead); + return; + } + + // If the current lookahead token is an extra token, mark it as extra. This means it won't + // be counted in error cost calculations. + unsigned n; + const TSParseAction *actions = ts_language_actions(self->language, 1, ts_subtree_symbol(lookahead), &n); + if (n > 0 && actions[n - 1].type == TSParseActionTypeShift && actions[n - 1].shift.extra) { + MutableSubtree mutable_lookahead = ts_subtree_make_mut(&self->tree_pool, lookahead); + ts_subtree_set_extra(&mutable_lookahead, true); + lookahead = ts_subtree_from_mut(mutable_lookahead); + } + + // Wrap the lookahead token in an ERROR. + LOG("skip_token symbol:%s", TREE_NAME(lookahead)); + SubtreeArray children = array_new(); + array_reserve(&children, 1); + array_push(&children, lookahead); + MutableSubtree error_repeat = ts_subtree_new_node( + ts_builtin_sym_error_repeat, + &children, + 0, + self->language + ); + + // If other tokens have already been skipped, so there is already an ERROR at the top of the + // stack, then pop that ERROR off the stack and wrap the two ERRORs together into one larger + // ERROR. + if (node_count_since_error > 0) { + StackSliceArray pop = ts_stack_pop_count(self->stack, version, 1); + + // TODO: Figure out how to make this condition occur. + // See https://github.com/atom/atom/issues/18450#issuecomment-439579778 + // If multiple stack versions have merged at this point, just pick one of the errors + // arbitrarily and discard the rest. + if (pop.size > 1) { + for (unsigned i = 1; i < pop.size; i++) { + ts_subtree_array_delete(&self->tree_pool, &array_get(&pop, i)->subtrees); + } + while (ts_stack_version_count(self->stack) > array_get(&pop, 0)->version + 1) { + ts_stack_remove_version(self->stack, array_get(&pop, 0)->version + 1); + } + } + + ts_stack_renumber_version(self->stack, array_get(&pop, 0)->version, version); + array_push(&array_get(&pop, 0)->subtrees, ts_subtree_from_mut(error_repeat)); + error_repeat = ts_subtree_new_node( + ts_builtin_sym_error_repeat, + &array_get(&pop, 0)->subtrees, + 0, + self->language + ); + } + + // Push the new ERROR onto the stack. + ts_stack_push(self->stack, version, ts_subtree_from_mut(error_repeat), false, ERROR_STATE); + if (ts_subtree_has_external_tokens(lookahead)) { + ts_stack_set_last_external_token( + self->stack, version, ts_subtree_last_external_token(lookahead) + ); + } + + bool has_error = true; + for (unsigned i = 0; i < ts_stack_version_count(self->stack); i++) { + ErrorStatus status = ts_parser__version_status(self, i); + if (!status.is_in_error) { + has_error = false; + break; + } + } + self->has_error = has_error; +} + +static void ts_parser__handle_error( + TSParser *self, + StackVersion version, + Subtree lookahead +) { + uint32_t previous_version_count = ts_stack_version_count(self->stack); + + // Perform any reductions that can happen in this state, regardless of the lookahead. After + // skipping one or more invalid tokens, the parser might find a token that would have allowed + // a reduction to take place. + ts_parser__do_all_potential_reductions(self, version, 0); + uint32_t version_count = ts_stack_version_count(self->stack); + Length position = ts_stack_position(self->stack, version); + + // Push a discontinuity onto the stack. Merge all of the stack versions that + // were created in the previous step. + bool did_insert_missing_token = false; + for (StackVersion v = version; v < version_count;) { + if (!did_insert_missing_token) { + TSStateId state = ts_stack_state(self->stack, v); + for ( + TSSymbol missing_symbol = 1; + missing_symbol < (uint16_t)self->language->token_count; + missing_symbol++ + ) { + TSStateId state_after_missing_symbol = ts_language_next_state( + self->language, state, missing_symbol + ); + if (state_after_missing_symbol == 0 || state_after_missing_symbol == state) { + continue; + } + + if (ts_language_has_reduce_action( + self->language, + state_after_missing_symbol, + ts_subtree_leaf_symbol(lookahead) + )) { + // In case the parser is currently outside of any included range, the lexer will + // snap to the beginning of the next included range. The missing token's padding + // must be assigned to position it within the next included range. + ts_lexer_reset(&self->lexer, position); + ts_lexer_mark_end(&self->lexer); + Length padding = length_sub(self->lexer.token_end_position, position); + uint32_t lookahead_bytes = ts_subtree_total_bytes(lookahead) + ts_subtree_lookahead_bytes(lookahead); + + StackVersion version_with_missing_tree = ts_stack_copy_version(self->stack, v); + Subtree missing_tree = ts_subtree_new_missing_leaf( + &self->tree_pool, missing_symbol, + padding, lookahead_bytes, + self->language + ); + ts_stack_push( + self->stack, version_with_missing_tree, + missing_tree, false, + state_after_missing_symbol + ); + + if (ts_parser__do_all_potential_reductions( + self, version_with_missing_tree, + ts_subtree_leaf_symbol(lookahead) + )) { + LOG( + "recover_with_missing symbol:%s, state:%u", + SYM_NAME(missing_symbol), + ts_stack_state(self->stack, version_with_missing_tree) + ); + did_insert_missing_token = true; + break; + } + } + } + } + + ts_stack_push(self->stack, v, NULL_SUBTREE, false, ERROR_STATE); + v = (v == version) ? previous_version_count : v + 1; + } + + for (unsigned i = previous_version_count; i < version_count; i++) { + bool did_merge = ts_stack_merge(self->stack, version, previous_version_count); + ts_assert(did_merge); + } + + ts_stack_record_summary(self->stack, version, MAX_SUMMARY_DEPTH); + + // Begin recovery with the current lookahead node, rather than waiting for the + // next turn of the parse loop. This ensures that the tree accounts for the + // current lookahead token's "lookahead bytes" value, which describes how far + // the lexer needed to look ahead beyond the content of the token in order to + // recognize it. + if (ts_subtree_child_count(lookahead) > 0) { + ts_parser__breakdown_lookahead(self, &lookahead, ERROR_STATE, &self->reusable_node); + } + ts_parser__recover(self, version, lookahead); + + LOG_STACK(); +} + +static bool ts_parser__check_progress(TSParser *self, Subtree *lookahead, const uint32_t *position, unsigned operations) { + self->operation_count += operations; + if (self->operation_count >= OP_COUNT_PER_PARSER_TIMEOUT_CHECK) { + self->operation_count = 0; + } + if (position != NULL) { + self->parse_state.current_byte_offset = *position; + self->parse_state.has_error = self->has_error; + } + if ( + self->operation_count == 0 && + ( + // TODO(amaanq): remove cancellation flag & clock checks before 0.26 + (self->cancellation_flag && atomic_load(self->cancellation_flag)) || + (!clock_is_null(self->end_clock) && clock_is_gt(clock_now(), self->end_clock)) || + (self->parse_options.progress_callback && self->parse_options.progress_callback(&self->parse_state)) + ) + ) { + if (lookahead && lookahead->ptr) { + ts_subtree_release(&self->tree_pool, *lookahead); + } + return false; + } + return true; +} + +static bool ts_parser__advance( + TSParser *self, + StackVersion version, + bool allow_node_reuse +) { + TSStateId state = ts_stack_state(self->stack, version); + uint32_t position = ts_stack_position(self->stack, version).bytes; + Subtree last_external_token = ts_stack_last_external_token(self->stack, version); + + bool did_reuse = true; + Subtree lookahead = NULL_SUBTREE; + TableEntry table_entry = {.action_count = 0}; + + // If possible, reuse a node from the previous syntax tree. + if (allow_node_reuse) { + lookahead = ts_parser__reuse_node( + self, version, &state, position, last_external_token, &table_entry + ); + } + + // If no node from the previous syntax tree could be reused, then try to + // reuse the token previously returned by the lexer. + if (!lookahead.ptr) { + did_reuse = false; + lookahead = ts_parser__get_cached_token( + self, state, position, last_external_token, &table_entry + ); + } + + bool needs_lex = !lookahead.ptr; + for (;;) { + // Otherwise, re-run the lexer. + if (needs_lex) { + needs_lex = false; + lookahead = ts_parser__lex(self, version, state); + if (self->has_scanner_error) return false; + + if (lookahead.ptr) { + ts_parser__set_cached_token(self, position, last_external_token, lookahead); + ts_language_table_entry(self->language, state, ts_subtree_symbol(lookahead), &table_entry); + } + + // When parsing a non-terminal extra, a null lookahead indicates the + // end of the rule. The reduction is stored in the EOF table entry. + // After the reduction, the lexer needs to be run again. + else { + ts_language_table_entry(self->language, state, ts_builtin_sym_end, &table_entry); + } + } + + // If a cancellation flag, timeout, or progress callback was provided, then check every + // time a fixed number of parse actions has been processed. + if (!ts_parser__check_progress(self, &lookahead, &position, 1)) { + return false; + } + + // Process each parse action for the current lookahead token in + // the current state. If there are multiple actions, then this is + // an ambiguous state. REDUCE actions always create a new stack + // version, whereas SHIFT actions update the existing stack version + // and terminate this loop. + bool did_reduce = false; + StackVersion last_reduction_version = STACK_VERSION_NONE; + for (uint32_t i = 0; i < table_entry.action_count; i++) { + TSParseAction action = table_entry.actions[i]; + + switch (action.type) { + case TSParseActionTypeShift: { + if (action.shift.repetition) break; + TSStateId next_state; + if (action.shift.extra) { + next_state = state; + LOG("shift_extra"); + } else { + next_state = action.shift.state; + LOG("shift state:%u", next_state); + } + + if (ts_subtree_child_count(lookahead) > 0) { + ts_parser__breakdown_lookahead(self, &lookahead, state, &self->reusable_node); + next_state = ts_language_next_state(self->language, state, ts_subtree_symbol(lookahead)); + } + + ts_parser__shift(self, version, next_state, lookahead, action.shift.extra); + if (did_reuse) reusable_node_advance(&self->reusable_node); + return true; + } + + case TSParseActionTypeReduce: { + bool is_fragile = table_entry.action_count > 1; + bool end_of_non_terminal_extra = lookahead.ptr == NULL; + LOG("reduce sym:%s, child_count:%u", SYM_NAME(action.reduce.symbol), action.reduce.child_count); + StackVersion reduction_version = ts_parser__reduce( + self, version, action.reduce.symbol, action.reduce.child_count, + action.reduce.dynamic_precedence, action.reduce.production_id, + is_fragile, end_of_non_terminal_extra + ); + did_reduce = true; + if (reduction_version != STACK_VERSION_NONE) { + last_reduction_version = reduction_version; + } + break; + } + + case TSParseActionTypeAccept: { + LOG("accept"); + ts_parser__accept(self, version, lookahead); + return true; + } + + case TSParseActionTypeRecover: { + if (ts_subtree_child_count(lookahead) > 0) { + ts_parser__breakdown_lookahead(self, &lookahead, ERROR_STATE, &self->reusable_node); + } + + ts_parser__recover(self, version, lookahead); + if (did_reuse) reusable_node_advance(&self->reusable_node); + return true; + } + } + } + + // If a reduction was performed, then replace the current stack version + // with one of the stack versions created by a reduction, and continue + // processing this version of the stack with the same lookahead symbol. + if (last_reduction_version != STACK_VERSION_NONE) { + ts_stack_renumber_version(self->stack, last_reduction_version, version); + LOG_STACK(); + state = ts_stack_state(self->stack, version); + + // At the end of a non-terminal extra rule, the lexer will return a + // null subtree, because the parser needs to perform a fixed reduction + // regardless of the lookahead node. After performing that reduction, + // (and completing the non-terminal extra rule) run the lexer again based + // on the current parse state. + if (!lookahead.ptr) { + needs_lex = true; + } else { + ts_language_table_entry( + self->language, + state, + ts_subtree_leaf_symbol(lookahead), + &table_entry + ); + } + + continue; + } + + // A reduction was performed, but was merged into an existing stack version. + // This version can be discarded. + if (did_reduce) { + if (lookahead.ptr) { + ts_subtree_release(&self->tree_pool, lookahead); + } + ts_stack_halt(self->stack, version); + return true; + } + + // If the current lookahead token is a keyword that is not valid, but the + // default word token *is* valid, then treat the lookahead token as the word + // token instead. + if ( + ts_subtree_is_keyword(lookahead) && + ts_subtree_symbol(lookahead) != self->language->keyword_capture_token && + !ts_language_is_reserved_word(self->language, state, ts_subtree_symbol(lookahead)) + ) { + ts_language_table_entry( + self->language, + state, + self->language->keyword_capture_token, + &table_entry + ); + if (table_entry.action_count > 0) { + LOG( + "switch from_keyword:%s, to_word_token:%s", + TREE_NAME(lookahead), + SYM_NAME(self->language->keyword_capture_token) + ); + + MutableSubtree mutable_lookahead = ts_subtree_make_mut(&self->tree_pool, lookahead); + ts_subtree_set_symbol(&mutable_lookahead, self->language->keyword_capture_token, self->language); + lookahead = ts_subtree_from_mut(mutable_lookahead); + continue; + } + } + + // If the current lookahead token is not valid and the previous subtree on + // the stack was reused from an old tree, then it wasn't actually valid to + // reuse that previous subtree. Remove it from the stack, and in its place, + // push each of its children. Then try again to process the current lookahead. + if (ts_parser__breakdown_top_of_stack(self, version)) { + state = ts_stack_state(self->stack, version); + ts_subtree_release(&self->tree_pool, lookahead); + needs_lex = true; + continue; + } + + // Otherwise, there is definitely an error in this version of the parse stack. + // Mark this version as paused and continue processing any other stack + // versions that exist. If some other version advances successfully, then + // this version can simply be removed. But if all versions end up paused, + // then error recovery is needed. + LOG("detect_error lookahead:%s", TREE_NAME(lookahead)); + ts_stack_pause(self->stack, version, lookahead); + return true; + } +} + +static unsigned ts_parser__condense_stack(TSParser *self) { + bool made_changes = false; + unsigned min_error_cost = UINT_MAX; + for (StackVersion i = 0; i < ts_stack_version_count(self->stack); i++) { + // Prune any versions that have been marked for removal. + if (ts_stack_is_halted(self->stack, i)) { + ts_stack_remove_version(self->stack, i); + i--; + continue; + } + + // Keep track of the minimum error cost of any stack version so + // that it can be returned. + ErrorStatus status_i = ts_parser__version_status(self, i); + if (!status_i.is_in_error && status_i.cost < min_error_cost) { + min_error_cost = status_i.cost; + } + + // Examine each pair of stack versions, removing any versions that + // are clearly worse than another version. Ensure that the versions + // are ordered from most promising to least promising. + for (StackVersion j = 0; j < i; j++) { + ErrorStatus status_j = ts_parser__version_status(self, j); + + switch (ts_parser__compare_versions(self, status_j, status_i)) { + case ErrorComparisonTakeLeft: + made_changes = true; + ts_stack_remove_version(self->stack, i); + i--; + j = i; + break; + + case ErrorComparisonPreferLeft: + case ErrorComparisonNone: + if (ts_stack_merge(self->stack, j, i)) { + made_changes = true; + i--; + j = i; + } + break; + + case ErrorComparisonPreferRight: + made_changes = true; + if (ts_stack_merge(self->stack, j, i)) { + i--; + j = i; + } else { + ts_stack_swap_versions(self->stack, i, j); + } + break; + + case ErrorComparisonTakeRight: + made_changes = true; + ts_stack_remove_version(self->stack, j); + i--; + j--; + break; + } + } + } + + // Enforce a hard upper bound on the number of stack versions by + // discarding the least promising versions. + while (ts_stack_version_count(self->stack) > MAX_VERSION_COUNT) { + ts_stack_remove_version(self->stack, MAX_VERSION_COUNT); + made_changes = true; + } + + // If the best-performing stack version is currently paused, or all + // versions are paused, then resume the best paused version and begin + // the error recovery process. Otherwise, remove the paused versions. + if (ts_stack_version_count(self->stack) > 0) { + bool has_unpaused_version = false; + for (StackVersion i = 0, n = ts_stack_version_count(self->stack); i < n; i++) { + if (ts_stack_is_paused(self->stack, i)) { + if (!has_unpaused_version && self->accept_count < MAX_VERSION_COUNT) { + LOG("resume version:%u", i); + min_error_cost = ts_stack_error_cost(self->stack, i); + Subtree lookahead = ts_stack_resume(self->stack, i); + ts_parser__handle_error(self, i, lookahead); + has_unpaused_version = true; + } else { + ts_stack_remove_version(self->stack, i); + made_changes = true; + i--; + n--; + } + } else { + has_unpaused_version = true; + } + } + } + + if (made_changes) { + LOG("condense"); + LOG_STACK(); + } + + return min_error_cost; +} + +static bool ts_parser__balance_subtree(TSParser *self) { + Subtree finished_tree = self->finished_tree; + + // If we haven't canceled balancing in progress before, then we want to clear the tree stack and + // push the initial finished tree onto it. Otherwise, if we're resuming balancing after a + // cancellation, we don't want to clear the tree stack. + if (!self->canceled_balancing) { + array_clear(&self->tree_pool.tree_stack); + if (ts_subtree_child_count(finished_tree) > 0 && finished_tree.ptr->ref_count == 1) { + array_push(&self->tree_pool.tree_stack, ts_subtree_to_mut_unsafe(finished_tree)); + } + } + + while (self->tree_pool.tree_stack.size > 0) { + if (!ts_parser__check_progress(self, NULL, NULL, 1)) { + return false; + } + + MutableSubtree tree = *array_get(&self->tree_pool.tree_stack, + self->tree_pool.tree_stack.size - 1 + ); + + if (tree.ptr->repeat_depth > 0) { + Subtree child1 = ts_subtree_children(tree)[0]; + Subtree child2 = ts_subtree_children(tree)[tree.ptr->child_count - 1]; + long repeat_delta = (long)ts_subtree_repeat_depth(child1) - (long)ts_subtree_repeat_depth(child2); + if (repeat_delta > 0) { + unsigned n = (unsigned)repeat_delta; + + for (unsigned i = n / 2; i > 0; i /= 2) { + ts_subtree_compress(tree, i, self->language, &self->tree_pool.tree_stack); + n -= i; + + // We scale the operation count increment in `ts_parser__check_progress` proportionately to the compression + // size since larger values of i take longer to process. Shifting by 4 empirically provides good check + // intervals (e.g. 193 operations when i=3100) to prevent blocking during large compressions. + uint8_t operations = i >> 4 > 0 ? i >> 4 : 1; + if (!ts_parser__check_progress(self, NULL, NULL, operations)) { + return false; + } + } + } + } + + (void)array_pop(&self->tree_pool.tree_stack); + + for (uint32_t i = 0; i < tree.ptr->child_count; i++) { + Subtree child = ts_subtree_children(tree)[i]; + if (ts_subtree_child_count(child) > 0 && child.ptr->ref_count == 1) { + array_push(&self->tree_pool.tree_stack, ts_subtree_to_mut_unsafe(child)); + } + } + } + + return true; +} + +static bool ts_parser_has_outstanding_parse(TSParser *self) { + return ( + self->canceled_balancing || + self->external_scanner_payload || + ts_stack_state(self->stack, 0) != 1 || + ts_stack_node_count_since_error(self->stack, 0) != 0 + ); +} + +// Parser - Public + +TSParser *ts_parser_new(void) { + TSParser *self = ts_calloc(1, sizeof(TSParser)); + ts_lexer_init(&self->lexer); + array_init(&self->reduce_actions); + array_reserve(&self->reduce_actions, 4); + self->tree_pool = ts_subtree_pool_new(32); + self->stack = ts_stack_new(&self->tree_pool); + self->finished_tree = NULL_SUBTREE; + self->reusable_node = reusable_node_new(); + self->dot_graph_file = NULL; + self->cancellation_flag = NULL; + self->timeout_duration = 0; + self->language = NULL; + self->has_scanner_error = false; + self->has_error = false; + self->canceled_balancing = false; + self->external_scanner_payload = NULL; + self->end_clock = clock_null(); + self->operation_count = 0; + self->old_tree = NULL_SUBTREE; + self->included_range_differences = (TSRangeArray) array_new(); + self->included_range_difference_index = 0; + ts_parser__set_cached_token(self, 0, NULL_SUBTREE, NULL_SUBTREE); + return self; +} + +void ts_parser_delete(TSParser *self) { + if (!self) return; + + ts_parser_set_language(self, NULL); + ts_stack_delete(self->stack); + if (self->reduce_actions.contents) { + array_delete(&self->reduce_actions); + } + if (self->included_range_differences.contents) { + array_delete(&self->included_range_differences); + } + if (self->old_tree.ptr) { + ts_subtree_release(&self->tree_pool, self->old_tree); + self->old_tree = NULL_SUBTREE; + } + ts_wasm_store_delete(self->wasm_store); + ts_lexer_delete(&self->lexer); + ts_parser__set_cached_token(self, 0, NULL_SUBTREE, NULL_SUBTREE); + ts_subtree_pool_delete(&self->tree_pool); + reusable_node_delete(&self->reusable_node); + array_delete(&self->trailing_extras); + array_delete(&self->trailing_extras2); + array_delete(&self->scratch_trees); + ts_free(self); +} + +const TSLanguage *ts_parser_language(const TSParser *self) { + return self->language; +} + +bool ts_parser_set_language(TSParser *self, const TSLanguage *language) { + ts_parser_reset(self); + ts_language_delete(self->language); + self->language = NULL; + + if (language) { + if ( + language->abi_version > TREE_SITTER_LANGUAGE_VERSION || + language->abi_version < TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION + ) return false; + + if (ts_language_is_wasm(language)) { + if ( + !self->wasm_store || + !ts_wasm_store_start(self->wasm_store, &self->lexer.data, language) + ) return false; + } + } + + self->language = ts_language_copy(language); + return true; +} + +TSLogger ts_parser_logger(const TSParser *self) { + return self->lexer.logger; +} + +void ts_parser_set_logger(TSParser *self, TSLogger logger) { + self->lexer.logger = logger; +} + +void ts_parser_print_dot_graphs(TSParser *self, int fd) { + if (self->dot_graph_file) { + fclose(self->dot_graph_file); + } + + if (fd >= 0) { + #ifdef _WIN32 + self->dot_graph_file = _fdopen(fd, "a"); + #else + self->dot_graph_file = fdopen(fd, "a"); + #endif + } else { + self->dot_graph_file = NULL; + } +} + +const size_t *ts_parser_cancellation_flag(const TSParser *self) { + return (const size_t *)self->cancellation_flag; +} + +void ts_parser_set_cancellation_flag(TSParser *self, const size_t *flag) { + self->cancellation_flag = (const volatile size_t *)flag; +} + +uint64_t ts_parser_timeout_micros(const TSParser *self) { + return duration_to_micros(self->timeout_duration); +} + +void ts_parser_set_timeout_micros(TSParser *self, uint64_t timeout_micros) { + self->timeout_duration = duration_from_micros(timeout_micros); +} + +bool ts_parser_set_included_ranges( + TSParser *self, + const TSRange *ranges, + uint32_t count +) { + return ts_lexer_set_included_ranges(&self->lexer, ranges, count); +} + +const TSRange *ts_parser_included_ranges(const TSParser *self, uint32_t *count) { + return ts_lexer_included_ranges(&self->lexer, count); +} + +void ts_parser_reset(TSParser *self) { + ts_parser__external_scanner_destroy(self); + if (self->wasm_store) { + ts_wasm_store_reset(self->wasm_store); + } + + if (self->old_tree.ptr) { + ts_subtree_release(&self->tree_pool, self->old_tree); + self->old_tree = NULL_SUBTREE; + } + + reusable_node_clear(&self->reusable_node); + ts_lexer_reset(&self->lexer, length_zero()); + ts_stack_clear(self->stack); + ts_parser__set_cached_token(self, 0, NULL_SUBTREE, NULL_SUBTREE); + if (self->finished_tree.ptr) { + ts_subtree_release(&self->tree_pool, self->finished_tree); + self->finished_tree = NULL_SUBTREE; + } + self->accept_count = 0; + self->has_scanner_error = false; + self->has_error = false; + self->canceled_balancing = false; + self->parse_options = (TSParseOptions) {0}; + self->parse_state = (TSParseState) {0}; +} + +TSTree *ts_parser_parse( + TSParser *self, + const TSTree *old_tree, + TSInput input +) { + TSTree *result = NULL; + if (!self->language || !input.read) return NULL; + + if (ts_language_is_wasm(self->language)) { + if (!self->wasm_store) return NULL; + ts_wasm_store_start(self->wasm_store, &self->lexer.data, self->language); + } + + ts_lexer_set_input(&self->lexer, input); + array_clear(&self->included_range_differences); + self->included_range_difference_index = 0; + + self->operation_count = 0; + if (self->timeout_duration) { + self->end_clock = clock_after(clock_now(), self->timeout_duration); + } else { + self->end_clock = clock_null(); + } + + if (ts_parser_has_outstanding_parse(self)) { + LOG("resume_parsing"); + if (self->canceled_balancing) goto balance; + } else { + ts_parser__external_scanner_create(self); + if (self->has_scanner_error) goto exit; + + if (old_tree) { + ts_subtree_retain(old_tree->root); + self->old_tree = old_tree->root; + ts_range_array_get_changed_ranges( + old_tree->included_ranges, old_tree->included_range_count, + self->lexer.included_ranges, self->lexer.included_range_count, + &self->included_range_differences + ); + reusable_node_reset(&self->reusable_node, old_tree->root); + LOG("parse_after_edit"); + LOG_TREE(self->old_tree); + for (unsigned i = 0; i < self->included_range_differences.size; i++) { + TSRange *range = array_get(&self->included_range_differences, i); + LOG("different_included_range %u - %u", range->start_byte, range->end_byte); + } + } else { + reusable_node_clear(&self->reusable_node); + LOG("new_parse"); + } + } + + uint32_t position = 0, last_position = 0, version_count = 0; + do { + for ( + StackVersion version = 0; + version_count = ts_stack_version_count(self->stack), + version < version_count; + version++ + ) { + bool allow_node_reuse = version_count == 1; + while (ts_stack_is_active(self->stack, version)) { + LOG( + "process version:%u, version_count:%u, state:%d, row:%u, col:%u", + version, + ts_stack_version_count(self->stack), + ts_stack_state(self->stack, version), + ts_stack_position(self->stack, version).extent.row, + ts_stack_position(self->stack, version).extent.column + ); + + if (!ts_parser__advance(self, version, allow_node_reuse)) { + if (self->has_scanner_error) goto exit; + return NULL; + } + + LOG_STACK(); + + position = ts_stack_position(self->stack, version).bytes; + if (position > last_position || (version > 0 && position == last_position)) { + last_position = position; + break; + } + } + } + + // After advancing each version of the stack, re-sort the versions by their cost, + // removing any versions that are no longer worth pursuing. + unsigned min_error_cost = ts_parser__condense_stack(self); + + // If there's already a finished parse tree that's better than any in-progress version, + // then terminate parsing. Clear the parse stack to remove any extra references to subtrees + // within the finished tree, ensuring that these subtrees can be safely mutated in-place + // for rebalancing. + if (self->finished_tree.ptr && ts_subtree_error_cost(self->finished_tree) < min_error_cost) { + ts_stack_clear(self->stack); + break; + } + + while (self->included_range_difference_index < self->included_range_differences.size) { + TSRange *range = array_get(&self->included_range_differences, self->included_range_difference_index); + if (range->end_byte <= position) { + self->included_range_difference_index++; + } else { + break; + } + } + } while (version_count != 0); + +balance: + ts_assert(self->finished_tree.ptr); + if (!ts_parser__balance_subtree(self)) { + self->canceled_balancing = true; + return false; + } + self->canceled_balancing = false; + LOG("done"); + LOG_TREE(self->finished_tree); + + result = ts_tree_new( + self->finished_tree, + self->language, + self->lexer.included_ranges, + self->lexer.included_range_count + ); + self->finished_tree = NULL_SUBTREE; + +exit: + ts_parser_reset(self); + return result; +} + +TSTree *ts_parser_parse_with_options( + TSParser *self, + const TSTree *old_tree, + TSInput input, + TSParseOptions parse_options +) { + self->parse_options = parse_options; + self->parse_state.payload = parse_options.payload; + TSTree *result = ts_parser_parse(self, old_tree, input); + // Reset parser options before further parse calls. + self->parse_options = (TSParseOptions) {0}; + return result; +} + +TSTree *ts_parser_parse_string( + TSParser *self, + const TSTree *old_tree, + const char *string, + uint32_t length +) { + return ts_parser_parse_string_encoding(self, old_tree, string, length, TSInputEncodingUTF8); +} + +TSTree *ts_parser_parse_string_encoding( + TSParser *self, + const TSTree *old_tree, + const char *string, + uint32_t length, + TSInputEncoding encoding +) { + TSStringInput input = {string, length}; + return ts_parser_parse(self, old_tree, (TSInput) { + &input, + ts_string_input_read, + encoding, + NULL, + }); +} + +void ts_parser_set_wasm_store(TSParser *self, TSWasmStore *store) { + if (self->language && ts_language_is_wasm(self->language)) { + // Copy the assigned language into the new store. + const TSLanguage *copy = ts_language_copy(self->language); + ts_parser_set_language(self, copy); + ts_language_delete(copy); + } + + ts_wasm_store_delete(self->wasm_store); + self->wasm_store = store; +} + +TSWasmStore *ts_parser_take_wasm_store(TSParser *self) { + if (self->language && ts_language_is_wasm(self->language)) { + ts_parser_set_language(self, NULL); + } + + TSWasmStore *result = self->wasm_store; + self->wasm_store = NULL; + return result; +} + +#undef LOG diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/parser.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/parser.h new file mode 100644 index 00000000000..858107deb9e --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/parser.h @@ -0,0 +1,286 @@ +#ifndef TREE_SITTER_PARSER_H_ +#define TREE_SITTER_PARSER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#define ts_builtin_sym_error ((TSSymbol)-1) +#define ts_builtin_sym_end 0 +#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 + +#ifndef TREE_SITTER_API_H_ +typedef uint16_t TSStateId; +typedef uint16_t TSSymbol; +typedef uint16_t TSFieldId; +typedef struct TSLanguage TSLanguage; +typedef struct TSLanguageMetadata { + uint8_t major_version; + uint8_t minor_version; + uint8_t patch_version; +} TSLanguageMetadata; +#endif + +typedef struct { + TSFieldId field_id; + uint8_t child_index; + bool inherited; +} TSFieldMapEntry; + +// Used to index the field and supertype maps. +typedef struct { + uint16_t index; + uint16_t length; +} TSMapSlice; + +typedef struct { + bool visible; + bool named; + bool supertype; +} TSSymbolMetadata; + +typedef struct TSLexer TSLexer; + +struct TSLexer { + int32_t lookahead; + TSSymbol result_symbol; + void (*advance)(TSLexer *, bool); + void (*mark_end)(TSLexer *); + uint32_t (*get_column)(TSLexer *); + bool (*is_at_included_range_start)(const TSLexer *); + bool (*eof)(const TSLexer *); + void (*log)(const TSLexer *, const char *, ...); +}; + +typedef enum { + TSParseActionTypeShift, + TSParseActionTypeReduce, + TSParseActionTypeAccept, + TSParseActionTypeRecover, +} TSParseActionType; + +typedef union { + struct { + uint8_t type; + TSStateId state; + bool extra; + bool repetition; + } shift; + struct { + uint8_t type; + uint8_t child_count; + TSSymbol symbol; + int16_t dynamic_precedence; + uint16_t production_id; + } reduce; + uint8_t type; +} TSParseAction; + +typedef struct { + uint16_t lex_state; + uint16_t external_lex_state; +} TSLexMode; + +typedef struct { + uint16_t lex_state; + uint16_t external_lex_state; + uint16_t reserved_word_set_id; +} TSLexerMode; + +typedef union { + TSParseAction action; + struct { + uint8_t count; + bool reusable; + } entry; +} TSParseActionEntry; + +typedef struct { + int32_t start; + int32_t end; +} TSCharacterRange; + +struct TSLanguage { + uint32_t abi_version; + uint32_t symbol_count; + uint32_t alias_count; + uint32_t token_count; + uint32_t external_token_count; + uint32_t state_count; + uint32_t large_state_count; + uint32_t production_id_count; + uint32_t field_count; + uint16_t max_alias_sequence_length; + const uint16_t *parse_table; + const uint16_t *small_parse_table; + const uint32_t *small_parse_table_map; + const TSParseActionEntry *parse_actions; + const char * const *symbol_names; + const char * const *field_names; + const TSMapSlice *field_map_slices; + const TSFieldMapEntry *field_map_entries; + const TSSymbolMetadata *symbol_metadata; + const TSSymbol *public_symbol_map; + const uint16_t *alias_map; + const TSSymbol *alias_sequences; + const TSLexerMode *lex_modes; + bool (*lex_fn)(TSLexer *, TSStateId); + bool (*keyword_lex_fn)(TSLexer *, TSStateId); + TSSymbol keyword_capture_token; + struct { + const bool *states; + const TSSymbol *symbol_map; + void *(*create)(void); + void (*destroy)(void *); + bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist); + unsigned (*serialize)(void *, char *); + void (*deserialize)(void *, const char *, unsigned); + } external_scanner; + const TSStateId *primary_state_ids; + const char *name; + const TSSymbol *reserved_words; + uint16_t max_reserved_word_set_size; + uint32_t supertype_count; + const TSSymbol *supertype_symbols; + const TSMapSlice *supertype_map_slices; + const TSSymbol *supertype_map_entries; + TSLanguageMetadata metadata; +}; + +static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) { + uint32_t index = 0; + uint32_t size = len - index; + while (size > 1) { + uint32_t half_size = size / 2; + uint32_t mid_index = index + half_size; + const TSCharacterRange *range = &ranges[mid_index]; + if (lookahead >= range->start && lookahead <= range->end) { + return true; + } else if (lookahead > range->end) { + index = mid_index; + } + size -= half_size; + } + const TSCharacterRange *range = &ranges[index]; + return (lookahead >= range->start && lookahead <= range->end); +} + +/* + * Lexer Macros + */ + +#ifdef _MSC_VER +#define UNUSED __pragma(warning(suppress : 4101)) +#else +#define UNUSED __attribute__((unused)) +#endif + +#define START_LEXER() \ + bool result = false; \ + bool skip = false; \ + UNUSED \ + bool eof = false; \ + int32_t lookahead; \ + goto start; \ + next_state: \ + lexer->advance(lexer, skip); \ + start: \ + skip = false; \ + lookahead = lexer->lookahead; + +#define ADVANCE(state_value) \ + { \ + state = state_value; \ + goto next_state; \ + } + +#define ADVANCE_MAP(...) \ + { \ + static const uint16_t map[] = { __VA_ARGS__ }; \ + for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \ + if (map[i] == lookahead) { \ + state = map[i + 1]; \ + goto next_state; \ + } \ + } \ + } + +#define SKIP(state_value) \ + { \ + skip = true; \ + state = state_value; \ + goto next_state; \ + } + +#define ACCEPT_TOKEN(symbol_value) \ + result = true; \ + lexer->result_symbol = symbol_value; \ + lexer->mark_end(lexer); + +#define END_STATE() return result; + +/* + * Parse Table Macros + */ + +#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT) + +#define STATE(id) id + +#define ACTIONS(id) id + +#define SHIFT(state_value) \ + {{ \ + .shift = { \ + .type = TSParseActionTypeShift, \ + .state = (state_value) \ + } \ + }} + +#define SHIFT_REPEAT(state_value) \ + {{ \ + .shift = { \ + .type = TSParseActionTypeShift, \ + .state = (state_value), \ + .repetition = true \ + } \ + }} + +#define SHIFT_EXTRA() \ + {{ \ + .shift = { \ + .type = TSParseActionTypeShift, \ + .extra = true \ + } \ + }} + +#define REDUCE(symbol_name, children, precedence, prod_id) \ + {{ \ + .reduce = { \ + .type = TSParseActionTypeReduce, \ + .symbol = symbol_name, \ + .child_count = children, \ + .dynamic_precedence = precedence, \ + .production_id = prod_id \ + }, \ + }} + +#define RECOVER() \ + {{ \ + .type = TSParseActionTypeRecover \ + }} + +#define ACCEPT_INPUT() \ + {{ \ + .type = TSParseActionTypeAccept \ + }} + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_PARSER_H_ diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/point.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/point.h new file mode 100644 index 00000000000..39581988435 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/point.h @@ -0,0 +1,48 @@ +#ifndef TREE_SITTER_POINT_H_ +#define TREE_SITTER_POINT_H_ + +#include "tree_sitter/api.h" + +#define POINT_ZERO ((TSPoint) {0, 0}) +#define POINT_MAX ((TSPoint) {UINT32_MAX, UINT32_MAX}) + +static inline TSPoint point__new(unsigned row, unsigned column) { + TSPoint result = {row, column}; + return result; +} + +static inline TSPoint point_add(TSPoint a, TSPoint b) { + if (b.row > 0) + return point__new(a.row + b.row, b.column); + else + return point__new(a.row, a.column + b.column); +} + +static inline TSPoint point_sub(TSPoint a, TSPoint b) { + if (a.row > b.row) + return point__new(a.row - b.row, a.column); + else + return point__new(0, (a.column >= b.column) ? a.column - b.column : 0); +} + +static inline bool point_lte(TSPoint a, TSPoint b) { + return (a.row < b.row) || (a.row == b.row && a.column <= b.column); +} + +static inline bool point_lt(TSPoint a, TSPoint b) { + return (a.row < b.row) || (a.row == b.row && a.column < b.column); +} + +static inline bool point_gt(TSPoint a, TSPoint b) { + return (a.row > b.row) || (a.row == b.row && a.column > b.column); +} + +static inline bool point_gte(TSPoint a, TSPoint b) { + return (a.row > b.row) || (a.row == b.row && a.column >= b.column); +} + +static inline bool point_eq(TSPoint a, TSPoint b) { + return a.row == b.row && a.column == b.column; +} + +#endif diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/portable/endian.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/portable/endian.h new file mode 100644 index 00000000000..a6560826a4b --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/portable/endian.h @@ -0,0 +1,241 @@ +// "License": Public Domain +// I, Mathias Panzenböck, place this file hereby into the public domain. Use it at your own risk for whatever you like. +// In case there are jurisdictions that don't support putting things in the public domain you can also consider it to +// be "dual licensed" under the BSD, MIT and Apache licenses, if you want to. This code is trivial anyway. Consider it +// an example on how to get the endian conversion functions on different platforms. + +// updates from https://github.com/mikepb/endian.h/issues/4 + +#ifndef ENDIAN_H +#define ENDIAN_H + +#if (defined(_WIN16) || defined(_WIN32) || defined(_WIN64)) && !defined(__WINDOWS__) + +# define __WINDOWS__ + +#endif + +#if defined(HAVE_ENDIAN_H) || \ + defined(__linux__) || \ + defined(__GNU__) || \ + defined(__HAIKU__) || \ + defined(__illumos__) || \ + defined(__NetBSD__) || \ + defined(__OpenBSD__) || \ + defined(__CYGWIN__) || \ + defined(__MSYS__) || \ + defined(__EMSCRIPTEN__) || \ + defined(__wasi__) || \ + defined(__wasm__) + +#if defined(__NetBSD__) +#define _NETBSD_SOURCE 1 +#endif + +# include + +#elif defined(HAVE_SYS_ENDIAN_H) || \ + defined(__FreeBSD__) || \ + defined(__DragonFly__) + +# include + +#elif defined(__APPLE__) +# define __BYTE_ORDER BYTE_ORDER +# define __BIG_ENDIAN BIG_ENDIAN +# define __LITTLE_ENDIAN LITTLE_ENDIAN +# define __PDP_ENDIAN PDP_ENDIAN + +# if !defined(_POSIX_C_SOURCE) +# include + +# define htobe16(x) OSSwapHostToBigInt16(x) +# define htole16(x) OSSwapHostToLittleInt16(x) +# define be16toh(x) OSSwapBigToHostInt16(x) +# define le16toh(x) OSSwapLittleToHostInt16(x) + +# define htobe32(x) OSSwapHostToBigInt32(x) +# define htole32(x) OSSwapHostToLittleInt32(x) +# define be32toh(x) OSSwapBigToHostInt32(x) +# define le32toh(x) OSSwapLittleToHostInt32(x) + +# define htobe64(x) OSSwapHostToBigInt64(x) +# define htole64(x) OSSwapHostToLittleInt64(x) +# define be64toh(x) OSSwapBigToHostInt64(x) +# define le64toh(x) OSSwapLittleToHostInt64(x) +# else +# if BYTE_ORDER == LITTLE_ENDIAN +# define htobe16(x) __builtin_bswap16(x) +# define htole16(x) (x) +# define be16toh(x) __builtin_bswap16(x) +# define le16toh(x) (x) + +# define htobe32(x) __builtin_bswap32(x) +# define htole32(x) (x) +# define be32toh(x) __builtin_bswap32(x) +# define le32toh(x) (x) + +# define htobe64(x) __builtin_bswap64(x) +# define htole64(x) (x) +# define be64toh(x) __builtin_bswap64(x) +# define le64toh(x) (x) +# elif BYTE_ORDER == BIG_ENDIAN +# define htobe16(x) (x) +# define htole16(x) __builtin_bswap16(x) +# define be16toh(x) (x) +# define le16toh(x) __builtin_bswap16(x) + +# define htobe32(x) (x) +# define htole32(x) __builtin_bswap32(x) +# define be32toh(x) (x) +# define le32toh(x) __builtin_bswap32(x) + +# define htobe64(x) (x) +# define htole64(x) __builtin_bswap64(x) +# define be64toh(x) (x) +# define le64toh(x) __builtin_bswap64(x) +# else +# error byte order not supported +# endif +# endif + +#elif defined(__WINDOWS__) + +# if defined(_MSC_VER) && !defined(__clang__) +# include +# define B_SWAP_16(x) _byteswap_ushort(x) +# define B_SWAP_32(x) _byteswap_ulong(x) +# define B_SWAP_64(x) _byteswap_uint64(x) +# else +# define B_SWAP_16(x) __builtin_bswap16(x) +# define B_SWAP_32(x) __builtin_bswap32(x) +# define B_SWAP_64(x) __builtin_bswap64(x) +# endif + +# if defined(__MINGW32__) || defined(HAVE_SYS_PARAM_H) +# include +# endif + +# ifndef BIG_ENDIAN +# ifdef __BIG_ENDIAN +# define BIG_ENDIAN __BIG_ENDIAN +# elif defined(__ORDER_BIG_ENDIAN__) +# define BIG_ENDIAN __ORDER_BIG_ENDIAN__ +# else +# define BIG_ENDIAN 4321 +# endif +# endif + +# ifndef LITTLE_ENDIAN +# ifdef __LITTLE_ENDIAN +# define LITTLE_ENDIAN __LITTLE_ENDIAN +# elif defined(__ORDER_LITTLE_ENDIAN__) +# define LITTLE_ENDIAN __ORDER_LITTLE_ENDIAN__ +# else +# define LITTLE_ENDIAN 1234 +# endif +# endif + +# ifndef BYTE_ORDER +# ifdef __BYTE_ORDER +# define BYTE_ORDER __BYTE_ORDER +# elif defined(__BYTE_ORDER__) +# define BYTE_ORDER __BYTE_ORDER__ +# else + /* assume LE on Windows if nothing was defined */ +# define BYTE_ORDER LITTLE_ENDIAN +# endif +# endif + +# if BYTE_ORDER == LITTLE_ENDIAN + +# define htobe16(x) B_SWAP_16(x) +# define htole16(x) (x) +# define be16toh(x) B_SWAP_16(x) +# define le16toh(x) (x) + +# define htobe32(x) B_SWAP_32(x) +# define htole32(x) (x) +# define be32toh(x) B_SWAP_32(x) +# define le32toh(x) (x) + +# define htobe64(x) B_SWAP_64(x) +# define htole64(x) (x) +# define be64toh(x) B_SWAP_64(x) +# define le64toh(x) (x) + +# elif BYTE_ORDER == BIG_ENDIAN + +# define htobe16(x) (x) +# define htole16(x) B_SWAP_16(x) +# define be16toh(x) (x) +# define le16toh(x) B_SWAP_16(x) + +# define htobe32(x) (x) +# define htole32(x) B_SWAP_32(x) +# define be32toh(x) (x) +# define le32toh(x) B_SWAP_32(x) + +# define htobe64(x) (x) +# define htole64(x) B_SWAP_64(x) +# define be64toh(x) (x) +# define le64toh(x) B_SWAP_64(x) + +# else + +# error byte order not supported + +# endif + +#elif defined(__QNXNTO__) + +# include + +# define __LITTLE_ENDIAN 1234 +# define __BIG_ENDIAN 4321 +# define __PDP_ENDIAN 3412 + +# if defined(__BIGENDIAN__) + +# define __BYTE_ORDER __BIG_ENDIAN + +# define htobe16(x) (x) +# define htobe32(x) (x) +# define htobe64(x) (x) + +# define htole16(x) ENDIAN_SWAP16(x) +# define htole32(x) ENDIAN_SWAP32(x) +# define htole64(x) ENDIAN_SWAP64(x) + +# elif defined(__LITTLEENDIAN__) + +# define __BYTE_ORDER __LITTLE_ENDIAN + +# define htole16(x) (x) +# define htole32(x) (x) +# define htole64(x) (x) + +# define htobe16(x) ENDIAN_SWAP16(x) +# define htobe32(x) ENDIAN_SWAP32(x) +# define htobe64(x) ENDIAN_SWAP64(x) + +# else + +# error byte order not supported + +# endif + +# define be16toh(x) ENDIAN_BE16(x) +# define be32toh(x) ENDIAN_BE32(x) +# define be64toh(x) ENDIAN_BE64(x) +# define le16toh(x) ENDIAN_LE16(x) +# define le32toh(x) ENDIAN_LE32(x) +# define le64toh(x) ENDIAN_LE64(x) + +#else + +# error platform not supported + +#endif + +#endif diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/query.c b/Externals/crystaledit/editlib/tree-sitter/lib/src/query.c new file mode 100644 index 00000000000..6c514887e0b --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/query.c @@ -0,0 +1,4347 @@ +/* + * On NetBSD, defining standard requirements like this removes symbols + * from the namespace; however, we need non-standard symbols for + * endian.h. + */ +#if defined(__NetBSD__) && defined(_POSIX_C_SOURCE) +#undef _POSIX_C_SOURCE +#endif + +#include "tree_sitter/api.h" +#include "./alloc.h" +#include "./array.h" +#include "./clock.h" +#include "./language.h" +#include "./point.h" +#include "./tree_cursor.h" +#include "./unicode.h" +#include + +// #define DEBUG_ANALYZE_QUERY +// #define DEBUG_EXECUTE_QUERY + +#define MAX_STEP_CAPTURE_COUNT 3 +#define MAX_NEGATED_FIELD_COUNT 8 +#define MAX_STATE_PREDECESSOR_COUNT 256 +#define MAX_ANALYSIS_STATE_DEPTH 8 +#define MAX_ANALYSIS_ITERATION_COUNT 256 + +/* + * Stream - A sequence of unicode characters derived from a UTF8 string. + * This struct is used in parsing queries from S-expressions. + */ +typedef struct { + const char *input; + const char *start; + const char *end; + int32_t next; + uint8_t next_size; +} Stream; + +/* + * QueryStep - A step in the process of matching a query. Each node within + * a query S-expression corresponds to one of these steps. An entire pattern + * is represented as a sequence of these steps. The basic properties of a + * node are represented by these fields: + * - `symbol` - The grammar symbol to match. A zero value represents the + * wildcard symbol, '_'. + * - `field` - The field name to match. A zero value means that a field name + * was not specified. + * - `capture_ids` - An array of integers representing the names of captures + * associated with this node in the pattern, terminated by a `NONE` value. + * - `depth` - The depth where this node occurs in the pattern. The root node + * of the pattern has depth zero. + * - `negated_field_list_id` - An id representing a set of fields that must + * not be present on a node matching this step. + * + * Steps have some additional fields in order to handle the `.` (or "anchor") operator, + * which forbids additional child nodes: + * - `is_immediate` - Indicates that the node matching this step cannot be preceded + * by other sibling nodes that weren't specified in the pattern. + * - `is_last_child` - Indicates that the node matching this step cannot have any + * subsequent named siblings. + * + * For simple patterns, steps are matched in sequential order. But in order to + * handle alternative/repeated/optional sub-patterns, query steps are not always + * structured as a linear sequence; they sometimes need to split and merge. This + * is done using the following fields: + * - `alternative_index` - The index of a different query step that serves as + * an alternative to this step. A `NONE` value represents no alternative. + * When a query state reaches a step with an alternative index, the state + * is duplicated, with one copy remaining at the original step, and one copy + * moving to the alternative step. The alternative may have its own alternative + * step, so this splitting is an iterative process. + * - `is_dead_end` - Indicates that this state cannot be passed directly, and + * exists only in order to redirect to an alternative index, with no splitting. + * - `is_pass_through` - Indicates that state has no matching logic of its own, + * and exists only to split a state. One copy of the state advances immediately + * to the next step, and one moves to the alternative step. + * - `alternative_is_immediate` - Indicates that this step's alternative step + * should be treated as if `is_immediate` is true. + * + * Steps also store some derived state that summarizes how they relate to other + * steps within the same pattern. This is used to optimize the matching process: + * - `contains_captures` - Indicates that this step or one of its child steps + * has a non-empty `capture_ids` list. + * - `parent_pattern_guaranteed` - Indicates that if this step is reached, then + * it and all of its subsequent sibling steps within the same parent pattern + * are guaranteed to match. + * - `root_pattern_guaranteed` - Similar to `parent_pattern_guaranteed`, but + * for the entire top-level pattern. When iterating through a query's + * captures using `ts_query_cursor_next_capture`, this field is used to + * detect that a capture can safely be returned from a match that has not + * even completed yet. + */ +typedef struct { + TSSymbol symbol; + TSSymbol supertype_symbol; + TSFieldId field; + uint16_t capture_ids[MAX_STEP_CAPTURE_COUNT]; + uint16_t depth; + uint16_t alternative_index; + uint16_t negated_field_list_id; + bool is_named: 1; + bool is_immediate: 1; + bool is_last_child: 1; + bool is_pass_through: 1; + bool is_dead_end: 1; + bool alternative_is_immediate: 1; + bool contains_captures: 1; + bool root_pattern_guaranteed: 1; + bool parent_pattern_guaranteed: 1; + bool is_missing: 1; +} QueryStep; + +/* + * Slice - A slice of an external array. Within a query, capture names, + * literal string values, and predicate step information are stored in three + * contiguous arrays. Individual captures, string values, and predicates are + * represented as slices of these three arrays. + */ +typedef struct { + uint32_t offset; + uint32_t length; +} Slice; + +/* + * SymbolTable - a two-way mapping of strings to ids. + */ +typedef struct { + Array(char) characters; + Array(Slice) slices; +} SymbolTable; + +/** + * CaptureQuantifiers - a data structure holding the quantifiers of pattern captures. + */ +typedef Array(uint8_t) CaptureQuantifiers; + +/* + * PatternEntry - Information about the starting point for matching a particular + * pattern. These entries are stored in a 'pattern map' - a sorted array that + * makes it possible to efficiently lookup patterns based on the symbol for their + * first step. The entry consists of the following fields: + * - `pattern_index` - the index of the pattern within the query + * - `step_index` - the index of the pattern's first step in the shared `steps` array + * - `is_rooted` - whether or not the pattern has a single root node. This property + * affects decisions about whether or not to start the pattern for nodes outside + * of a QueryCursor's range restriction. + */ +typedef struct { + uint16_t step_index; + uint16_t pattern_index; + bool is_rooted; +} PatternEntry; + +typedef struct { + Slice steps; + Slice predicate_steps; + uint32_t start_byte; + uint32_t end_byte; + bool is_non_local; +} QueryPattern; + +typedef struct { + uint32_t byte_offset; + uint16_t step_index; +} StepOffset; + +/* + * QueryState - The state of an in-progress match of a particular pattern + * in a query. While executing, a `TSQueryCursor` must keep track of a number + * of possible in-progress matches. Each of those possible matches is + * represented as one of these states. Fields: + * - `id` - A numeric id that is exposed to the public API. This allows the + * caller to remove a given match, preventing any more of its captures + * from being returned. + * - `start_depth` - The depth in the tree where the first step of the state's + * pattern was matched. + * - `pattern_index` - The pattern that the state is matching. + * - `consumed_capture_count` - The number of captures from this match that + * have already been returned. + * - `capture_list_id` - A numeric id that can be used to retrieve the state's + * list of captures from the `CaptureListPool`. + * - `seeking_immediate_match` - A flag that indicates that the state's next + * step must be matched by the very next sibling. This is used when + * processing repetitions, or when processing a wildcard node followed by + * an anchor. + * - `has_in_progress_alternatives` - A flag that indicates that there is are + * other states that have the same captures as this state, but are at + * different steps in their pattern. This means that in order to obey the + * 'longest-match' rule, this state should not be returned as a match until + * it is clear that there can be no other alternative match with more captures. + */ +typedef struct { + uint32_t id; + uint32_t capture_list_id; + uint16_t start_depth; + uint16_t step_index; + uint16_t pattern_index; + uint16_t consumed_capture_count: 12; + bool seeking_immediate_match: 1; + bool has_in_progress_alternatives: 1; + bool dead: 1; + bool needs_parent: 1; +} QueryState; + +typedef Array(TSQueryCapture) CaptureList; + +/* + * CaptureListPool - A collection of *lists* of captures. Each query state needs + * to maintain its own list of captures. To avoid repeated allocations, this struct + * maintains a fixed set of capture lists, and keeps track of which ones are + * currently in use by a query state. + */ +typedef struct { + Array(CaptureList) list; + CaptureList empty_list; + // The maximum number of capture lists that we are allowed to allocate. We + // never allow `list` to allocate more entries than this, dropping pending + // matches if needed to stay under the limit. + uint32_t max_capture_list_count; + // The number of capture lists allocated in `list` that are not currently in + // use. We reuse those existing-but-unused capture lists before trying to + // allocate any new ones. We use an invalid value (UINT32_MAX) for a capture + // list's length to indicate that it's not in use. + uint32_t free_capture_list_count; +} CaptureListPool; + +/* + * AnalysisState - The state needed for walking the parse table when analyzing + * a query pattern, to determine at which steps the pattern might fail to match. + */ +typedef struct { + TSStateId parse_state; + TSSymbol parent_symbol; + uint16_t child_index; + TSFieldId field_id: 15; + bool done: 1; +} AnalysisStateEntry; + +typedef struct { + AnalysisStateEntry stack[MAX_ANALYSIS_STATE_DEPTH]; + uint16_t depth; + uint16_t step_index; + TSSymbol root_symbol; +} AnalysisState; + +typedef Array(AnalysisState *) AnalysisStateSet; + +typedef struct { + AnalysisStateSet states; + AnalysisStateSet next_states; + AnalysisStateSet deeper_states; + AnalysisStateSet state_pool; + Array(uint16_t) final_step_indices; + Array(TSSymbol) finished_parent_symbols; + bool did_abort; +} QueryAnalysis; + +/* + * AnalysisSubgraph - A subset of the states in the parse table that are used + * in constructing nodes with a certain symbol. Each state is accompanied by + * some information about the possible node that could be produced in + * downstream states. + */ +typedef struct { + TSStateId state; + uint16_t production_id; + uint8_t child_index: 7; + bool done: 1; +} AnalysisSubgraphNode; + +typedef struct { + TSSymbol symbol; + Array(TSStateId) start_states; + Array(AnalysisSubgraphNode) nodes; +} AnalysisSubgraph; + +typedef Array(AnalysisSubgraph) AnalysisSubgraphArray; + +/* + * StatePredecessorMap - A map that stores the predecessors of each parse state. + * This is used during query analysis to determine which parse states can lead + * to which reduce actions. + */ +typedef struct { + TSStateId *contents; +} StatePredecessorMap; + +/* + * TSQuery - A tree query, compiled from a string of S-expressions. The query + * itself is immutable. The mutable state used in the process of executing the + * query is stored in a `TSQueryCursor`. + */ +struct TSQuery { + SymbolTable captures; + SymbolTable predicate_values; + Array(CaptureQuantifiers) capture_quantifiers; + Array(QueryStep) steps; + Array(PatternEntry) pattern_map; + Array(TSQueryPredicateStep) predicate_steps; + Array(QueryPattern) patterns; + Array(StepOffset) step_offsets; + Array(TSFieldId) negated_fields; + Array(char) string_buffer; + Array(TSSymbol) repeat_symbols_with_rootless_patterns; + const TSLanguage *language; + uint16_t wildcard_root_pattern_count; +}; + +/* + * TSQueryCursor - A stateful struct used to execute a query on a tree. + */ +struct TSQueryCursor { + const TSQuery *query; + TSTreeCursor cursor; + Array(QueryState) states; + Array(QueryState) finished_states; + CaptureListPool capture_list_pool; + uint32_t depth; + uint32_t max_start_depth; + uint32_t start_byte; + uint32_t end_byte; + TSPoint start_point; + TSPoint end_point; + uint32_t next_state_id; + TSClock end_clock; + TSDuration timeout_duration; + const TSQueryCursorOptions *query_options; + TSQueryCursorState query_state; + unsigned operation_count; + bool on_visible_node; + bool ascending; + bool halted; + bool did_exceed_match_limit; +}; + +static const TSQueryError PARENT_DONE = -1; +static const uint16_t PATTERN_DONE_MARKER = UINT16_MAX; +static const uint16_t NONE = UINT16_MAX; +static const TSSymbol WILDCARD_SYMBOL = 0; +static const unsigned OP_COUNT_PER_QUERY_TIMEOUT_CHECK = 100; + +/********** + * Stream + **********/ + +// Advance to the next unicode code point in the stream. +static bool stream_advance(Stream *self) { + self->input += self->next_size; + if (self->input < self->end) { + uint32_t size = ts_decode_utf8( + (const uint8_t *)self->input, + (uint32_t)(self->end - self->input), + &self->next + ); + if (size > 0) { + self->next_size = size; + return true; + } + } else { + self->next_size = 0; + self->next = '\0'; + } + return false; +} + +// Reset the stream to the given input position, represented as a pointer +// into the input string. +static void stream_reset(Stream *self, const char *input) { + self->input = input; + self->next_size = 0; + stream_advance(self); +} + +static Stream stream_new(const char *string, uint32_t length) { + Stream self = { + .next = 0, + .input = string, + .start = string, + .end = string + length, + }; + stream_advance(&self); + return self; +} + +static void stream_skip_whitespace(Stream *self) { + for (;;) { + if (iswspace(self->next)) { + stream_advance(self); + } else if (self->next == ';') { + // skip over comments + stream_advance(self); + while (self->next && self->next != '\n') { + if (!stream_advance(self)) break; + } + } else { + break; + } + } +} + +static bool stream_is_ident_start(Stream *self) { + return iswalnum(self->next) || self->next == '_' || self->next == '-'; +} + +static void stream_scan_identifier(Stream *stream) { + do { + stream_advance(stream); + } while ( + iswalnum(stream->next) || + stream->next == '_' || + stream->next == '-' || + stream->next == '.' || + stream->next == '?' || + stream->next == '!' + ); +} + +static uint32_t stream_offset(Stream *self) { + return (uint32_t)(self->input - self->start); +} + +/****************** + * CaptureListPool + ******************/ + +static CaptureListPool capture_list_pool_new(void) { + return (CaptureListPool) { + .list = array_new(), + .empty_list = array_new(), + .max_capture_list_count = UINT32_MAX, + .free_capture_list_count = 0, + }; +} + +static void capture_list_pool_reset(CaptureListPool *self) { + for (uint16_t i = 0; i < (uint16_t)self->list.size; i++) { + // This invalid size means that the list is not in use. + array_get(&self->list, i)->size = UINT32_MAX; + } + self->free_capture_list_count = self->list.size; +} + +static void capture_list_pool_delete(CaptureListPool *self) { + for (uint16_t i = 0; i < (uint16_t)self->list.size; i++) { + array_delete(array_get(&self->list, i)); + } + array_delete(&self->list); +} + +static const CaptureList *capture_list_pool_get(const CaptureListPool *self, uint16_t id) { + if (id >= self->list.size) return &self->empty_list; + return array_get(&self->list, id); +} + +static CaptureList *capture_list_pool_get_mut(CaptureListPool *self, uint16_t id) { + ts_assert(id < self->list.size); + return array_get(&self->list, id); +} + +static bool capture_list_pool_is_empty(const CaptureListPool *self) { + // The capture list pool is empty if all allocated lists are in use, and we + // have reached the maximum allowed number of allocated lists. + return self->free_capture_list_count == 0 && self->list.size >= self->max_capture_list_count; +} + +static uint16_t capture_list_pool_acquire(CaptureListPool *self) { + // First see if any already allocated capture list is currently unused. + if (self->free_capture_list_count > 0) { + for (uint16_t i = 0; i < (uint16_t)self->list.size; i++) { + if (array_get(&self->list, i)->size == UINT32_MAX) { + array_clear(array_get(&self->list, i)); + self->free_capture_list_count--; + return i; + } + } + } + + // Otherwise allocate and initialize a new capture list, as long as that + // doesn't put us over the requested maximum. + uint32_t i = self->list.size; + if (i >= self->max_capture_list_count) { + return NONE; + } + CaptureList list; + array_init(&list); + array_push(&self->list, list); + return i; +} + +static void capture_list_pool_release(CaptureListPool *self, uint16_t id) { + if (id >= self->list.size) return; + array_get(&self->list, id)->size = UINT32_MAX; + self->free_capture_list_count++; +} + +/************** + * Quantifiers + **************/ + +static TSQuantifier quantifier_mul( + TSQuantifier left, + TSQuantifier right +) { + switch (left) + { + case TSQuantifierZero: + return TSQuantifierZero; + case TSQuantifierZeroOrOne: + switch (right) { + case TSQuantifierZero: + return TSQuantifierZero; + case TSQuantifierZeroOrOne: + case TSQuantifierOne: + return TSQuantifierZeroOrOne; + case TSQuantifierZeroOrMore: + case TSQuantifierOneOrMore: + return TSQuantifierZeroOrMore; + }; + break; + case TSQuantifierZeroOrMore: + switch (right) { + case TSQuantifierZero: + return TSQuantifierZero; + case TSQuantifierZeroOrOne: + case TSQuantifierZeroOrMore: + case TSQuantifierOne: + case TSQuantifierOneOrMore: + return TSQuantifierZeroOrMore; + }; + break; + case TSQuantifierOne: + return right; + case TSQuantifierOneOrMore: + switch (right) { + case TSQuantifierZero: + return TSQuantifierZero; + case TSQuantifierZeroOrOne: + case TSQuantifierZeroOrMore: + return TSQuantifierZeroOrMore; + case TSQuantifierOne: + case TSQuantifierOneOrMore: + return TSQuantifierOneOrMore; + }; + break; + } + return TSQuantifierZero; // to make compiler happy, but all cases should be covered above! +} + +static TSQuantifier quantifier_join( + TSQuantifier left, + TSQuantifier right +) { + switch (left) + { + case TSQuantifierZero: + switch (right) { + case TSQuantifierZero: + return TSQuantifierZero; + case TSQuantifierZeroOrOne: + case TSQuantifierOne: + return TSQuantifierZeroOrOne; + case TSQuantifierZeroOrMore: + case TSQuantifierOneOrMore: + return TSQuantifierZeroOrMore; + }; + break; + case TSQuantifierZeroOrOne: + switch (right) { + case TSQuantifierZero: + case TSQuantifierZeroOrOne: + case TSQuantifierOne: + return TSQuantifierZeroOrOne; + break; + case TSQuantifierZeroOrMore: + case TSQuantifierOneOrMore: + return TSQuantifierZeroOrMore; + break; + }; + break; + case TSQuantifierZeroOrMore: + return TSQuantifierZeroOrMore; + case TSQuantifierOne: + switch (right) { + case TSQuantifierZero: + case TSQuantifierZeroOrOne: + return TSQuantifierZeroOrOne; + case TSQuantifierZeroOrMore: + return TSQuantifierZeroOrMore; + case TSQuantifierOne: + return TSQuantifierOne; + case TSQuantifierOneOrMore: + return TSQuantifierOneOrMore; + }; + break; + case TSQuantifierOneOrMore: + switch (right) { + case TSQuantifierZero: + case TSQuantifierZeroOrOne: + case TSQuantifierZeroOrMore: + return TSQuantifierZeroOrMore; + case TSQuantifierOne: + case TSQuantifierOneOrMore: + return TSQuantifierOneOrMore; + }; + break; + } + return TSQuantifierZero; // to make compiler happy, but all cases should be covered above! +} + +static TSQuantifier quantifier_add( + TSQuantifier left, + TSQuantifier right +) { + switch (left) + { + case TSQuantifierZero: + return right; + case TSQuantifierZeroOrOne: + switch (right) { + case TSQuantifierZero: + return TSQuantifierZeroOrOne; + case TSQuantifierZeroOrOne: + case TSQuantifierZeroOrMore: + return TSQuantifierZeroOrMore; + case TSQuantifierOne: + case TSQuantifierOneOrMore: + return TSQuantifierOneOrMore; + }; + break; + case TSQuantifierZeroOrMore: + switch (right) { + case TSQuantifierZero: + return TSQuantifierZeroOrMore; + case TSQuantifierZeroOrOne: + case TSQuantifierZeroOrMore: + return TSQuantifierZeroOrMore; + case TSQuantifierOne: + case TSQuantifierOneOrMore: + return TSQuantifierOneOrMore; + }; + break; + case TSQuantifierOne: + switch (right) { + case TSQuantifierZero: + return TSQuantifierOne; + case TSQuantifierZeroOrOne: + case TSQuantifierZeroOrMore: + case TSQuantifierOne: + case TSQuantifierOneOrMore: + return TSQuantifierOneOrMore; + }; + break; + case TSQuantifierOneOrMore: + return TSQuantifierOneOrMore; + } + return TSQuantifierZero; // to make compiler happy, but all cases should be covered above! +} + +// Create new capture quantifiers structure +static CaptureQuantifiers capture_quantifiers_new(void) { + return (CaptureQuantifiers) array_new(); +} + +// Delete capture quantifiers structure +static void capture_quantifiers_delete( + CaptureQuantifiers *self +) { + array_delete(self); +} + +// Clear capture quantifiers structure +static void capture_quantifiers_clear( + CaptureQuantifiers *self +) { + array_clear(self); +} + +// Replace capture quantifiers with the given quantifiers +static void capture_quantifiers_replace( + CaptureQuantifiers *self, + CaptureQuantifiers *quantifiers +) { + array_clear(self); + array_push_all(self, quantifiers); +} + +// Return capture quantifier for the given capture id +static TSQuantifier capture_quantifier_for_id( + const CaptureQuantifiers *self, + uint16_t id +) { + return (self->size <= id) ? TSQuantifierZero : (TSQuantifier) *array_get(self, id); +} + +// Add the given quantifier to the current value for id +static void capture_quantifiers_add_for_id( + CaptureQuantifiers *self, + uint16_t id, + TSQuantifier quantifier +) { + if (self->size <= id) { + array_grow_by(self, id + 1 - self->size); + } + uint8_t *own_quantifier = array_get(self, id); + *own_quantifier = (uint8_t) quantifier_add((TSQuantifier) *own_quantifier, quantifier); +} + +// Point-wise add the given quantifiers to the current values +static void capture_quantifiers_add_all( + CaptureQuantifiers *self, + CaptureQuantifiers *quantifiers +) { + if (self->size < quantifiers->size) { + array_grow_by(self, quantifiers->size - self->size); + } + for (uint16_t id = 0; id < (uint16_t)quantifiers->size; id++) { + uint8_t *quantifier = array_get(quantifiers, id); + uint8_t *own_quantifier = array_get(self, id); + *own_quantifier = (uint8_t) quantifier_add((TSQuantifier) *own_quantifier, (TSQuantifier) *quantifier); + } +} + +// Join the given quantifier with the current values +static void capture_quantifiers_mul( + CaptureQuantifiers *self, + TSQuantifier quantifier +) { + for (uint16_t id = 0; id < (uint16_t)self->size; id++) { + uint8_t *own_quantifier = array_get(self, id); + *own_quantifier = (uint8_t) quantifier_mul((TSQuantifier) *own_quantifier, quantifier); + } +} + +// Point-wise join the quantifiers from a list of alternatives with the current values +static void capture_quantifiers_join_all( + CaptureQuantifiers *self, + CaptureQuantifiers *quantifiers +) { + if (self->size < quantifiers->size) { + array_grow_by(self, quantifiers->size - self->size); + } + for (uint32_t id = 0; id < quantifiers->size; id++) { + uint8_t *quantifier = array_get(quantifiers, id); + uint8_t *own_quantifier = array_get(self, id); + *own_quantifier = (uint8_t) quantifier_join((TSQuantifier) *own_quantifier, (TSQuantifier) *quantifier); + } + for (uint32_t id = quantifiers->size; id < self->size; id++) { + uint8_t *own_quantifier = array_get(self, id); + *own_quantifier = (uint8_t) quantifier_join((TSQuantifier) *own_quantifier, TSQuantifierZero); + } +} + +/************** + * SymbolTable + **************/ + +static SymbolTable symbol_table_new(void) { + return (SymbolTable) { + .characters = array_new(), + .slices = array_new(), + }; +} + +static void symbol_table_delete(SymbolTable *self) { + array_delete(&self->characters); + array_delete(&self->slices); +} + +static int symbol_table_id_for_name( + const SymbolTable *self, + const char *name, + uint32_t length +) { + for (unsigned i = 0; i < self->slices.size; i++) { + Slice slice = *array_get(&self->slices, i); + if ( + slice.length == length && + !strncmp(array_get(&self->characters, slice.offset), name, length) + ) return i; + } + return -1; +} + +static const char *symbol_table_name_for_id( + const SymbolTable *self, + uint16_t id, + uint32_t *length +) { + Slice slice = *(array_get(&self->slices,id)); + *length = slice.length; + return array_get(&self->characters, slice.offset); +} + +static uint16_t symbol_table_insert_name( + SymbolTable *self, + const char *name, + uint32_t length +) { + int id = symbol_table_id_for_name(self, name, length); + if (id >= 0) return (uint16_t)id; + Slice slice = { + .offset = self->characters.size, + .length = length, + }; + array_grow_by(&self->characters, length + 1); + memcpy(array_get(&self->characters, slice.offset), name, length); + *array_get(&self->characters, self->characters.size - 1) = 0; + array_push(&self->slices, slice); + return self->slices.size - 1; +} + +/************ + * QueryStep + ************/ + +static QueryStep query_step__new( + TSSymbol symbol, + uint16_t depth, + bool is_immediate +) { + QueryStep step = { + .symbol = symbol, + .depth = depth, + .field = 0, + .alternative_index = NONE, + .negated_field_list_id = 0, + .contains_captures = false, + .is_last_child = false, + .is_named = false, + .is_pass_through = false, + .is_dead_end = false, + .root_pattern_guaranteed = false, + .is_immediate = is_immediate, + .alternative_is_immediate = false, + }; + for (unsigned i = 0; i < MAX_STEP_CAPTURE_COUNT; i++) { + step.capture_ids[i] = NONE; + } + return step; +} + +static void query_step__add_capture(QueryStep *self, uint16_t capture_id) { + for (unsigned i = 0; i < MAX_STEP_CAPTURE_COUNT; i++) { + if (self->capture_ids[i] == NONE) { + self->capture_ids[i] = capture_id; + break; + } + } +} + +static void query_step__remove_capture(QueryStep *self, uint16_t capture_id) { + for (unsigned i = 0; i < MAX_STEP_CAPTURE_COUNT; i++) { + if (self->capture_ids[i] == capture_id) { + self->capture_ids[i] = NONE; + while (i + 1 < MAX_STEP_CAPTURE_COUNT) { + if (self->capture_ids[i + 1] == NONE) break; + self->capture_ids[i] = self->capture_ids[i + 1]; + self->capture_ids[i + 1] = NONE; + i++; + } + break; + } + } +} + +/********************** + * StatePredecessorMap + **********************/ + +static inline StatePredecessorMap state_predecessor_map_new( + const TSLanguage *language +) { + return (StatePredecessorMap) { + .contents = ts_calloc( + (size_t)language->state_count * (MAX_STATE_PREDECESSOR_COUNT + 1), + sizeof(TSStateId) + ), + }; +} + +static inline void state_predecessor_map_delete(StatePredecessorMap *self) { + ts_free(self->contents); +} + +static inline void state_predecessor_map_add( + StatePredecessorMap *self, + TSStateId state, + TSStateId predecessor +) { + size_t index = (size_t)state * (MAX_STATE_PREDECESSOR_COUNT + 1); + TSStateId *count = &self->contents[index]; + if ( + *count == 0 || + (*count < MAX_STATE_PREDECESSOR_COUNT && self->contents[index + *count] != predecessor) + ) { + (*count)++; + self->contents[index + *count] = predecessor; + } +} + +static inline const TSStateId *state_predecessor_map_get( + const StatePredecessorMap *self, + TSStateId state, + unsigned *count +) { + size_t index = (size_t)state * (MAX_STATE_PREDECESSOR_COUNT + 1); + *count = self->contents[index]; + return &self->contents[index + 1]; +} + +/**************** + * AnalysisState + ****************/ + +static unsigned analysis_state__recursion_depth(const AnalysisState *self) { + unsigned result = 0; + for (unsigned i = 0; i < self->depth; i++) { + TSSymbol symbol = self->stack[i].parent_symbol; + for (unsigned j = 0; j < i; j++) { + if (self->stack[j].parent_symbol == symbol) { + result++; + break; + } + } + } + return result; +} + +static inline int analysis_state__compare( + AnalysisState *const *self, + AnalysisState *const *other +) { + if ((*self)->depth < (*other)->depth) return 1; + for (unsigned i = 0; i < (*self)->depth; i++) { + if (i >= (*other)->depth) return -1; + AnalysisStateEntry s1 = (*self)->stack[i]; + AnalysisStateEntry s2 = (*other)->stack[i]; + if (s1.child_index < s2.child_index) return -1; + if (s1.child_index > s2.child_index) return 1; + if (s1.parent_symbol < s2.parent_symbol) return -1; + if (s1.parent_symbol > s2.parent_symbol) return 1; + if (s1.parse_state < s2.parse_state) return -1; + if (s1.parse_state > s2.parse_state) return 1; + if (s1.field_id < s2.field_id) return -1; + if (s1.field_id > s2.field_id) return 1; + } + if ((*self)->step_index < (*other)->step_index) return -1; + if ((*self)->step_index > (*other)->step_index) return 1; + return 0; +} + +static inline AnalysisStateEntry *analysis_state__top(AnalysisState *self) { + if (self->depth == 0) { + return &self->stack[0]; + } + return &self->stack[self->depth - 1]; +} + +static inline bool analysis_state__has_supertype(AnalysisState *self, TSSymbol symbol) { + for (unsigned i = 0; i < self->depth; i++) { + if (self->stack[i].parent_symbol == symbol) return true; + } + return false; +} + +/****************** + * AnalysisStateSet + ******************/ + +// Obtains an `AnalysisState` instance, either by consuming one from this set's object pool, or by +// cloning one from scratch. +static inline AnalysisState *analysis_state_pool__clone_or_reuse( + AnalysisStateSet *self, + AnalysisState *borrowed_item +) { + AnalysisState *new_item; + if (self->size) { + new_item = array_pop(self); + } else { + new_item = ts_malloc(sizeof(AnalysisState)); + } + *new_item = *borrowed_item; + return new_item; +} + +// Inserts a clone of the passed-in item at the appropriate position to maintain ordering in this +// set. The set does not contain duplicates, so if the item is already present, it will not be +// inserted, and no clone will be made. +// +// The caller retains ownership of the passed-in memory. However, the clone that is created by this +// function will be managed by the state set. +static inline void analysis_state_set__insert_sorted( + AnalysisStateSet *self, + AnalysisStateSet *pool, + AnalysisState *borrowed_item +) { + unsigned index, exists; + array_search_sorted_with(self, analysis_state__compare, &borrowed_item, &index, &exists); + if (!exists) { + AnalysisState *new_item = analysis_state_pool__clone_or_reuse(pool, borrowed_item); + array_insert(self, index, new_item); + } +} + +// Inserts a clone of the passed-in item at the end position of this list. +// +// IMPORTANT: The caller MUST ENSURE that this item is larger (by the comparison function +// `analysis_state__compare`) than largest item already in this set. If items are inserted in the +// wrong order, the set will not function properly for future use. +// +// The caller retains ownership of the passed-in memory. However, the clone that is created by this +// function will be managed by the state set. +static inline void analysis_state_set__push( + AnalysisStateSet *self, + AnalysisStateSet *pool, + AnalysisState *borrowed_item +) { + AnalysisState *new_item = analysis_state_pool__clone_or_reuse(pool, borrowed_item); + array_push(self, new_item); +} + +// Removes all items from this set, returning it to an empty state. +static inline void analysis_state_set__clear(AnalysisStateSet *self, AnalysisStateSet *pool) { + array_push_all(pool, self); + array_clear(self); +} + +// Releases all memory that is managed with this state set, including any items currently present. +// After calling this function, the set is no longer suitable for use. +static inline void analysis_state_set__delete(AnalysisStateSet *self) { + for (unsigned i = 0; i < self->size; i++) { + ts_free(self->contents[i]); + } + array_delete(self); +} + +/**************** + * QueryAnalyzer + ****************/ + +static inline QueryAnalysis query_analysis__new(void) { + return (QueryAnalysis) { + .states = array_new(), + .next_states = array_new(), + .deeper_states = array_new(), + .state_pool = array_new(), + .final_step_indices = array_new(), + .finished_parent_symbols = array_new(), + .did_abort = false, + }; +} + +static inline void query_analysis__delete(QueryAnalysis *self) { + analysis_state_set__delete(&self->states); + analysis_state_set__delete(&self->next_states); + analysis_state_set__delete(&self->deeper_states); + analysis_state_set__delete(&self->state_pool); + array_delete(&self->final_step_indices); + array_delete(&self->finished_parent_symbols); +} + +/*********************** + * AnalysisSubgraphNode + ***********************/ + +static inline int analysis_subgraph_node__compare(const AnalysisSubgraphNode *self, const AnalysisSubgraphNode *other) { + if (self->state < other->state) return -1; + if (self->state > other->state) return 1; + if (self->child_index < other->child_index) return -1; + if (self->child_index > other->child_index) return 1; + if (self->done < other->done) return -1; + if (self->done > other->done) return 1; + if (self->production_id < other->production_id) return -1; + if (self->production_id > other->production_id) return 1; + return 0; +} + +/********* + * Query + *********/ + +// The `pattern_map` contains a mapping from TSSymbol values to indices in the +// `steps` array. For a given syntax node, the `pattern_map` makes it possible +// to quickly find the starting steps of all of the patterns whose root matches +// that node. Each entry has two fields: a `pattern_index`, which identifies one +// of the patterns in the query, and a `step_index`, which indicates the start +// offset of that pattern's steps within the `steps` array. +// +// The entries are sorted by the patterns' root symbols, and lookups use a +// binary search. This ensures that the cost of this initial lookup step +// scales logarithmically with the number of patterns in the query. +// +// This returns `true` if the symbol is present and `false` otherwise. +// If the symbol is not present `*result` is set to the index where the +// symbol should be inserted. +static inline bool ts_query__pattern_map_search( + const TSQuery *self, + TSSymbol needle, + uint32_t *result +) { + uint32_t base_index = self->wildcard_root_pattern_count; + uint32_t size = self->pattern_map.size - base_index; + if (size == 0) { + *result = base_index; + return false; + } + while (size > 1) { + uint32_t half_size = size / 2; + uint32_t mid_index = base_index + half_size; + TSSymbol mid_symbol = array_get(&self->steps, + array_get(&self->pattern_map, mid_index)->step_index + )->symbol; + if (needle > mid_symbol) base_index = mid_index; + size -= half_size; + } + + TSSymbol symbol = array_get(&self->steps, + array_get(&self->pattern_map, base_index)->step_index + )->symbol; + + if (needle > symbol) { + base_index++; + if (base_index < self->pattern_map.size) { + symbol = array_get(&self->steps, + array_get(&self->pattern_map, base_index)->step_index + )->symbol; + } + } + + *result = base_index; + return needle == symbol; +} + +// Insert a new pattern's start index into the pattern map, maintaining +// the pattern map's ordering invariant. +static inline void ts_query__pattern_map_insert( + TSQuery *self, + TSSymbol symbol, + PatternEntry new_entry +) { + uint32_t index; + ts_query__pattern_map_search(self, symbol, &index); + + // Ensure that the entries are sorted not only by symbol, but also + // by pattern_index. This way, states for earlier patterns will be + // initiated first, which allows the ordering of the states array + // to be maintained more efficiently. + while (index < self->pattern_map.size) { + PatternEntry *entry = array_get(&self->pattern_map, index); + if ( + array_get(&self->steps, entry->step_index)->symbol == symbol && + entry->pattern_index < new_entry.pattern_index + ) { + index++; + } else { + break; + } + } + + array_insert(&self->pattern_map, index, new_entry); +} + +// Walk the subgraph for this non-terminal, tracking all of the possible +// sequences of progress within the pattern. +static void ts_query__perform_analysis( + TSQuery *self, + const AnalysisSubgraphArray *subgraphs, + QueryAnalysis *analysis +) { + unsigned recursion_depth_limit = 0; + unsigned prev_final_step_count = 0; + array_clear(&analysis->final_step_indices); + array_clear(&analysis->finished_parent_symbols); + + for (unsigned iteration = 0;; iteration++) { + if (iteration == MAX_ANALYSIS_ITERATION_COUNT) { + analysis->did_abort = true; + break; + } + + #ifdef DEBUG_ANALYZE_QUERY + printf("Iteration: %u. Final step indices:", iteration); + for (unsigned j = 0; j < analysis->final_step_indices.size; j++) { + printf(" %4u", *array_get(&analysis->final_step_indices, j)); + } + printf("\n"); + for (unsigned j = 0; j < analysis->states.size; j++) { + AnalysisState *state = *array_get(&analysis->states, j); + printf(" %3u: step: %u, stack: [", j, state->step_index); + for (unsigned k = 0; k < state->depth; k++) { + printf( + " {%s, child: %u, state: %4u", + self->language->symbol_names[state->stack[k].parent_symbol], + state->stack[k].child_index, + state->stack[k].parse_state + ); + if (state->stack[k].field_id) printf(", field: %s", self->language->field_names[state->stack[k].field_id]); + if (state->stack[k].done) printf(", DONE"); + printf("}"); + } + printf(" ]\n"); + } + #endif + + // If no further progress can be made within the current recursion depth limit, then + // bump the depth limit by one, and continue to process the states the exceeded the + // limit. But only allow this if progress has been made since the last time the depth + // limit was increased. + if (analysis->states.size == 0) { + if ( + analysis->deeper_states.size > 0 && + analysis->final_step_indices.size > prev_final_step_count + ) { + #ifdef DEBUG_ANALYZE_QUERY + printf("Increase recursion depth limit to %u\n", recursion_depth_limit + 1); + #endif + + prev_final_step_count = analysis->final_step_indices.size; + recursion_depth_limit++; + AnalysisStateSet _states = analysis->states; + analysis->states = analysis->deeper_states; + analysis->deeper_states = _states; + continue; + } + + break; + } + + analysis_state_set__clear(&analysis->next_states, &analysis->state_pool); + for (unsigned j = 0; j < analysis->states.size; j++) { + AnalysisState * const state = *array_get(&analysis->states, j); + + // For efficiency, it's important to avoid processing the same analysis state more + // than once. To achieve this, keep the states in order of ascending position within + // their hypothetical syntax trees. In each iteration of this loop, start by advancing + // the states that have made the least progress. Avoid advancing states that have already + // made more progress. + if (analysis->next_states.size > 0) { + int comparison = analysis_state__compare( + &state, + array_back(&analysis->next_states) + ); + if (comparison == 0) { + analysis_state_set__insert_sorted(&analysis->next_states, &analysis->state_pool, state); + continue; + } else if (comparison > 0) { + #ifdef DEBUG_ANALYZE_QUERY + printf("Terminate iteration at state %u\n", j); + #endif + while (j < analysis->states.size) { + analysis_state_set__push( + &analysis->next_states, + &analysis->state_pool, + *array_get(&analysis->states, j) + ); + j++; + } + break; + } + } + + const TSStateId parse_state = analysis_state__top(state)->parse_state; + const TSSymbol parent_symbol = analysis_state__top(state)->parent_symbol; + const TSFieldId parent_field_id = analysis_state__top(state)->field_id; + const unsigned child_index = analysis_state__top(state)->child_index; + const QueryStep * const step = array_get(&self->steps, state->step_index); + + unsigned subgraph_index, exists; + array_search_sorted_by(subgraphs, .symbol, parent_symbol, &subgraph_index, &exists); + if (!exists) continue; + const AnalysisSubgraph *subgraph = array_get(subgraphs, subgraph_index); + + // Follow every possible path in the parse table, but only visit states that + // are part of the subgraph for the current symbol. + LookaheadIterator lookahead_iterator = ts_language_lookaheads(self->language, parse_state); + while (ts_lookahead_iterator__next(&lookahead_iterator)) { + TSSymbol sym = lookahead_iterator.symbol; + + AnalysisSubgraphNode successor = { + .state = parse_state, + .child_index = child_index, + }; + if (lookahead_iterator.action_count) { + const TSParseAction *action = &lookahead_iterator.actions[lookahead_iterator.action_count - 1]; + if (action->type == TSParseActionTypeShift) { + if (!action->shift.extra) { + successor.state = action->shift.state; + successor.child_index++; + } + } else { + continue; + } + } else if (lookahead_iterator.next_state != 0) { + successor.state = lookahead_iterator.next_state; + successor.child_index++; + } else { + continue; + } + + unsigned node_index; + array_search_sorted_with( + &subgraph->nodes, + analysis_subgraph_node__compare, &successor, + &node_index, &exists + ); + while (node_index < subgraph->nodes.size) { + AnalysisSubgraphNode *node = array_get(&subgraph->nodes, node_index); + node_index++; + if (node->state != successor.state || node->child_index != successor.child_index) break; + + // Use the subgraph to determine what alias and field will eventually be applied + // to this child node. + TSSymbol alias = ts_language_alias_at(self->language, node->production_id, child_index); + TSSymbol visible_symbol = alias + ? alias + : self->language->symbol_metadata[sym].visible + ? self->language->public_symbol_map[sym] + : 0; + TSFieldId field_id = parent_field_id; + if (!field_id) { + const TSFieldMapEntry *field_map, *field_map_end; + ts_language_field_map(self->language, node->production_id, &field_map, &field_map_end); + for (; field_map != field_map_end; field_map++) { + if (!field_map->inherited && field_map->child_index == child_index) { + field_id = field_map->field_id; + break; + } + } + } + + // Create a new state that has advanced past this hypothetical subtree. + AnalysisState next_state = *state; + AnalysisStateEntry *next_state_top = analysis_state__top(&next_state); + next_state_top->child_index = successor.child_index; + next_state_top->parse_state = successor.state; + if (node->done) next_state_top->done = true; + + // Determine if this hypothetical child node would match the current step + // of the query pattern. + bool does_match = false; + + // ERROR nodes can appear anywhere, so if the step is + // looking for an ERROR node, consider it potentially matchable. + if (step->symbol == ts_builtin_sym_error) { + does_match = true; + } else if (visible_symbol) { + does_match = true; + if (step->symbol == WILDCARD_SYMBOL) { + if ( + step->is_named && + !self->language->symbol_metadata[visible_symbol].named + ) does_match = false; + } else if (step->symbol != visible_symbol) { + does_match = false; + } + if (step->field && step->field != field_id) { + does_match = false; + } + if ( + step->supertype_symbol && + !analysis_state__has_supertype(state, step->supertype_symbol) + ) does_match = false; + } + + // If this child is hidden, then descend into it and walk through its children. + // If the top entry of the stack is at the end of its rule, then that entry can + // be replaced. Otherwise, push a new entry onto the stack. + else if (sym >= self->language->token_count) { + if (!next_state_top->done) { + if (next_state.depth + 1 >= MAX_ANALYSIS_STATE_DEPTH) { + #ifdef DEBUG_ANALYZE_QUERY + printf("Exceeded depth limit for state %u\n", j); + #endif + + analysis->did_abort = true; + continue; + } + + next_state.depth++; + next_state_top = analysis_state__top(&next_state); + } + + *next_state_top = (AnalysisStateEntry) { + .parse_state = parse_state, + .parent_symbol = sym, + .child_index = 0, + .field_id = field_id, + .done = false, + }; + + if (analysis_state__recursion_depth(&next_state) > recursion_depth_limit) { + analysis_state_set__insert_sorted( + &analysis->deeper_states, + &analysis->state_pool, + &next_state + ); + continue; + } + } + + // Pop from the stack when this state reached the end of its current syntax node. + while (next_state.depth > 0 && next_state_top->done) { + next_state.depth--; + next_state_top = analysis_state__top(&next_state); + } + + // If this hypothetical child did match the current step of the query pattern, + // then advance to the next step at the current depth. This involves skipping + // over any descendant steps of the current child. + const QueryStep *next_step = step; + if (does_match) { + for (;;) { + next_state.step_index++; + next_step = array_get(&self->steps, next_state.step_index); + if ( + next_step->depth == PATTERN_DONE_MARKER || + next_step->depth <= step->depth + ) break; + } + } else if (successor.state == parse_state) { + continue; + } + + for (;;) { + // Skip pass-through states. Although these states have alternatives, they are only + // used to implement repetitions, and query analysis does not need to process + // repetitions in order to determine whether steps are possible and definite. + if (next_step->is_pass_through) { + next_state.step_index++; + next_step++; + continue; + } + + // If the pattern is finished or hypothetical parent node is complete, then + // record that matching can terminate at this step of the pattern. Otherwise, + // add this state to the list of states to process on the next iteration. + if (!next_step->is_dead_end) { + bool did_finish_pattern = array_get(&self->steps, next_state.step_index)->depth != step->depth; + if (did_finish_pattern) { + array_insert_sorted_by(&analysis->finished_parent_symbols, , state->root_symbol); + } else if (next_state.depth == 0) { + array_insert_sorted_by(&analysis->final_step_indices, , next_state.step_index); + } else { + analysis_state_set__insert_sorted(&analysis->next_states, &analysis->state_pool, &next_state); + } + } + + // If the state has advanced to a step with an alternative step, then add another state + // at that alternative step. This process is simpler than the process of actually matching a + // pattern during query execution, because for the purposes of query analysis, there is no + // need to process repetitions. + if ( + does_match && + next_step->alternative_index != NONE && + next_step->alternative_index > next_state.step_index + ) { + next_state.step_index = next_step->alternative_index; + next_step = array_get(&self->steps, next_state.step_index); + } else { + break; + } + } + } + } + } + + AnalysisStateSet _states = analysis->states; + analysis->states = analysis->next_states; + analysis->next_states = _states; + } +} + +static bool ts_query__analyze_patterns(TSQuery *self, unsigned *error_offset) { + Array(uint16_t) non_rooted_pattern_start_steps = array_new(); + for (unsigned i = 0; i < self->pattern_map.size; i++) { + PatternEntry *pattern = array_get(&self->pattern_map, i); + if (!pattern->is_rooted) { + QueryStep *step = array_get(&self->steps, pattern->step_index); + if (step->symbol != WILDCARD_SYMBOL) { + array_push(&non_rooted_pattern_start_steps, i); + } + } + } + + // Walk forward through all of the steps in the query, computing some + // basic information about each step. Mark all of the steps that contain + // captures, and record the indices of all of the steps that have child steps. + Array(uint32_t) parent_step_indices = array_new(); + for (unsigned i = 0; i < self->steps.size; i++) { + QueryStep *step = array_get(&self->steps, i); + if (step->depth == PATTERN_DONE_MARKER) { + step->parent_pattern_guaranteed = true; + step->root_pattern_guaranteed = true; + continue; + } + + bool has_children = false; + bool is_wildcard = step->symbol == WILDCARD_SYMBOL; + step->contains_captures = step->capture_ids[0] != NONE; + for (unsigned j = i + 1; j < self->steps.size; j++) { + QueryStep *next_step = array_get(&self->steps, j); + if ( + next_step->depth == PATTERN_DONE_MARKER || + next_step->depth <= step->depth + ) break; + if (next_step->capture_ids[0] != NONE) { + step->contains_captures = true; + } + if (!is_wildcard) { + next_step->root_pattern_guaranteed = true; + next_step->parent_pattern_guaranteed = true; + } + has_children = true; + } + + if (has_children && !is_wildcard) { + array_push(&parent_step_indices, i); + } + } + + // For every parent symbol in the query, initialize an 'analysis subgraph'. + // This subgraph lists all of the states in the parse table that are directly + // involved in building subtrees for this symbol. + // + // In addition to the parent symbols in the query, construct subgraphs for all + // of the hidden symbols in the grammar, because these might occur within + // one of the parent nodes, such that their children appear to belong to the + // parent. + AnalysisSubgraphArray subgraphs = array_new(); + for (unsigned i = 0; i < parent_step_indices.size; i++) { + uint32_t parent_step_index = *array_get(&parent_step_indices, i); + TSSymbol parent_symbol = array_get(&self->steps, parent_step_index)->symbol; + AnalysisSubgraph subgraph = { .symbol = parent_symbol }; + array_insert_sorted_by(&subgraphs, .symbol, subgraph); + } + for (TSSymbol sym = (uint16_t)self->language->token_count; sym < (uint16_t)self->language->symbol_count; sym++) { + if (!ts_language_symbol_metadata(self->language, sym).visible) { + AnalysisSubgraph subgraph = { .symbol = sym }; + array_insert_sorted_by(&subgraphs, .symbol, subgraph); + } + } + + // Scan the parse table to find the data needed to populate these subgraphs. + // Collect three things during this scan: + // 1) All of the parse states where one of these symbols can start. + // 2) All of the parse states where one of these symbols can end, along + // with information about the node that would be created. + // 3) A list of predecessor states for each state. + StatePredecessorMap predecessor_map = state_predecessor_map_new(self->language); + for (TSStateId state = 1; state < (uint16_t)self->language->state_count; state++) { + unsigned subgraph_index, exists; + LookaheadIterator lookahead_iterator = ts_language_lookaheads(self->language, state); + while (ts_lookahead_iterator__next(&lookahead_iterator)) { + if (lookahead_iterator.action_count) { + for (unsigned i = 0; i < lookahead_iterator.action_count; i++) { + const TSParseAction *action = &lookahead_iterator.actions[i]; + if (action->type == TSParseActionTypeReduce) { + const TSSymbol *aliases, *aliases_end; + ts_language_aliases_for_symbol( + self->language, + action->reduce.symbol, + &aliases, + &aliases_end + ); + for (const TSSymbol *symbol = aliases; symbol < aliases_end; symbol++) { + array_search_sorted_by( + &subgraphs, + .symbol, + *symbol, + &subgraph_index, + &exists + ); + if (exists) { + AnalysisSubgraph *subgraph = array_get(&subgraphs, subgraph_index); + if (subgraph->nodes.size == 0 || array_back(&subgraph->nodes)->state != state) { + array_push(&subgraph->nodes, ((AnalysisSubgraphNode) { + .state = state, + .production_id = action->reduce.production_id, + .child_index = action->reduce.child_count, + .done = true, + })); + } + } + } + } else if (action->type == TSParseActionTypeShift && !action->shift.extra) { + TSStateId next_state = action->shift.state; + state_predecessor_map_add(&predecessor_map, next_state, state); + } + } + } else if (lookahead_iterator.next_state != 0) { + if (lookahead_iterator.next_state != state) { + state_predecessor_map_add(&predecessor_map, lookahead_iterator.next_state, state); + } + if (ts_language_state_is_primary(self->language, state)) { + const TSSymbol *aliases, *aliases_end; + ts_language_aliases_for_symbol( + self->language, + lookahead_iterator.symbol, + &aliases, + &aliases_end + ); + for (const TSSymbol *symbol = aliases; symbol < aliases_end; symbol++) { + array_search_sorted_by( + &subgraphs, + .symbol, + *symbol, + &subgraph_index, + &exists + ); + if (exists) { + AnalysisSubgraph *subgraph = array_get(&subgraphs, subgraph_index); + if ( + subgraph->start_states.size == 0 || + *array_back(&subgraph->start_states) != state + ) + array_push(&subgraph->start_states, state); + } + } + } + } + } + } + + // For each subgraph, compute the preceding states by walking backward + // from the end states using the predecessor map. + Array(AnalysisSubgraphNode) next_nodes = array_new(); + for (unsigned i = 0; i < subgraphs.size; i++) { + AnalysisSubgraph *subgraph = array_get(&subgraphs, i); + if (subgraph->nodes.size == 0) { + array_delete(&subgraph->start_states); + array_erase(&subgraphs, i); + i--; + continue; + } + array_assign(&next_nodes, &subgraph->nodes); + while (next_nodes.size > 0) { + AnalysisSubgraphNode node = array_pop(&next_nodes); + if (node.child_index > 1) { + unsigned predecessor_count; + const TSStateId *predecessors = state_predecessor_map_get( + &predecessor_map, + node.state, + &predecessor_count + ); + for (unsigned j = 0; j < predecessor_count; j++) { + AnalysisSubgraphNode predecessor_node = { + .state = predecessors[j], + .child_index = node.child_index - 1, + .production_id = node.production_id, + .done = false, + }; + unsigned index, exists; + array_search_sorted_with( + &subgraph->nodes, analysis_subgraph_node__compare, &predecessor_node, + &index, &exists + ); + if (!exists) { + array_insert(&subgraph->nodes, index, predecessor_node); + array_push(&next_nodes, predecessor_node); + } + } + } + } + } + + #ifdef DEBUG_ANALYZE_QUERY + printf("\nSubgraphs:\n"); + for (unsigned i = 0; i < subgraphs.size; i++) { + AnalysisSubgraph *subgraph = array_get(&subgraphs, i); + printf(" %u, %s:\n", subgraph->symbol, ts_language_symbol_name(self->language, subgraph->symbol)); + for (unsigned j = 0; j < subgraph->start_states.size; j++) { + printf( + " {state: %u}\n", + *array_get(&subgraph->start_states, j) + ); + } + for (unsigned j = 0; j < subgraph->nodes.size; j++) { + AnalysisSubgraphNode *node = array_get(&subgraph->nodes, j); + printf( + " {state: %u, child_index: %u, production_id: %u, done: %d}\n", + node->state, node->child_index, node->production_id, node->done + ); + } + printf("\n"); + } + #endif + + // For each non-terminal pattern, determine if the pattern can successfully match, + // and identify all of the possible children within the pattern where matching could fail. + bool all_patterns_are_valid = true; + QueryAnalysis analysis = query_analysis__new(); + for (unsigned i = 0; i < parent_step_indices.size; i++) { + uint16_t parent_step_index = *array_get(&parent_step_indices, i); + uint16_t parent_depth = array_get(&self->steps, parent_step_index)->depth; + TSSymbol parent_symbol = array_get(&self->steps, parent_step_index)->symbol; + if (parent_symbol == ts_builtin_sym_error) continue; + + // Find the subgraph that corresponds to this pattern's root symbol. If the pattern's + // root symbol is a terminal, then return an error. + unsigned subgraph_index, exists; + array_search_sorted_by(&subgraphs, .symbol, parent_symbol, &subgraph_index, &exists); + if (!exists) { + unsigned first_child_step_index = parent_step_index + 1; + uint32_t j, child_exists; + array_search_sorted_by(&self->step_offsets, .step_index, first_child_step_index, &j, &child_exists); + ts_assert(child_exists); + *error_offset = array_get(&self->step_offsets, j)->byte_offset; + all_patterns_are_valid = false; + break; + } + + // Initialize an analysis state at every parse state in the table where + // this parent symbol can occur. + AnalysisSubgraph *subgraph = array_get(&subgraphs, subgraph_index); + analysis_state_set__clear(&analysis.states, &analysis.state_pool); + analysis_state_set__clear(&analysis.deeper_states, &analysis.state_pool); + for (unsigned j = 0; j < subgraph->start_states.size; j++) { + TSStateId parse_state = *array_get(&subgraph->start_states, j); + analysis_state_set__push(&analysis.states, &analysis.state_pool, &((AnalysisState) { + .step_index = parent_step_index + 1, + .stack = { + [0] = { + .parse_state = parse_state, + .parent_symbol = parent_symbol, + .child_index = 0, + .field_id = 0, + .done = false, + }, + }, + .depth = 1, + .root_symbol = parent_symbol, + })); + } + + #ifdef DEBUG_ANALYZE_QUERY + printf( + "\nWalk states for %s:\n", + ts_language_symbol_name(self->language, (*array_get(&analysis.states, 0))->stack[0].parent_symbol) + ); + #endif + + analysis.did_abort = false; + ts_query__perform_analysis(self, &subgraphs, &analysis); + + // If this pattern could not be fully analyzed, then every step should + // be considered fallible. + if (analysis.did_abort) { + for (unsigned j = parent_step_index + 1; j < self->steps.size; j++) { + QueryStep *step = array_get(&self->steps, j); + if ( + step->depth <= parent_depth || + step->depth == PATTERN_DONE_MARKER + ) break; + if (!step->is_dead_end) { + step->parent_pattern_guaranteed = false; + step->root_pattern_guaranteed = false; + } + } + continue; + } + + // If this pattern cannot match, store the pattern index so that it can be + // returned to the caller. + if (analysis.finished_parent_symbols.size == 0) { + ts_assert(analysis.final_step_indices.size > 0); + uint16_t impossible_step_index = *array_back(&analysis.final_step_indices); + uint32_t j, impossible_exists; + array_search_sorted_by(&self->step_offsets, .step_index, impossible_step_index, &j, &impossible_exists); + if (j >= self->step_offsets.size) j = self->step_offsets.size - 1; + *error_offset = array_get(&self->step_offsets, j)->byte_offset; + all_patterns_are_valid = false; + break; + } + + // Mark as fallible any step where a match terminated. + // Later, this property will be propagated to all of the step's predecessors. + for (unsigned j = 0; j < analysis.final_step_indices.size; j++) { + uint32_t final_step_index = *array_get(&analysis.final_step_indices, j); + QueryStep *step = array_get(&self->steps, final_step_index); + if ( + step->depth != PATTERN_DONE_MARKER && + step->depth > parent_depth && + !step->is_dead_end + ) { + step->parent_pattern_guaranteed = false; + step->root_pattern_guaranteed = false; + } + } + } + + // Mark as indefinite any step with captures that are used in predicates. + Array(uint16_t) predicate_capture_ids = array_new(); + for (unsigned i = 0; i < self->patterns.size; i++) { + QueryPattern *pattern = array_get(&self->patterns, i); + + // Gather all of the captures that are used in predicates for this pattern. + array_clear(&predicate_capture_ids); + for ( + unsigned start = pattern->predicate_steps.offset, + end = start + pattern->predicate_steps.length, + j = start; j < end; j++ + ) { + TSQueryPredicateStep *step = array_get(&self->predicate_steps, j); + if (step->type == TSQueryPredicateStepTypeCapture) { + uint16_t value_id = step->value_id; + array_insert_sorted_by(&predicate_capture_ids, , value_id); + } + } + + // Find all of the steps that have these captures. + for ( + unsigned start = pattern->steps.offset, + end = start + pattern->steps.length, + j = start; j < end; j++ + ) { + QueryStep *step = array_get(&self->steps, j); + for (unsigned k = 0; k < MAX_STEP_CAPTURE_COUNT; k++) { + uint16_t capture_id = step->capture_ids[k]; + if (capture_id == NONE) break; + unsigned index, exists; + array_search_sorted_by(&predicate_capture_ids, , capture_id, &index, &exists); + if (exists) { + step->root_pattern_guaranteed = false; + break; + } + } + } + } + + // Propagate fallibility. If a pattern is fallible at a given step, then it is + // fallible at all of its preceding steps. + bool done = self->steps.size == 0; + while (!done) { + done = true; + for (unsigned i = self->steps.size - 1; i > 0; i--) { + QueryStep *step = array_get(&self->steps, i); + if (step->depth == PATTERN_DONE_MARKER) continue; + + // Determine if this step is definite or has definite alternatives. + bool parent_pattern_guaranteed = false; + for (;;) { + if (step->root_pattern_guaranteed) { + parent_pattern_guaranteed = true; + break; + } + if (step->alternative_index == NONE || step->alternative_index < i) { + break; + } + step = array_get(&self->steps, step->alternative_index); + } + + // If not, mark its predecessor as indefinite. + if (!parent_pattern_guaranteed) { + QueryStep *prev_step = array_get(&self->steps, i - 1); + if ( + !prev_step->is_dead_end && + prev_step->depth != PATTERN_DONE_MARKER && + prev_step->root_pattern_guaranteed + ) { + prev_step->root_pattern_guaranteed = false; + done = false; + } + } + } + } + + #ifdef DEBUG_ANALYZE_QUERY + printf("Steps:\n"); + for (unsigned i = 0; i < self->steps.size; i++) { + QueryStep *step = array_get(&self->steps, i); + if (step->depth == PATTERN_DONE_MARKER) { + printf(" %u: DONE\n", i); + } else { + printf( + " %u: {symbol: %s, field: %s, depth: %u, parent_pattern_guaranteed: %d, root_pattern_guaranteed: %d}\n", + i, + (step->symbol == WILDCARD_SYMBOL) + ? "ANY" + : ts_language_symbol_name(self->language, step->symbol), + (step->field ? ts_language_field_name_for_id(self->language, step->field) : "-"), + step->depth, + step->parent_pattern_guaranteed, + step->root_pattern_guaranteed + ); + } + } + #endif + + // Determine which repetition symbols in this language have the possibility + // of matching non-rooted patterns in this query. These repetition symbols + // prevent certain optimizations with range restrictions. + analysis.did_abort = false; + for (uint32_t i = 0; i < non_rooted_pattern_start_steps.size; i++) { + uint16_t pattern_entry_index = *array_get(&non_rooted_pattern_start_steps, i); + PatternEntry *pattern_entry = array_get(&self->pattern_map, pattern_entry_index); + + analysis_state_set__clear(&analysis.states, &analysis.state_pool); + analysis_state_set__clear(&analysis.deeper_states, &analysis.state_pool); + for (unsigned j = 0; j < subgraphs.size; j++) { + AnalysisSubgraph *subgraph = array_get(&subgraphs, j); + TSSymbolMetadata metadata = ts_language_symbol_metadata(self->language, subgraph->symbol); + if (metadata.visible || metadata.named) continue; + + for (uint32_t k = 0; k < subgraph->start_states.size; k++) { + TSStateId parse_state = *array_get(&subgraph->start_states, k); + analysis_state_set__push(&analysis.states, &analysis.state_pool, &((AnalysisState) { + .step_index = pattern_entry->step_index, + .stack = { + [0] = { + .parse_state = parse_state, + .parent_symbol = subgraph->symbol, + .child_index = 0, + .field_id = 0, + .done = false, + }, + }, + .root_symbol = subgraph->symbol, + .depth = 1, + })); + } + } + + #ifdef DEBUG_ANALYZE_QUERY + printf("\nWalk states for rootless pattern step %u:\n", pattern_entry->step_index); + #endif + + ts_query__perform_analysis( + self, + &subgraphs, + &analysis + ); + + if (analysis.finished_parent_symbols.size > 0) { + array_get(&self->patterns, pattern_entry->pattern_index)->is_non_local = true; + } + + for (unsigned k = 0; k < analysis.finished_parent_symbols.size; k++) { + TSSymbol symbol = *array_get(&analysis.finished_parent_symbols, k); + array_insert_sorted_by(&self->repeat_symbols_with_rootless_patterns, , symbol); + } + } + + #ifdef DEBUG_ANALYZE_QUERY + if (self->repeat_symbols_with_rootless_patterns.size > 0) { + printf("\nRepetition symbols with rootless patterns:\n"); + printf("aborted analysis: %d\n", analysis.did_abort); + for (unsigned i = 0; i < self->repeat_symbols_with_rootless_patterns.size; i++) { + TSSymbol symbol = *array_get(&self->repeat_symbols_with_rootless_patterns, i); + printf(" %u, %s\n", symbol, ts_language_symbol_name(self->language, symbol)); + } + printf("\n"); + } + #endif + + // Cleanup + for (unsigned i = 0; i < subgraphs.size; i++) { + array_delete(&array_get(&subgraphs, i)->start_states); + array_delete(&array_get(&subgraphs, i)->nodes); + } + array_delete(&subgraphs); + query_analysis__delete(&analysis); + array_delete(&next_nodes); + array_delete(&non_rooted_pattern_start_steps); + array_delete(&parent_step_indices); + array_delete(&predicate_capture_ids); + state_predecessor_map_delete(&predecessor_map); + + return all_patterns_are_valid; +} + +static void ts_query__add_negated_fields( + TSQuery *self, + uint16_t step_index, + TSFieldId *field_ids, + uint16_t field_count +) { + QueryStep *step = array_get(&self->steps, step_index); + + // The negated field array stores a list of field lists, separated by zeros. + // Try to find the start index of an existing list that matches this new list. + bool failed_match = false; + unsigned match_count = 0; + unsigned start_i = 0; + for (unsigned i = 0; i < self->negated_fields.size; i++) { + TSFieldId existing_field_id = *array_get(&self->negated_fields, i); + + // At each zero value, terminate the match attempt. If we've exactly + // matched the new field list, then reuse this index. Otherwise, + // start over the matching process. + if (existing_field_id == 0) { + if (match_count == field_count) { + step->negated_field_list_id = start_i; + return; + } else { + start_i = i + 1; + match_count = 0; + failed_match = false; + } + } + + // If the existing list matches our new list so far, then advance + // to the next element of the new list. + else if ( + match_count < field_count && + existing_field_id == field_ids[match_count] && + !failed_match + ) { + match_count++; + } + + // Otherwise, this existing list has failed to match. + else { + match_count = 0; + failed_match = true; + } + } + + step->negated_field_list_id = self->negated_fields.size; + array_extend(&self->negated_fields, field_count, field_ids); + array_push(&self->negated_fields, 0); +} + +static TSQueryError ts_query__parse_string_literal( + TSQuery *self, + Stream *stream +) { + const char *string_start = stream->input; + if (stream->next != '"') return TSQueryErrorSyntax; + stream_advance(stream); + const char *prev_position = stream->input; + + bool is_escaped = false; + array_clear(&self->string_buffer); + for (;;) { + if (is_escaped) { + is_escaped = false; + switch (stream->next) { + case 'n': + array_push(&self->string_buffer, '\n'); + break; + case 'r': + array_push(&self->string_buffer, '\r'); + break; + case 't': + array_push(&self->string_buffer, '\t'); + break; + case '0': + array_push(&self->string_buffer, '\0'); + break; + default: + array_extend(&self->string_buffer, stream->next_size, stream->input); + break; + } + prev_position = stream->input + stream->next_size; + } else { + if (stream->next == '\\') { + array_extend(&self->string_buffer, (uint32_t)(stream->input - prev_position), prev_position); + prev_position = stream->input + 1; + is_escaped = true; + } else if (stream->next == '"') { + array_extend(&self->string_buffer, (uint32_t)(stream->input - prev_position), prev_position); + stream_advance(stream); + return TSQueryErrorNone; + } else if (stream->next == '\n') { + stream_reset(stream, string_start); + return TSQueryErrorSyntax; + } + } + if (!stream_advance(stream)) { + stream_reset(stream, string_start); + return TSQueryErrorSyntax; + } + } +} + +// Parse a single predicate associated with a pattern, adding it to the +// query's internal `predicate_steps` array. Predicates are arbitrary +// S-expressions associated with a pattern which are meant to be handled at +// a higher level of abstraction, such as the Rust/JavaScript bindings. They +// can contain '@'-prefixed capture names, double-quoted strings, and bare +// symbols, which also represent strings. +static TSQueryError ts_query__parse_predicate( + TSQuery *self, + Stream *stream +) { + if (!stream_is_ident_start(stream)) return TSQueryErrorSyntax; + const char *predicate_name = stream->input; + stream_scan_identifier(stream); + uint32_t length = (uint32_t)(stream->input - predicate_name); + uint16_t id = symbol_table_insert_name( + &self->predicate_values, + predicate_name, + length + ); + array_push(&self->predicate_steps, ((TSQueryPredicateStep) { + .type = TSQueryPredicateStepTypeString, + .value_id = id, + })); + stream_skip_whitespace(stream); + + for (;;) { + if (stream->next == ')') { + stream_advance(stream); + stream_skip_whitespace(stream); + array_push(&self->predicate_steps, ((TSQueryPredicateStep) { + .type = TSQueryPredicateStepTypeDone, + .value_id = 0, + })); + break; + } + + // Parse an '@'-prefixed capture name + else if (stream->next == '@') { + stream_advance(stream); + + // Parse the capture name + if (!stream_is_ident_start(stream)) return TSQueryErrorSyntax; + const char *capture_name = stream->input; + stream_scan_identifier(stream); + uint32_t capture_length = (uint32_t)(stream->input - capture_name); + + // Add the capture id to the first step of the pattern + int capture_id = symbol_table_id_for_name( + &self->captures, + capture_name, + capture_length + ); + if (capture_id == -1) { + stream_reset(stream, capture_name); + return TSQueryErrorCapture; + } + + array_push(&self->predicate_steps, ((TSQueryPredicateStep) { + .type = TSQueryPredicateStepTypeCapture, + .value_id = capture_id, + })); + } + + // Parse a string literal + else if (stream->next == '"') { + TSQueryError e = ts_query__parse_string_literal(self, stream); + if (e) return e; + uint16_t query_id = symbol_table_insert_name( + &self->predicate_values, + self->string_buffer.contents, + self->string_buffer.size + ); + array_push(&self->predicate_steps, ((TSQueryPredicateStep) { + .type = TSQueryPredicateStepTypeString, + .value_id = query_id, + })); + } + + // Parse a bare symbol + else if (stream_is_ident_start(stream)) { + const char *symbol_start = stream->input; + stream_scan_identifier(stream); + uint32_t symbol_length = (uint32_t)(stream->input - symbol_start); + uint16_t query_id = symbol_table_insert_name( + &self->predicate_values, + symbol_start, + symbol_length + ); + array_push(&self->predicate_steps, ((TSQueryPredicateStep) { + .type = TSQueryPredicateStepTypeString, + .value_id = query_id, + })); + } + + else { + return TSQueryErrorSyntax; + } + + stream_skip_whitespace(stream); + } + + return 0; +} + +// Read one S-expression pattern from the stream, and incorporate it into +// the query's internal state machine representation. For nested patterns, +// this function calls itself recursively. +// +// The caller is responsible for passing in a dedicated CaptureQuantifiers. +// These should not be shared between different calls to ts_query__parse_pattern! +static TSQueryError ts_query__parse_pattern( + TSQuery *self, + Stream *stream, + uint32_t depth, + bool is_immediate, + CaptureQuantifiers *capture_quantifiers +) { + if (stream->next == 0) return TSQueryErrorSyntax; + if (stream->next == ')' || stream->next == ']') return PARENT_DONE; + + const uint32_t starting_step_index = self->steps.size; + + // Store the byte offset of each step in the query. + if ( + self->step_offsets.size == 0 || + array_back(&self->step_offsets)->step_index != starting_step_index + ) { + array_push(&self->step_offsets, ((StepOffset) { + .step_index = starting_step_index, + .byte_offset = stream_offset(stream), + })); + } + + // An open bracket is the start of an alternation. + if (stream->next == '[') { + stream_advance(stream); + stream_skip_whitespace(stream); + + // Parse each branch, and add a placeholder step in between the branches. + Array(uint32_t) branch_step_indices = array_new(); + CaptureQuantifiers branch_capture_quantifiers = capture_quantifiers_new(); + for (;;) { + uint32_t start_index = self->steps.size; + TSQueryError e = ts_query__parse_pattern( + self, + stream, + depth, + is_immediate, + &branch_capture_quantifiers + ); + + if (e == PARENT_DONE) { + if (stream->next == ']' && branch_step_indices.size > 0) { + stream_advance(stream); + break; + } + e = TSQueryErrorSyntax; + } + if (e) { + capture_quantifiers_delete(&branch_capture_quantifiers); + array_delete(&branch_step_indices); + return e; + } + + if (start_index == starting_step_index) { + capture_quantifiers_replace(capture_quantifiers, &branch_capture_quantifiers); + } else { + capture_quantifiers_join_all(capture_quantifiers, &branch_capture_quantifiers); + } + + array_push(&branch_step_indices, start_index); + array_push(&self->steps, query_step__new(0, depth, false)); + capture_quantifiers_clear(&branch_capture_quantifiers); + } + (void)array_pop(&self->steps); + + // For all of the branches except for the last one, add the subsequent branch as an + // alternative, and link the end of the branch to the current end of the steps. + for (unsigned i = 0; i < branch_step_indices.size - 1; i++) { + uint32_t step_index = *array_get(&branch_step_indices, i); + uint32_t next_step_index = *array_get(&branch_step_indices, i + 1); + QueryStep *start_step = array_get(&self->steps, step_index); + QueryStep *end_step = array_get(&self->steps, next_step_index - 1); + start_step->alternative_index = next_step_index; + end_step->alternative_index = self->steps.size; + end_step->is_dead_end = true; + } + + capture_quantifiers_delete(&branch_capture_quantifiers); + array_delete(&branch_step_indices); + } + + // An open parenthesis can be the start of three possible constructs: + // * A grouped sequence + // * A predicate + // * A named node + else if (stream->next == '(') { + stream_advance(stream); + stream_skip_whitespace(stream); + + // If this parenthesis is followed by a node, then it represents a grouped sequence. + if (stream->next == '(' || stream->next == '"' || stream->next == '[') { + bool child_is_immediate = is_immediate; + CaptureQuantifiers child_capture_quantifiers = capture_quantifiers_new(); + for (;;) { + if (stream->next == '.') { + child_is_immediate = true; + stream_advance(stream); + stream_skip_whitespace(stream); + } + TSQueryError e = ts_query__parse_pattern( + self, + stream, + depth, + child_is_immediate, + &child_capture_quantifiers + ); + if (e == PARENT_DONE) { + if (stream->next == ')') { + stream_advance(stream); + break; + } + e = TSQueryErrorSyntax; + } + if (e) { + capture_quantifiers_delete(&child_capture_quantifiers); + return e; + } + + capture_quantifiers_add_all(capture_quantifiers, &child_capture_quantifiers); + capture_quantifiers_clear(&child_capture_quantifiers); + child_is_immediate = false; + } + + capture_quantifiers_delete(&child_capture_quantifiers); + } + + // A dot/pound character indicates the start of a predicate. + else if (stream->next == '.' || stream->next == '#') { + stream_advance(stream); + return ts_query__parse_predicate(self, stream); + } + + // Otherwise, this parenthesis is the start of a named node. + else { + TSSymbol symbol; + bool is_missing = false; + const char *node_name = stream->input; + + // Parse a normal node name + if (stream_is_ident_start(stream)) { + stream_scan_identifier(stream); + uint32_t length = (uint32_t)(stream->input - node_name); + + // Parse the wildcard symbol + if (length == 1 && node_name[0] == '_') { + symbol = WILDCARD_SYMBOL; + } else if (!strncmp(node_name, "MISSING", length)) { + is_missing = true; + stream_skip_whitespace(stream); + + if (stream_is_ident_start(stream)) { + const char *missing_node_name = stream->input; + stream_scan_identifier(stream); + uint32_t missing_node_length = (uint32_t)(stream->input - missing_node_name); + symbol = ts_language_symbol_for_name( + self->language, + missing_node_name, + missing_node_length, + true + ); + if (!symbol) { + stream_reset(stream, missing_node_name); + return TSQueryErrorNodeType; + } + } + + else if (stream->next == '"') { + const char *string_start = stream->input; + TSQueryError e = ts_query__parse_string_literal(self, stream); + if (e) return e; + + symbol = ts_language_symbol_for_name( + self->language, + self->string_buffer.contents, + self->string_buffer.size, + false + ); + if (!symbol) { + stream_reset(stream, string_start + 1); + return TSQueryErrorNodeType; + } + } + + else if (stream->next == ')') { + symbol = WILDCARD_SYMBOL; + } + + else { + stream_reset(stream, stream->input); + return TSQueryErrorSyntax; + } + } + + else { + symbol = ts_language_symbol_for_name( + self->language, + node_name, + length, + true + ); + if (!symbol) { + stream_reset(stream, node_name); + return TSQueryErrorNodeType; + } + } + } else { + return TSQueryErrorSyntax; + } + + // Add a step for the node. + array_push(&self->steps, query_step__new(symbol, depth, is_immediate)); + QueryStep *step = array_back(&self->steps); + if (ts_language_symbol_metadata(self->language, symbol).supertype) { + step->supertype_symbol = step->symbol; + step->symbol = WILDCARD_SYMBOL; + } + if (is_missing) { + step->is_missing = true; + } + if (symbol == WILDCARD_SYMBOL) { + step->is_named = true; + } + + stream_skip_whitespace(stream); + + if (stream->next == '/') { + if (!step->supertype_symbol) { + stream_reset(stream, node_name - 1); // reset to the start of the node + return TSQueryErrorStructure; + } + + stream_advance(stream); + if (!stream_is_ident_start(stream)) { + return TSQueryErrorSyntax; + } + + const char *subtype_node_name = stream->input; + stream_scan_identifier(stream); + uint32_t length = (uint32_t)(stream->input - subtype_node_name); + + step->symbol = ts_language_symbol_for_name( + self->language, + subtype_node_name, + length, + true + ); + if (!step->symbol) { + stream_reset(stream, subtype_node_name); + return TSQueryErrorNodeType; + } + + // Get all the possible subtypes for the given supertype, + // and check if the given subtype is valid. + if (self->language->abi_version >= LANGUAGE_VERSION_WITH_RESERVED_WORDS) { + uint32_t subtype_length; + const TSSymbol *subtypes = ts_language_subtypes( + self->language, + step->supertype_symbol, + &subtype_length + ); + + bool subtype_is_valid = false; + for (uint32_t i = 0; i < subtype_length; i++) { + if (subtypes[i] == step->symbol) { + subtype_is_valid = true; + break; + } + } + + // This subtype is not valid for the given supertype. + if (!subtype_is_valid) { + stream_reset(stream, node_name - 1); // reset to the start of the node + return TSQueryErrorStructure; + } + } + + stream_skip_whitespace(stream); + } + + // Parse the child patterns + bool child_is_immediate = false; + uint16_t last_child_step_index = 0; + uint16_t negated_field_count = 0; + TSFieldId negated_field_ids[MAX_NEGATED_FIELD_COUNT]; + CaptureQuantifiers child_capture_quantifiers = capture_quantifiers_new(); + for (;;) { + // Parse a negated field assertion + if (stream->next == '!') { + stream_advance(stream); + stream_skip_whitespace(stream); + if (!stream_is_ident_start(stream)) { + capture_quantifiers_delete(&child_capture_quantifiers); + return TSQueryErrorSyntax; + } + const char *field_name = stream->input; + stream_scan_identifier(stream); + uint32_t length = (uint32_t)(stream->input - field_name); + stream_skip_whitespace(stream); + + TSFieldId field_id = ts_language_field_id_for_name( + self->language, + field_name, + length + ); + if (!field_id) { + stream->input = field_name; + capture_quantifiers_delete(&child_capture_quantifiers); + return TSQueryErrorField; + } + + // Keep the field ids sorted. + if (negated_field_count < MAX_NEGATED_FIELD_COUNT) { + negated_field_ids[negated_field_count] = field_id; + negated_field_count++; + } + + continue; + } + + // Parse a sibling anchor + if (stream->next == '.') { + child_is_immediate = true; + stream_advance(stream); + stream_skip_whitespace(stream); + } + + uint16_t step_index = self->steps.size; + TSQueryError e = ts_query__parse_pattern( + self, + stream, + depth + 1, + child_is_immediate, + &child_capture_quantifiers + ); + // In the event we only parsed a predicate, meaning no new steps were added, + // then subtract one so we're not indexing past the end of the array + if (step_index == self->steps.size) step_index--; + if (e == PARENT_DONE) { + if (stream->next == ')') { + if (child_is_immediate) { + if (last_child_step_index == 0) { + capture_quantifiers_delete(&child_capture_quantifiers); + return TSQueryErrorSyntax; + } + // Mark this step *and* its alternatives as the last child of the parent. + QueryStep *last_child_step = array_get(&self->steps, last_child_step_index); + last_child_step->is_last_child = true; + if ( + last_child_step->alternative_index != NONE && + last_child_step->alternative_index < self->steps.size + ) { + QueryStep *alternative_step = array_get(&self->steps, last_child_step->alternative_index); + alternative_step->is_last_child = true; + while ( + alternative_step->alternative_index != NONE && + alternative_step->alternative_index < self->steps.size + ) { + alternative_step = array_get(&self->steps, alternative_step->alternative_index); + alternative_step->is_last_child = true; + } + } + } + + if (negated_field_count) { + ts_query__add_negated_fields( + self, + starting_step_index, + negated_field_ids, + negated_field_count + ); + } + + stream_advance(stream); + break; + } + e = TSQueryErrorSyntax; + } + if (e) { + capture_quantifiers_delete(&child_capture_quantifiers); + return e; + } + + capture_quantifiers_add_all(capture_quantifiers, &child_capture_quantifiers); + + last_child_step_index = step_index; + child_is_immediate = false; + capture_quantifiers_clear(&child_capture_quantifiers); + } + capture_quantifiers_delete(&child_capture_quantifiers); + } + } + + // Parse a wildcard pattern + else if (stream->next == '_') { + stream_advance(stream); + stream_skip_whitespace(stream); + + // Add a step that matches any kind of node + array_push(&self->steps, query_step__new(WILDCARD_SYMBOL, depth, is_immediate)); + } + + // Parse a double-quoted anonymous leaf node expression + else if (stream->next == '"') { + const char *string_start = stream->input; + TSQueryError e = ts_query__parse_string_literal(self, stream); + if (e) return e; + + // Add a step for the node + TSSymbol symbol = ts_language_symbol_for_name( + self->language, + self->string_buffer.contents, + self->string_buffer.size, + false + ); + if (!symbol) { + stream_reset(stream, string_start + 1); + return TSQueryErrorNodeType; + } + array_push(&self->steps, query_step__new(symbol, depth, is_immediate)); + } + + // Parse a field-prefixed pattern + else if (stream_is_ident_start(stream)) { + // Parse the field name + const char *field_name = stream->input; + stream_scan_identifier(stream); + uint32_t length = (uint32_t)(stream->input - field_name); + stream_skip_whitespace(stream); + + if (stream->next != ':') { + stream_reset(stream, field_name); + return TSQueryErrorSyntax; + } + stream_advance(stream); + stream_skip_whitespace(stream); + + // Parse the pattern + CaptureQuantifiers field_capture_quantifiers = capture_quantifiers_new(); + TSQueryError e = ts_query__parse_pattern( + self, + stream, + depth, + is_immediate, + &field_capture_quantifiers + ); + if (e) { + capture_quantifiers_delete(&field_capture_quantifiers); + if (e == PARENT_DONE) e = TSQueryErrorSyntax; + return e; + } + + // Add the field name to the first step of the pattern + TSFieldId field_id = ts_language_field_id_for_name( + self->language, + field_name, + length + ); + if (!field_id) { + stream->input = field_name; + return TSQueryErrorField; + } + + uint32_t step_index = starting_step_index; + QueryStep *step = array_get(&self->steps, step_index); + for (;;) { + step->field = field_id; + if ( + step->alternative_index != NONE && + step->alternative_index > step_index && + step->alternative_index < self->steps.size + ) { + step_index = step->alternative_index; + step = array_get(&self->steps, step_index); + } else { + break; + } + } + + capture_quantifiers_add_all(capture_quantifiers, &field_capture_quantifiers); + capture_quantifiers_delete(&field_capture_quantifiers); + } + + else { + return TSQueryErrorSyntax; + } + + stream_skip_whitespace(stream); + + // Parse suffixes modifiers for this pattern + TSQuantifier quantifier = TSQuantifierOne; + for (;;) { + // Parse the one-or-more operator. + if (stream->next == '+') { + quantifier = quantifier_join(TSQuantifierOneOrMore, quantifier); + + stream_advance(stream); + stream_skip_whitespace(stream); + + QueryStep repeat_step = query_step__new(WILDCARD_SYMBOL, depth, false); + repeat_step.alternative_index = starting_step_index; + repeat_step.is_pass_through = true; + repeat_step.alternative_is_immediate = true; + array_push(&self->steps, repeat_step); + } + + // Parse the zero-or-more repetition operator. + else if (stream->next == '*') { + quantifier = quantifier_join(TSQuantifierZeroOrMore, quantifier); + + stream_advance(stream); + stream_skip_whitespace(stream); + + QueryStep repeat_step = query_step__new(WILDCARD_SYMBOL, depth, false); + repeat_step.alternative_index = starting_step_index; + repeat_step.is_pass_through = true; + repeat_step.alternative_is_immediate = true; + array_push(&self->steps, repeat_step); + + // Stop when `step->alternative_index` is `NONE` or it points to + // `repeat_step` or beyond. Note that having just been pushed, + // `repeat_step` occupies slot `self->steps.size - 1`. + QueryStep *step = array_get(&self->steps, starting_step_index); + while (step->alternative_index != NONE && step->alternative_index < self->steps.size - 1) { + step = array_get(&self->steps, step->alternative_index); + } + step->alternative_index = self->steps.size; + } + + // Parse the optional operator. + else if (stream->next == '?') { + quantifier = quantifier_join(TSQuantifierZeroOrOne, quantifier); + + stream_advance(stream); + stream_skip_whitespace(stream); + + QueryStep *step = array_get(&self->steps, starting_step_index); + while (step->alternative_index != NONE && step->alternative_index < self->steps.size) { + step = array_get(&self->steps, step->alternative_index); + } + step->alternative_index = self->steps.size; + } + + // Parse an '@'-prefixed capture pattern + else if (stream->next == '@') { + stream_advance(stream); + if (!stream_is_ident_start(stream)) return TSQueryErrorSyntax; + const char *capture_name = stream->input; + stream_scan_identifier(stream); + uint32_t length = (uint32_t)(stream->input - capture_name); + stream_skip_whitespace(stream); + + // Add the capture id to the first step of the pattern + uint16_t capture_id = symbol_table_insert_name( + &self->captures, + capture_name, + length + ); + + // Add the capture quantifier + capture_quantifiers_add_for_id(capture_quantifiers, capture_id, TSQuantifierOne); + + uint32_t step_index = starting_step_index; + for (;;) { + QueryStep *step = array_get(&self->steps, step_index); + query_step__add_capture(step, capture_id); + if ( + step->alternative_index != NONE && + step->alternative_index > step_index && + step->alternative_index < self->steps.size + ) { + step_index = step->alternative_index; + } else { + break; + } + } + } + + // No more suffix modifiers + else { + break; + } + } + + capture_quantifiers_mul(capture_quantifiers, quantifier); + + return 0; +} + +TSQuery *ts_query_new( + const TSLanguage *language, + const char *source, + uint32_t source_len, + uint32_t *error_offset, + TSQueryError *error_type +) { + if ( + !language || + language->abi_version > TREE_SITTER_LANGUAGE_VERSION || + language->abi_version < TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION + ) { + *error_type = TSQueryErrorLanguage; + return NULL; + } + + TSQuery *self = ts_malloc(sizeof(TSQuery)); + *self = (TSQuery) { + .steps = array_new(), + .pattern_map = array_new(), + .captures = symbol_table_new(), + .capture_quantifiers = array_new(), + .predicate_values = symbol_table_new(), + .predicate_steps = array_new(), + .patterns = array_new(), + .step_offsets = array_new(), + .string_buffer = array_new(), + .negated_fields = array_new(), + .repeat_symbols_with_rootless_patterns = array_new(), + .wildcard_root_pattern_count = 0, + .language = ts_language_copy(language), + }; + + array_push(&self->negated_fields, 0); + + // Parse all of the S-expressions in the given string. + Stream stream = stream_new(source, source_len); + stream_skip_whitespace(&stream); + while (stream.input < stream.end) { + uint32_t pattern_index = self->patterns.size; + uint32_t start_step_index = self->steps.size; + uint32_t start_predicate_step_index = self->predicate_steps.size; + array_push(&self->patterns, ((QueryPattern) { + .steps = (Slice) {.offset = start_step_index}, + .predicate_steps = (Slice) {.offset = start_predicate_step_index}, + .start_byte = stream_offset(&stream), + .is_non_local = false, + })); + CaptureQuantifiers capture_quantifiers = capture_quantifiers_new(); + *error_type = ts_query__parse_pattern(self, &stream, 0, false, &capture_quantifiers); + array_push(&self->steps, query_step__new(0, PATTERN_DONE_MARKER, false)); + + QueryPattern *pattern = array_back(&self->patterns); + pattern->steps.length = self->steps.size - start_step_index; + pattern->predicate_steps.length = self->predicate_steps.size - start_predicate_step_index; + pattern->end_byte = stream_offset(&stream); + + // If any pattern could not be parsed, then report the error information + // and terminate. + if (*error_type) { + if (*error_type == PARENT_DONE) *error_type = TSQueryErrorSyntax; + *error_offset = stream_offset(&stream); + capture_quantifiers_delete(&capture_quantifiers); + ts_query_delete(self); + return NULL; + } + + // Maintain a list of capture quantifiers for each pattern + array_push(&self->capture_quantifiers, capture_quantifiers); + + // Maintain a map that can look up patterns for a given root symbol. + uint16_t wildcard_root_alternative_index = NONE; + for (;;) { + QueryStep *step = array_get(&self->steps, start_step_index); + + // If a pattern has a wildcard at its root, but it has a non-wildcard child, + // then optimize the matching process by skipping matching the wildcard. + // Later, during the matching process, the query cursor will check that + // there is a parent node, and capture it if necessary. + if (step->symbol == WILDCARD_SYMBOL && step->depth == 0 && !step->field) { + QueryStep *second_step = array_get(&self->steps, start_step_index + 1); + if (second_step->symbol != WILDCARD_SYMBOL && second_step->depth == 1 && !second_step->is_immediate) { + wildcard_root_alternative_index = step->alternative_index; + start_step_index += 1; + step = second_step; + } + } + + // Determine whether the pattern has a single root node. This affects + // decisions about whether or not to start matching the pattern when + // a query cursor has a range restriction or when immediately within an + // error node. + uint32_t start_depth = step->depth; + bool is_rooted = start_depth == 0; + for (uint32_t step_index = start_step_index + 1; step_index < self->steps.size; step_index++) { + QueryStep *child_step = array_get(&self->steps, step_index); + if (child_step->is_dead_end) break; + if (child_step->depth == start_depth) { + is_rooted = false; + break; + } + } + + ts_query__pattern_map_insert(self, step->symbol, (PatternEntry) { + .step_index = start_step_index, + .pattern_index = pattern_index, + .is_rooted = is_rooted + }); + if (step->symbol == WILDCARD_SYMBOL) { + self->wildcard_root_pattern_count++; + } + + // If there are alternatives or options at the root of the pattern, + // then add multiple entries to the pattern map. + if (step->alternative_index != NONE) { + start_step_index = step->alternative_index; + } else if (wildcard_root_alternative_index != NONE) { + start_step_index = wildcard_root_alternative_index; + wildcard_root_alternative_index = NONE; + } else { + break; + } + } + } + + if (!ts_query__analyze_patterns(self, error_offset)) { + *error_type = TSQueryErrorStructure; + ts_query_delete(self); + return NULL; + } + + array_delete(&self->string_buffer); + return self; +} + +void ts_query_delete(TSQuery *self) { + if (self) { + array_delete(&self->steps); + array_delete(&self->pattern_map); + array_delete(&self->predicate_steps); + array_delete(&self->patterns); + array_delete(&self->step_offsets); + array_delete(&self->string_buffer); + array_delete(&self->negated_fields); + array_delete(&self->repeat_symbols_with_rootless_patterns); + ts_language_delete(self->language); + symbol_table_delete(&self->captures); + symbol_table_delete(&self->predicate_values); + for (uint32_t index = 0; index < self->capture_quantifiers.size; index++) { + CaptureQuantifiers *capture_quantifiers = array_get(&self->capture_quantifiers, index); + capture_quantifiers_delete(capture_quantifiers); + } + array_delete(&self->capture_quantifiers); + ts_free(self); + } +} + +uint32_t ts_query_pattern_count(const TSQuery *self) { + return self->patterns.size; +} + +uint32_t ts_query_capture_count(const TSQuery *self) { + return self->captures.slices.size; +} + +uint32_t ts_query_string_count(const TSQuery *self) { + return self->predicate_values.slices.size; +} + +const char *ts_query_capture_name_for_id( + const TSQuery *self, + uint32_t index, + uint32_t *length +) { + return symbol_table_name_for_id(&self->captures, index, length); +} + +TSQuantifier ts_query_capture_quantifier_for_id( + const TSQuery *self, + uint32_t pattern_index, + uint32_t capture_index +) { + CaptureQuantifiers *capture_quantifiers = array_get(&self->capture_quantifiers, pattern_index); + return capture_quantifier_for_id(capture_quantifiers, capture_index); +} + +const char *ts_query_string_value_for_id( + const TSQuery *self, + uint32_t index, + uint32_t *length +) { + return symbol_table_name_for_id(&self->predicate_values, index, length); +} + +const TSQueryPredicateStep *ts_query_predicates_for_pattern( + const TSQuery *self, + uint32_t pattern_index, + uint32_t *step_count +) { + Slice slice = array_get(&self->patterns, pattern_index)->predicate_steps; + *step_count = slice.length; + if (slice.length == 0) return NULL; + return array_get(&self->predicate_steps, slice.offset); +} + +uint32_t ts_query_start_byte_for_pattern( + const TSQuery *self, + uint32_t pattern_index +) { + return array_get(&self->patterns, pattern_index)->start_byte; +} + +uint32_t ts_query_end_byte_for_pattern( + const TSQuery *self, + uint32_t pattern_index +) { + return array_get(&self->patterns, pattern_index)->end_byte; +} + +bool ts_query_is_pattern_rooted( + const TSQuery *self, + uint32_t pattern_index +) { + for (unsigned i = 0; i < self->pattern_map.size; i++) { + PatternEntry *entry = array_get(&self->pattern_map, i); + if (entry->pattern_index == pattern_index) { + if (!entry->is_rooted) return false; + } + } + return true; +} + +bool ts_query_is_pattern_non_local( + const TSQuery *self, + uint32_t pattern_index +) { + if (pattern_index < self->patterns.size) { + return array_get(&self->patterns, pattern_index)->is_non_local; + } else { + return false; + } +} + +bool ts_query_is_pattern_guaranteed_at_step( + const TSQuery *self, + uint32_t byte_offset +) { + uint32_t step_index = UINT32_MAX; + for (unsigned i = 0; i < self->step_offsets.size; i++) { + StepOffset *step_offset = array_get(&self->step_offsets, i); + if (step_offset->byte_offset > byte_offset) break; + step_index = step_offset->step_index; + } + if (step_index < self->steps.size) { + return array_get(&self->steps, step_index)->root_pattern_guaranteed; + } else { + return false; + } +} + +bool ts_query__step_is_fallible( + const TSQuery *self, + uint16_t step_index +) { + ts_assert((uint32_t)step_index + 1 < self->steps.size); + QueryStep *step = array_get(&self->steps, step_index); + QueryStep *next_step = array_get(&self->steps, step_index + 1); + return ( + next_step->depth != PATTERN_DONE_MARKER && + next_step->depth > step->depth && + (!next_step->parent_pattern_guaranteed || step->symbol == WILDCARD_SYMBOL) + ); +} + +void ts_query_disable_capture( + TSQuery *self, + const char *name, + uint32_t length +) { + // Remove capture information for any pattern step that previously + // captured with the given name. + int id = symbol_table_id_for_name(&self->captures, name, length); + if (id != -1) { + for (unsigned i = 0; i < self->steps.size; i++) { + QueryStep *step = array_get(&self->steps, i); + query_step__remove_capture(step, id); + } + } +} + +void ts_query_disable_pattern( + TSQuery *self, + uint32_t pattern_index +) { + // Remove the given pattern from the pattern map. Its steps will still + // be in the `steps` array, but they will never be read. + for (unsigned i = 0; i < self->pattern_map.size; i++) { + PatternEntry *pattern = array_get(&self->pattern_map, i); + if (pattern->pattern_index == pattern_index) { + array_erase(&self->pattern_map, i); + i--; + } + } +} + +/*************** + * QueryCursor + ***************/ + +TSQueryCursor *ts_query_cursor_new(void) { + TSQueryCursor *self = ts_malloc(sizeof(TSQueryCursor)); + *self = (TSQueryCursor) { + .did_exceed_match_limit = false, + .ascending = false, + .halted = false, + .states = array_new(), + .finished_states = array_new(), + .capture_list_pool = capture_list_pool_new(), + .start_byte = 0, + .end_byte = UINT32_MAX, + .start_point = {0, 0}, + .end_point = POINT_MAX, + .max_start_depth = UINT32_MAX, + .timeout_duration = 0, + .end_clock = clock_null(), + .operation_count = 0, + }; + array_reserve(&self->states, 8); + array_reserve(&self->finished_states, 8); + return self; +} + +void ts_query_cursor_delete(TSQueryCursor *self) { + array_delete(&self->states); + array_delete(&self->finished_states); + ts_tree_cursor_delete(&self->cursor); + capture_list_pool_delete(&self->capture_list_pool); + ts_free(self); +} + +bool ts_query_cursor_did_exceed_match_limit(const TSQueryCursor *self) { + return self->did_exceed_match_limit; +} + +uint32_t ts_query_cursor_match_limit(const TSQueryCursor *self) { + return self->capture_list_pool.max_capture_list_count; +} + +void ts_query_cursor_set_match_limit(TSQueryCursor *self, uint32_t limit) { + self->capture_list_pool.max_capture_list_count = limit; +} + +uint64_t ts_query_cursor_timeout_micros(const TSQueryCursor *self) { + return duration_to_micros(self->timeout_duration); +} + +void ts_query_cursor_set_timeout_micros(TSQueryCursor *self, uint64_t timeout_micros) { + self->timeout_duration = duration_from_micros(timeout_micros); +} + +#ifdef DEBUG_EXECUTE_QUERY +#define LOG(...) fprintf(stderr, __VA_ARGS__) +#else +#define LOG(...) +#endif + +void ts_query_cursor_exec( + TSQueryCursor *self, + const TSQuery *query, + TSNode node +) { + if (query) { + LOG("query steps:\n"); + for (unsigned i = 0; i < query->steps.size; i++) { + QueryStep *step = array_get(&query->steps, i); + LOG(" %u: {", i); + if (step->depth == PATTERN_DONE_MARKER) { + LOG("DONE"); + } else if (step->is_dead_end) { + LOG("dead_end"); + } else if (step->is_pass_through) { + LOG("pass_through"); + } else if (step->symbol != WILDCARD_SYMBOL) { + LOG("symbol: %s", query->language->symbol_names[step->symbol]); + } else { + LOG("symbol: *"); + } + if (step->field) { + LOG(", field: %s", query->language->field_names[step->field]); + } + if (step->alternative_index != NONE) { + LOG(", alternative: %u", step->alternative_index); + } + LOG("},\n"); + } + } + + array_clear(&self->states); + array_clear(&self->finished_states); + ts_tree_cursor_reset(&self->cursor, node); + capture_list_pool_reset(&self->capture_list_pool); + self->on_visible_node = true; + self->next_state_id = 0; + self->depth = 0; + self->ascending = false; + self->halted = false; + self->query = query; + self->did_exceed_match_limit = false; + self->operation_count = 0; + if (self->timeout_duration) { + self->end_clock = clock_after(clock_now(), self->timeout_duration); + } else { + self->end_clock = clock_null(); + } + self->query_options = NULL; + self->query_state = (TSQueryCursorState) {0}; +} + +void ts_query_cursor_exec_with_options( + TSQueryCursor *self, + const TSQuery *query, + TSNode node, + const TSQueryCursorOptions *query_options +) { + ts_query_cursor_exec(self, query, node); + if (query_options) { + self->query_options = query_options; + self->query_state = (TSQueryCursorState) { + .payload = query_options->payload + }; + } +} + +bool ts_query_cursor_set_byte_range( + TSQueryCursor *self, + uint32_t start_byte, + uint32_t end_byte +) { + if (end_byte == 0) { + end_byte = UINT32_MAX; + } + if (start_byte > end_byte) { + return false; + } + self->start_byte = start_byte; + self->end_byte = end_byte; + return true; +} + +bool ts_query_cursor_set_point_range( + TSQueryCursor *self, + TSPoint start_point, + TSPoint end_point +) { + if (end_point.row == 0 && end_point.column == 0) { + end_point = POINT_MAX; + } + if (point_gt(start_point, end_point)) { + return false; + } + self->start_point = start_point; + self->end_point = end_point; + return true; +} + +// Search through all of the in-progress states, and find the captured +// node that occurs earliest in the document. +static bool ts_query_cursor__first_in_progress_capture( + TSQueryCursor *self, + uint32_t *state_index, + uint32_t *byte_offset, + uint32_t *pattern_index, + bool *is_definite +) { + bool result = false; + *state_index = UINT32_MAX; + *byte_offset = UINT32_MAX; + *pattern_index = UINT32_MAX; + for (unsigned i = 0; i < self->states.size; i++) { + QueryState *state = array_get(&self->states, i); + if (state->dead) continue; + + const CaptureList *captures = capture_list_pool_get( + &self->capture_list_pool, + state->capture_list_id + ); + if (state->consumed_capture_count >= captures->size) { + continue; + } + + TSNode node = array_get(captures, state->consumed_capture_count)->node; + if ( + ts_node_end_byte(node) <= self->start_byte || + point_lte(ts_node_end_point(node), self->start_point) + ) { + state->consumed_capture_count++; + i--; + continue; + } + + uint32_t node_start_byte = ts_node_start_byte(node); + if ( + !result || + node_start_byte < *byte_offset || + (node_start_byte == *byte_offset && state->pattern_index < *pattern_index) + ) { + QueryStep *step = array_get(&self->query->steps, state->step_index); + if (is_definite) { + // We're being a bit conservative here by asserting that the following step + // is not immediate, because this capture might end up being discarded if the + // following symbol in the tree isn't the required symbol for this step. + *is_definite = step->root_pattern_guaranteed && !step->is_immediate; + } else if (step->root_pattern_guaranteed) { + continue; + } + + result = true; + *state_index = i; + *byte_offset = node_start_byte; + *pattern_index = state->pattern_index; + } + } + return result; +} + +// Determine which node is first in a depth-first traversal +int ts_query_cursor__compare_nodes(TSNode left, TSNode right) { + if (left.id != right.id) { + uint32_t left_start = ts_node_start_byte(left); + uint32_t right_start = ts_node_start_byte(right); + if (left_start < right_start) return -1; + if (left_start > right_start) return 1; + uint32_t left_node_count = ts_node_end_byte(left); + uint32_t right_node_count = ts_node_end_byte(right); + if (left_node_count > right_node_count) return -1; + if (left_node_count < right_node_count) return 1; + } + return 0; +} + +// Determine if either state contains a superset of the other state's captures. +void ts_query_cursor__compare_captures( + TSQueryCursor *self, + QueryState *left_state, + QueryState *right_state, + bool *left_contains_right, + bool *right_contains_left +) { + const CaptureList *left_captures = capture_list_pool_get( + &self->capture_list_pool, + left_state->capture_list_id + ); + const CaptureList *right_captures = capture_list_pool_get( + &self->capture_list_pool, + right_state->capture_list_id + ); + *left_contains_right = true; + *right_contains_left = true; + unsigned i = 0, j = 0; + for (;;) { + if (i < left_captures->size) { + if (j < right_captures->size) { + TSQueryCapture *left = array_get(left_captures, i); + TSQueryCapture *right = array_get(right_captures, j); + if (left->node.id == right->node.id && left->index == right->index) { + i++; + j++; + } else { + switch (ts_query_cursor__compare_nodes(left->node, right->node)) { + case -1: + *right_contains_left = false; + i++; + break; + case 1: + *left_contains_right = false; + j++; + break; + default: + *right_contains_left = false; + *left_contains_right = false; + i++; + j++; + break; + } + } + } else { + *right_contains_left = false; + break; + } + } else { + if (j < right_captures->size) { + *left_contains_right = false; + } + break; + } + } +} + +static void ts_query_cursor__add_state( + TSQueryCursor *self, + const PatternEntry *pattern +) { + QueryStep *step = array_get(&self->query->steps, pattern->step_index); + uint32_t start_depth = self->depth - step->depth; + + // Keep the states array in ascending order of start_depth and pattern_index, + // so that it can be processed more efficiently elsewhere. Usually, there is + // no work to do here because of two facts: + // * States with lower start_depth are naturally added first due to the + // order in which nodes are visited. + // * Earlier patterns are naturally added first because of the ordering of the + // pattern_map data structure that's used to initiate matches. + // + // This loop is only needed in cases where two conditions hold: + // * A pattern consists of more than one sibling node, so that its states + // remain in progress after exiting the node that started the match. + // * The first node in the pattern matches against multiple nodes at the + // same depth. + // + // An example of this is the pattern '((comment)* (function))'. If multiple + // `comment` nodes appear in a row, then we may initiate a new state for this + // pattern while another state for the same pattern is already in progress. + // If there are multiple patterns like this in a query, then this loop will + // need to execute in order to keep the states ordered by pattern_index. + uint32_t index = self->states.size; + while (index > 0) { + QueryState *prev_state = array_get(&self->states, index - 1); + if (prev_state->start_depth < start_depth) break; + if (prev_state->start_depth == start_depth) { + // Avoid inserting an unnecessary duplicate state, which would be + // immediately pruned by the longest-match criteria. + if ( + prev_state->pattern_index == pattern->pattern_index && + prev_state->step_index == pattern->step_index + ) return; + if (prev_state->pattern_index <= pattern->pattern_index) break; + } + index--; + } + + LOG( + " start state. pattern:%u, step:%u\n", + pattern->pattern_index, + pattern->step_index + ); + array_insert(&self->states, index, ((QueryState) { + .id = UINT32_MAX, + .capture_list_id = NONE, + .step_index = pattern->step_index, + .pattern_index = pattern->pattern_index, + .start_depth = start_depth, + .consumed_capture_count = 0, + .seeking_immediate_match = true, + .has_in_progress_alternatives = false, + .needs_parent = step->depth == 1, + .dead = false, + })); +} + +// Acquire a capture list for this state. If there are no capture lists left in the +// pool, this will steal the capture list from another existing state, and mark that +// other state as 'dead'. +static CaptureList *ts_query_cursor__prepare_to_capture( + TSQueryCursor *self, + QueryState *state, + unsigned state_index_to_preserve +) { + if (state->capture_list_id == NONE) { + state->capture_list_id = capture_list_pool_acquire(&self->capture_list_pool); + + // If there are no capture lists left in the pool, then terminate whichever + // state has captured the earliest node in the document, and steal its + // capture list. + if (state->capture_list_id == NONE) { + self->did_exceed_match_limit = true; + uint32_t state_index, byte_offset, pattern_index; + if ( + ts_query_cursor__first_in_progress_capture( + self, + &state_index, + &byte_offset, + &pattern_index, + NULL + ) && + state_index != state_index_to_preserve + ) { + LOG( + " abandon state. index:%u, pattern:%u, offset:%u.\n", + state_index, pattern_index, byte_offset + ); + QueryState *other_state = array_get(&self->states, state_index); + state->capture_list_id = other_state->capture_list_id; + other_state->capture_list_id = NONE; + other_state->dead = true; + CaptureList *list = capture_list_pool_get_mut( + &self->capture_list_pool, + state->capture_list_id + ); + array_clear(list); + return list; + } else { + LOG(" ran out of capture lists"); + return NULL; + } + } + } + return capture_list_pool_get_mut(&self->capture_list_pool, state->capture_list_id); +} + +static void ts_query_cursor__capture( + TSQueryCursor *self, + QueryState *state, + QueryStep *step, + TSNode node +) { + if (state->dead) return; + CaptureList *capture_list = ts_query_cursor__prepare_to_capture(self, state, UINT32_MAX); + if (!capture_list) { + state->dead = true; + return; + } + + for (unsigned j = 0; j < MAX_STEP_CAPTURE_COUNT; j++) { + uint16_t capture_id = step->capture_ids[j]; + if (step->capture_ids[j] == NONE) break; + array_push(capture_list, ((TSQueryCapture) { node, capture_id })); + LOG( + " capture node. type:%s, pattern:%u, capture_id:%u, capture_count:%u\n", + ts_node_type(node), + state->pattern_index, + capture_id, + capture_list->size + ); + } +} + +// Duplicate the given state and insert the newly-created state immediately after +// the given state in the `states` array. Ensures that the given state reference is +// still valid, even if the states array is reallocated. +static QueryState *ts_query_cursor__copy_state( + TSQueryCursor *self, + QueryState **state_ref +) { + const QueryState *state = *state_ref; + uint32_t state_index = (uint32_t)(state - self->states.contents); + QueryState copy = *state; + copy.capture_list_id = NONE; + + // If the state has captures, copy its capture list. + if (state->capture_list_id != NONE) { + CaptureList *new_captures = ts_query_cursor__prepare_to_capture(self, ©, state_index); + if (!new_captures) return NULL; + const CaptureList *old_captures = capture_list_pool_get( + &self->capture_list_pool, + state->capture_list_id + ); + array_push_all(new_captures, old_captures); + } + + array_insert(&self->states, state_index + 1, copy); + *state_ref = array_get(&self->states, state_index); + return array_get(&self->states, state_index + 1); +} + +static inline bool ts_query_cursor__should_descend( + TSQueryCursor *self, + bool node_intersects_range +) { + + if (node_intersects_range && self->depth < self->max_start_depth) { + return true; + } + + // If there are in-progress matches whose remaining steps occur + // deeper in the tree, then descend. + for (unsigned i = 0; i < self->states.size; i++) { + QueryState *state = array_get(&self->states, i); + QueryStep *next_step = array_get(&self->query->steps, state->step_index); + if ( + next_step->depth != PATTERN_DONE_MARKER && + state->start_depth + next_step->depth > self->depth + ) { + return true; + } + } + + if (self->depth >= self->max_start_depth) { + return false; + } + + // If the current node is hidden, then a non-rooted pattern might match + // one if its roots inside of this node, and match another of its roots + // as part of a sibling node, so we may need to descend. + if (!self->on_visible_node) { + // Descending into a repetition node outside of the range can be + // expensive, because these nodes can have many visible children. + // Avoid descending into repetition nodes unless we have already + // determined that this query can match rootless patterns inside + // of this type of repetition node. + Subtree subtree = ts_tree_cursor_current_subtree(&self->cursor); + if (ts_subtree_is_repetition(subtree)) { + bool exists; + uint32_t index; + array_search_sorted_by( + &self->query->repeat_symbols_with_rootless_patterns,, + ts_subtree_symbol(subtree), + &index, + &exists + ); + return exists; + } + + return true; + } + + return false; +} + +// Walk the tree, processing patterns until at least one pattern finishes, +// If one or more patterns finish, return `true` and store their states in the +// `finished_states` array. Multiple patterns can finish on the same node. If +// there are no more matches, return `false`. +static inline bool ts_query_cursor__advance( + TSQueryCursor *self, + bool stop_on_definite_step +) { + bool did_match = false; + for (;;) { + if (self->halted) { + while (self->states.size > 0) { + QueryState state = array_pop(&self->states); + capture_list_pool_release( + &self->capture_list_pool, + state.capture_list_id + ); + } + } + + if (++self->operation_count == OP_COUNT_PER_QUERY_TIMEOUT_CHECK) { + self->operation_count = 0; + } + + if (self->query_options && self->query_options->progress_callback) { + self->query_state.current_byte_offset = ts_node_start_byte(ts_tree_cursor_current_node(&self->cursor)); + } + if ( + did_match || + self->halted || + ( + self->operation_count == 0 && + ( + (!clock_is_null(self->end_clock) && clock_is_gt(clock_now(), self->end_clock)) || + (self->query_options && self->query_options->progress_callback && self->query_options->progress_callback(&self->query_state)) + ) + ) + ) { + return did_match; + } + + // Exit the current node. + if (self->ascending) { + if (self->on_visible_node) { + LOG( + "leave node. depth:%u, type:%s\n", + self->depth, + ts_node_type(ts_tree_cursor_current_node(&self->cursor)) + ); + + // After leaving a node, remove any states that cannot make further progress. + uint32_t deleted_count = 0; + for (unsigned i = 0, n = self->states.size; i < n; i++) { + QueryState *state = array_get(&self->states, i); + QueryStep *step = array_get(&self->query->steps, state->step_index); + + // If a state completed its pattern inside of this node, but was deferred from finishing + // in order to search for longer matches, mark it as finished. + if ( + step->depth == PATTERN_DONE_MARKER && + (state->start_depth > self->depth || self->depth == 0) + ) { + LOG(" finish pattern %u\n", state->pattern_index); + array_push(&self->finished_states, *state); + did_match = true; + deleted_count++; + } + + // If a state needed to match something within this node, then remove that state + // as it has failed to match. + else if ( + step->depth != PATTERN_DONE_MARKER && + (uint32_t)state->start_depth + (uint32_t)step->depth > self->depth + ) { + LOG( + " failed to match. pattern:%u, step:%u\n", + state->pattern_index, + state->step_index + ); + capture_list_pool_release( + &self->capture_list_pool, + state->capture_list_id + ); + deleted_count++; + } + + else if (deleted_count > 0) { + *array_get(&self->states, i - deleted_count) = *state; + } + } + self->states.size -= deleted_count; + } + + // Leave this node by stepping to its next sibling or to its parent. + switch (ts_tree_cursor_goto_next_sibling_internal(&self->cursor)) { + case TreeCursorStepVisible: + if (!self->on_visible_node) { + self->depth++; + self->on_visible_node = true; + } + self->ascending = false; + break; + case TreeCursorStepHidden: + if (self->on_visible_node) { + self->depth--; + self->on_visible_node = false; + } + self->ascending = false; + break; + default: + if (ts_tree_cursor_goto_parent(&self->cursor)) { + self->depth--; + } else { + LOG("halt at root\n"); + self->halted = true; + } + } + } + + // Enter a new node. + else { + // Get the properties of the current node. + TSNode node = ts_tree_cursor_current_node(&self->cursor); + TSNode parent_node = ts_tree_cursor_parent_node(&self->cursor); + + uint32_t start_byte = ts_node_start_byte(node); + uint32_t end_byte = ts_node_end_byte(node); + TSPoint start_point = ts_node_start_point(node); + TSPoint end_point = ts_node_end_point(node); + bool is_empty = start_byte == end_byte; + + bool parent_precedes_range = !ts_node_is_null(parent_node) && ( + ts_node_end_byte(parent_node) <= self->start_byte || + point_lte(ts_node_end_point(parent_node), self->start_point) + ); + bool parent_follows_range = !ts_node_is_null(parent_node) && ( + ts_node_start_byte(parent_node) >= self->end_byte || + point_gte(ts_node_start_point(parent_node), self->end_point) + ); + bool node_precedes_range = + parent_precedes_range || + end_byte < self->start_byte || + point_lt(end_point, self->start_point) || + (!is_empty && end_byte == self->start_byte) || + (!is_empty && point_eq(end_point, self->start_point)); + + bool node_follows_range = parent_follows_range || ( + start_byte >= self->end_byte || + point_gte(start_point, self->end_point) + ); + bool parent_intersects_range = !parent_precedes_range && !parent_follows_range; + bool node_intersects_range = !node_precedes_range && !node_follows_range; + + if (self->on_visible_node) { + TSSymbol symbol = ts_node_symbol(node); + bool is_named = ts_node_is_named(node); + bool is_missing = ts_node_is_missing(node); + bool has_later_siblings; + bool has_later_named_siblings; + bool can_have_later_siblings_with_this_field; + TSFieldId field_id = 0; + TSSymbol supertypes[8] = {0}; + unsigned supertype_count = 8; + ts_tree_cursor_current_status( + &self->cursor, + &field_id, + &has_later_siblings, + &has_later_named_siblings, + &can_have_later_siblings_with_this_field, + supertypes, + &supertype_count + ); + LOG( + "enter node. depth:%u, type:%s, field:%s, row:%u state_count:%u, finished_state_count:%u\n", + self->depth, + ts_node_type(node), + ts_language_field_name_for_id(self->query->language, field_id), + ts_node_start_point(node).row, + self->states.size, + self->finished_states.size + ); + + bool node_is_error = symbol == ts_builtin_sym_error; + bool parent_is_error = + !ts_node_is_null(parent_node) && + ts_node_symbol(parent_node) == ts_builtin_sym_error; + + // Add new states for any patterns whose root node is a wildcard. + if (!node_is_error) { + for (unsigned i = 0; i < self->query->wildcard_root_pattern_count; i++) { + PatternEntry *pattern = array_get(&self->query->pattern_map, i); + + // If this node matches the first step of the pattern, then add a new + // state at the start of this pattern. + QueryStep *step = array_get(&self->query->steps, pattern->step_index); + uint32_t start_depth = self->depth - step->depth; + if ( + (pattern->is_rooted ? + node_intersects_range : + (parent_intersects_range && !parent_is_error)) && + (!step->field || field_id == step->field) && + (!step->supertype_symbol || supertype_count > 0) && + (start_depth <= self->max_start_depth) + ) { + ts_query_cursor__add_state(self, pattern); + } + } + } + + // Add new states for any patterns whose root node matches this node. + unsigned i; + if (ts_query__pattern_map_search(self->query, symbol, &i)) { + PatternEntry *pattern = array_get(&self->query->pattern_map, i); + + QueryStep *step = array_get(&self->query->steps, pattern->step_index); + uint32_t start_depth = self->depth - step->depth; + do { + // If this node matches the first step of the pattern, then add a new + // state at the start of this pattern. + if ( + (pattern->is_rooted ? + node_intersects_range : + (parent_intersects_range && !parent_is_error)) && + (!step->field || field_id == step->field) && + (start_depth <= self->max_start_depth) + ) { + ts_query_cursor__add_state(self, pattern); + } + + // Advance to the next pattern whose root node matches this node. + i++; + if (i == self->query->pattern_map.size) break; + pattern = array_get(&self->query->pattern_map, i); + step = array_get(&self->query->steps, pattern->step_index); + } while (step->symbol == symbol); + } + + // Update all of the in-progress states with current node. + for (unsigned j = 0, copy_count = 0; j < self->states.size; j += 1 + copy_count) { + QueryState *state = array_get(&self->states, j); + QueryStep *step = array_get(&self->query->steps, state->step_index); + state->has_in_progress_alternatives = false; + copy_count = 0; + + // Check that the node matches all of the criteria for the next + // step of the pattern. + if ((uint32_t)state->start_depth + (uint32_t)step->depth != self->depth) continue; + + // Determine if this node matches this step of the pattern, and also + // if this node can have later siblings that match this step of the + // pattern. + bool node_does_match = false; + if (step->symbol == WILDCARD_SYMBOL) { + if (step->is_missing) { + node_does_match = is_missing; + } else { + node_does_match = !node_is_error && (is_named || !step->is_named); + } + } else { + node_does_match = symbol == step->symbol && (!step->is_missing || is_missing); + } + bool later_sibling_can_match = has_later_siblings; + if ((step->is_immediate && is_named) || state->seeking_immediate_match) { + later_sibling_can_match = false; + } + if (step->is_last_child && has_later_named_siblings) { + node_does_match = false; + } + if (step->supertype_symbol) { + bool has_supertype = false; + for (unsigned k = 0; k < supertype_count; k++) { + if (supertypes[k] == step->supertype_symbol) { + has_supertype = true; + break; + } + } + if (!has_supertype) node_does_match = false; + } + if (step->field) { + if (step->field == field_id) { + if (!can_have_later_siblings_with_this_field) { + later_sibling_can_match = false; + } + } else { + node_does_match = false; + } + } + + if (step->negated_field_list_id) { + TSFieldId *negated_field_ids = array_get(&self->query->negated_fields, step->negated_field_list_id); + for (;;) { + TSFieldId negated_field_id = *negated_field_ids; + if (negated_field_id) { + negated_field_ids++; + if (ts_node_child_by_field_id(node, negated_field_id).id) { + node_does_match = false; + break; + } + } else { + break; + } + } + } + + // Remove states immediately if it is ever clear that they cannot match. + if (!node_does_match) { + if (!later_sibling_can_match) { + LOG( + " discard state. pattern:%u, step:%u\n", + state->pattern_index, + state->step_index + ); + capture_list_pool_release( + &self->capture_list_pool, + state->capture_list_id + ); + array_erase(&self->states, j); + j--; + } + continue; + } + + // Some patterns can match their root node in multiple ways, capturing different + // children. If this pattern step could match later children within the same + // parent, then this query state cannot simply be updated in place. It must be + // split into two states: one that matches this node, and one which skips over + // this node, to preserve the possibility of matching later siblings. + if (later_sibling_can_match && ( + step->contains_captures || + ts_query__step_is_fallible(self->query, state->step_index) + )) { + if (ts_query_cursor__copy_state(self, &state)) { + LOG( + " split state for capture. pattern:%u, step:%u\n", + state->pattern_index, + state->step_index + ); + copy_count++; + } + } + + // If this pattern started with a wildcard, such that the pattern map + // actually points to the *second* step of the pattern, then check + // that the node has a parent, and capture the parent node if necessary. + if (state->needs_parent) { + TSNode parent = ts_tree_cursor_parent_node(&self->cursor); + if (ts_node_is_null(parent)) { + LOG(" missing parent node\n"); + state->dead = true; + } else { + state->needs_parent = false; + QueryStep *skipped_wildcard_step = step; + do { + skipped_wildcard_step--; + } while ( + skipped_wildcard_step->is_dead_end || + skipped_wildcard_step->is_pass_through || + skipped_wildcard_step->depth > 0 + ); + if (skipped_wildcard_step->capture_ids[0] != NONE) { + LOG(" capture wildcard parent\n"); + ts_query_cursor__capture( + self, + state, + skipped_wildcard_step, + parent + ); + } + } + } + + // If the current node is captured in this pattern, add it to the capture list. + if (step->capture_ids[0] != NONE) { + ts_query_cursor__capture(self, state, step, node); + } + + if (state->dead) { + array_erase(&self->states, j); + j--; + continue; + } + + // Advance this state to the next step of its pattern. + state->step_index++; + LOG( + " advance state. pattern:%u, step:%u\n", + state->pattern_index, + state->step_index + ); + + QueryStep *next_step = array_get(&self->query->steps, state->step_index); + + // For a given step, if the current symbol is the wildcard symbol, `_`, and it is **not** + // named, meaning it should capture anonymous nodes, **and** the next step is immediate, + // we reuse the `seeking_immediate_match` flag to indicate that we are looking for an + // immediate match due to an unnamed wildcard symbol. + // + // The reason for this is that typically, anchors will not consider anonymous nodes, + // but we're special casing the wildcard symbol to allow for any immediate matches, + // regardless of whether they are named or not. + if (step->symbol == WILDCARD_SYMBOL && !step->is_named && next_step->is_immediate) { + state->seeking_immediate_match = true; + } else { + state->seeking_immediate_match = false; + } + + if (stop_on_definite_step && next_step->root_pattern_guaranteed) did_match = true; + + // If this state's next step has an alternative step, then copy the state in order + // to pursue both alternatives. The alternative step itself may have an alternative, + // so this is an interactive process. + unsigned end_index = j + 1; + for (unsigned k = j; k < end_index; k++) { + QueryState *child_state = array_get(&self->states, k); + QueryStep *child_step = array_get(&self->query->steps, child_state->step_index); + if (child_step->alternative_index != NONE) { + // A "dead-end" step exists only to add a non-sequential jump into the step sequence, + // via its alternative index. When a state reaches a dead-end step, it jumps straight + // to the step's alternative. + if (child_step->is_dead_end) { + child_state->step_index = child_step->alternative_index; + k--; + continue; + } + + // A "pass-through" step exists only to add a branch into the step sequence, + // via its alternative_index. When a state reaches a pass-through step, it splits + // in order to process the alternative step, and then it advances to the next step. + if (child_step->is_pass_through) { + child_state->step_index++; + k--; + } + + QueryState *copy = ts_query_cursor__copy_state(self, &child_state); + if (copy) { + LOG( + " split state for branch. pattern:%u, from_step:%u, to_step:%u, immediate:%d, capture_count: %u\n", + copy->pattern_index, + copy->step_index, + next_step->alternative_index, + next_step->alternative_is_immediate, + capture_list_pool_get(&self->capture_list_pool, copy->capture_list_id)->size + ); + end_index++; + copy_count++; + copy->step_index = child_step->alternative_index; + if (child_step->alternative_is_immediate) { + copy->seeking_immediate_match = true; + } + } + } + } + } + + for (unsigned j = 0; j < self->states.size; j++) { + QueryState *state = array_get(&self->states, j); + if (state->dead) { + array_erase(&self->states, j); + j--; + continue; + } + + // Enforce the longest-match criteria. When a query pattern contains optional or + // repeated nodes, this is necessary to avoid multiple redundant states, where + // one state has a strict subset of another state's captures. + bool did_remove = false; + for (unsigned k = j + 1; k < self->states.size; k++) { + QueryState *other_state = array_get(&self->states, k); + + // Query states are kept in ascending order of start_depth and pattern_index. + // Since the longest-match criteria is only used for deduping matches of the same + // pattern and root node, we only need to perform pairwise comparisons within a + // small slice of the states array. + if ( + other_state->start_depth != state->start_depth || + other_state->pattern_index != state->pattern_index + ) break; + + bool left_contains_right, right_contains_left; + ts_query_cursor__compare_captures( + self, + state, + other_state, + &left_contains_right, + &right_contains_left + ); + if (left_contains_right) { + if (state->step_index == other_state->step_index) { + LOG( + " drop shorter state. pattern: %u, step_index: %u\n", + state->pattern_index, + state->step_index + ); + capture_list_pool_release(&self->capture_list_pool, other_state->capture_list_id); + array_erase(&self->states, k); + k--; + continue; + } + other_state->has_in_progress_alternatives = true; + } + if (right_contains_left) { + if (state->step_index == other_state->step_index) { + LOG( + " drop shorter state. pattern: %u, step_index: %u\n", + state->pattern_index, + state->step_index + ); + capture_list_pool_release(&self->capture_list_pool, state->capture_list_id); + array_erase(&self->states, j); + j--; + did_remove = true; + break; + } + state->has_in_progress_alternatives = true; + } + } + + // If the state is at the end of its pattern, remove it from the list + // of in-progress states and add it to the list of finished states. + if (!did_remove) { + LOG( + " keep state. pattern: %u, start_depth: %u, step_index: %u, capture_count: %u\n", + state->pattern_index, + state->start_depth, + state->step_index, + capture_list_pool_get(&self->capture_list_pool, state->capture_list_id)->size + ); + QueryStep *next_step = array_get(&self->query->steps, state->step_index); + if (next_step->depth == PATTERN_DONE_MARKER) { + if (state->has_in_progress_alternatives) { + LOG(" defer finishing pattern %u\n", state->pattern_index); + } else { + LOG(" finish pattern %u\n", state->pattern_index); + array_push(&self->finished_states, *state); + array_erase(&self->states, (uint32_t)(state - self->states.contents)); + did_match = true; + j--; + } + } + } + } + } + + if (ts_query_cursor__should_descend(self, node_intersects_range)) { + switch (ts_tree_cursor_goto_first_child_internal(&self->cursor)) { + case TreeCursorStepVisible: + self->depth++; + self->on_visible_node = true; + continue; + case TreeCursorStepHidden: + self->on_visible_node = false; + continue; + default: + break; + } + } + + self->ascending = true; + } + } +} + +bool ts_query_cursor_next_match( + TSQueryCursor *self, + TSQueryMatch *match +) { + if (self->finished_states.size == 0) { + if (!ts_query_cursor__advance(self, false)) { + return false; + } + } + + QueryState *state = array_get(&self->finished_states, 0); + if (state->id == UINT32_MAX) state->id = self->next_state_id++; + match->id = state->id; + match->pattern_index = state->pattern_index; + const CaptureList *captures = capture_list_pool_get( + &self->capture_list_pool, + state->capture_list_id + ); + match->captures = captures->contents; + match->capture_count = captures->size; + capture_list_pool_release(&self->capture_list_pool, state->capture_list_id); + array_erase(&self->finished_states, 0); + return true; +} + +void ts_query_cursor_remove_match( + TSQueryCursor *self, + uint32_t match_id +) { + for (unsigned i = 0; i < self->finished_states.size; i++) { + const QueryState *state = array_get(&self->finished_states, i); + if (state->id == match_id) { + capture_list_pool_release( + &self->capture_list_pool, + state->capture_list_id + ); + array_erase(&self->finished_states, i); + return; + } + } + + // Remove unfinished query states as well to prevent future + // captures for a match being removed. + for (unsigned i = 0; i < self->states.size; i++) { + const QueryState *state = array_get(&self->states, i); + if (state->id == match_id) { + capture_list_pool_release( + &self->capture_list_pool, + state->capture_list_id + ); + array_erase(&self->states, i); + return; + } + } +} + +bool ts_query_cursor_next_capture( + TSQueryCursor *self, + TSQueryMatch *match, + uint32_t *capture_index +) { + // The goal here is to return captures in order, even though they may not + // be discovered in order, because patterns can overlap. Search for matches + // until there is a finished capture that is before any unfinished capture. + for (;;) { + // First, find the earliest capture in an unfinished match. + uint32_t first_unfinished_capture_byte; + uint32_t first_unfinished_pattern_index; + uint32_t first_unfinished_state_index; + bool first_unfinished_state_is_definite = false; + bool found_unfinished_state = ts_query_cursor__first_in_progress_capture( + self, + &first_unfinished_state_index, + &first_unfinished_capture_byte, + &first_unfinished_pattern_index, + &first_unfinished_state_is_definite + ); + + // Then find the earliest capture in a finished match. It must occur + // before the first capture in an *unfinished* match. + QueryState *first_finished_state = NULL; + uint32_t first_finished_capture_byte = first_unfinished_capture_byte; + uint32_t first_finished_pattern_index = first_unfinished_pattern_index; + for (unsigned i = 0; i < self->finished_states.size;) { + QueryState *state = array_get(&self->finished_states, i); + const CaptureList *captures = capture_list_pool_get( + &self->capture_list_pool, + state->capture_list_id + ); + + // Remove states whose captures are all consumed. + if (state->consumed_capture_count >= captures->size) { + capture_list_pool_release( + &self->capture_list_pool, + state->capture_list_id + ); + array_erase(&self->finished_states, i); + continue; + } + + TSNode node = array_get(captures, state->consumed_capture_count)->node; + + bool node_precedes_range = ( + ts_node_end_byte(node) <= self->start_byte || + point_lte(ts_node_end_point(node), self->start_point) + ); + bool node_follows_range = ( + ts_node_start_byte(node) >= self->end_byte || + point_gte(ts_node_start_point(node), self->end_point) + ); + bool node_outside_of_range = node_precedes_range || node_follows_range; + + // Skip captures that are outside of the cursor's range. + if (node_outside_of_range) { + state->consumed_capture_count++; + continue; + } + + uint32_t node_start_byte = ts_node_start_byte(node); + if ( + node_start_byte < first_finished_capture_byte || + ( + node_start_byte == first_finished_capture_byte && + state->pattern_index < first_finished_pattern_index + ) + ) { + first_finished_state = state; + first_finished_capture_byte = node_start_byte; + first_finished_pattern_index = state->pattern_index; + } + i++; + } + + // If there is finished capture that is clearly before any unfinished + // capture, then return its match, and its capture index. Internally + // record the fact that the capture has been 'consumed'. + QueryState *state; + if (first_finished_state) { + state = first_finished_state; + } else if (first_unfinished_state_is_definite) { + state = array_get(&self->states, first_unfinished_state_index); + } else { + state = NULL; + } + + if (state) { + if (state->id == UINT32_MAX) state->id = self->next_state_id++; + match->id = state->id; + match->pattern_index = state->pattern_index; + const CaptureList *captures = capture_list_pool_get( + &self->capture_list_pool, + state->capture_list_id + ); + match->captures = captures->contents; + match->capture_count = captures->size; + *capture_index = state->consumed_capture_count; + state->consumed_capture_count++; + return true; + } + + if (capture_list_pool_is_empty(&self->capture_list_pool) && found_unfinished_state) { + LOG( + " abandon state. index:%u, pattern:%u, offset:%u.\n", + first_unfinished_state_index, + first_unfinished_pattern_index, + first_unfinished_capture_byte + ); + capture_list_pool_release( + &self->capture_list_pool, + array_get(&self->states, first_unfinished_state_index)->capture_list_id + ); + array_erase(&self->states, first_unfinished_state_index); + } + + // If there are no finished matches that are ready to be returned, then + // continue finding more matches. + if ( + !ts_query_cursor__advance(self, true) && + self->finished_states.size == 0 + ) return false; + } +} + +void ts_query_cursor_set_max_start_depth( + TSQueryCursor *self, + uint32_t max_start_depth +) { + self->max_start_depth = max_start_depth; +} + +#undef LOG diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/reduce_action.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/reduce_action.h new file mode 100644 index 00000000000..72aff08d73d --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/reduce_action.h @@ -0,0 +1,34 @@ +#ifndef TREE_SITTER_REDUCE_ACTION_H_ +#define TREE_SITTER_REDUCE_ACTION_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "./array.h" +#include "tree_sitter/api.h" + +typedef struct { + uint32_t count; + TSSymbol symbol; + int dynamic_precedence; + unsigned short production_id; +} ReduceAction; + +typedef Array(ReduceAction) ReduceActionSet; + +static inline void ts_reduce_action_set_add(ReduceActionSet *self, + ReduceAction new_action) { + for (uint32_t i = 0; i < self->size; i++) { + ReduceAction action = self->contents[i]; + if (action.symbol == new_action.symbol && action.count == new_action.count) + return; + } + array_push(self, new_action); +} + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_REDUCE_ACTION_H_ diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/reusable_node.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/reusable_node.h new file mode 100644 index 00000000000..63fe3c1a36c --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/reusable_node.h @@ -0,0 +1,95 @@ +#include "./subtree.h" + +typedef struct { + Subtree tree; + uint32_t child_index; + uint32_t byte_offset; +} StackEntry; + +typedef struct { + Array(StackEntry) stack; + Subtree last_external_token; +} ReusableNode; + +static inline ReusableNode reusable_node_new(void) { + return (ReusableNode) {array_new(), NULL_SUBTREE}; +} + +static inline void reusable_node_clear(ReusableNode *self) { + array_clear(&self->stack); + self->last_external_token = NULL_SUBTREE; +} + +static inline Subtree reusable_node_tree(ReusableNode *self) { + return self->stack.size > 0 + ? self->stack.contents[self->stack.size - 1].tree + : NULL_SUBTREE; +} + +static inline uint32_t reusable_node_byte_offset(ReusableNode *self) { + return self->stack.size > 0 + ? self->stack.contents[self->stack.size - 1].byte_offset + : UINT32_MAX; +} + +static inline void reusable_node_delete(ReusableNode *self) { + array_delete(&self->stack); +} + +static inline void reusable_node_advance(ReusableNode *self) { + StackEntry last_entry = *array_back(&self->stack); + uint32_t byte_offset = last_entry.byte_offset + ts_subtree_total_bytes(last_entry.tree); + if (ts_subtree_has_external_tokens(last_entry.tree)) { + self->last_external_token = ts_subtree_last_external_token(last_entry.tree); + } + + Subtree tree; + uint32_t next_index; + do { + StackEntry popped_entry = array_pop(&self->stack); + next_index = popped_entry.child_index + 1; + if (self->stack.size == 0) return; + tree = array_back(&self->stack)->tree; + } while (ts_subtree_child_count(tree) <= next_index); + + array_push(&self->stack, ((StackEntry) { + .tree = ts_subtree_children(tree)[next_index], + .child_index = next_index, + .byte_offset = byte_offset, + })); +} + +static inline bool reusable_node_descend(ReusableNode *self) { + StackEntry last_entry = *array_back(&self->stack); + if (ts_subtree_child_count(last_entry.tree) > 0) { + array_push(&self->stack, ((StackEntry) { + .tree = ts_subtree_children(last_entry.tree)[0], + .child_index = 0, + .byte_offset = last_entry.byte_offset, + })); + return true; + } else { + return false; + } +} + +static inline void reusable_node_advance_past_leaf(ReusableNode *self) { + while (reusable_node_descend(self)) {} + reusable_node_advance(self); +} + +static inline void reusable_node_reset(ReusableNode *self, Subtree tree) { + reusable_node_clear(self); + array_push(&self->stack, ((StackEntry) { + .tree = tree, + .child_index = 0, + .byte_offset = 0, + })); + + // Never reuse the root node, because it has a non-standard internal structure + // due to transformations that are applied when it is accepted: adding the EOF + // child and any extra children. + if (!reusable_node_descend(self)) { + reusable_node_clear(self); + } +} diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/stack.c b/Externals/crystaledit/editlib/tree-sitter/lib/src/stack.c new file mode 100644 index 00000000000..91420074760 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/stack.c @@ -0,0 +1,912 @@ +#include "./alloc.h" +#include "./language.h" +#include "./subtree.h" +#include "./array.h" +#include "./stack.h" +#include "./length.h" +#include +#include +#include + +#define MAX_LINK_COUNT 8 +#define MAX_NODE_POOL_SIZE 50 +#define MAX_ITERATOR_COUNT 64 + +#if defined _WIN32 && !defined __GNUC__ +#define forceinline __forceinline +#else +#define forceinline static inline __attribute__((always_inline)) +#endif + +typedef struct StackNode StackNode; + +typedef struct { + StackNode *node; + Subtree subtree; + bool is_pending; +} StackLink; + +struct StackNode { + TSStateId state; + Length position; + StackLink links[MAX_LINK_COUNT]; + short unsigned int link_count; + uint32_t ref_count; + unsigned error_cost; + unsigned node_count; + int dynamic_precedence; +}; + +typedef struct { + StackNode *node; + SubtreeArray subtrees; + uint32_t subtree_count; + bool is_pending; +} StackIterator; + +typedef Array(StackNode *) StackNodeArray; + +typedef enum { + StackStatusActive, + StackStatusPaused, + StackStatusHalted, +} StackStatus; + +typedef struct { + StackNode *node; + StackSummary *summary; + unsigned node_count_at_last_error; + Subtree last_external_token; + Subtree lookahead_when_paused; + StackStatus status; +} StackHead; + +struct Stack { + Array(StackHead) heads; + StackSliceArray slices; + Array(StackIterator) iterators; + StackNodeArray node_pool; + StackNode *base_node; + SubtreePool *subtree_pool; +}; + +typedef unsigned StackAction; +enum { + StackActionNone, + StackActionStop = 1, + StackActionPop = 2, +}; + +typedef StackAction (*StackCallback)(void *, const StackIterator *); + +static void stack_node_retain(StackNode *self) { + if (!self) + return; + ts_assert(self->ref_count > 0); + self->ref_count++; + ts_assert(self->ref_count != 0); +} + +static void stack_node_release( + StackNode *self, + StackNodeArray *pool, + SubtreePool *subtree_pool +) { +recur: + ts_assert(self->ref_count != 0); + self->ref_count--; + if (self->ref_count > 0) return; + + StackNode *first_predecessor = NULL; + if (self->link_count > 0) { + for (unsigned i = self->link_count - 1; i > 0; i--) { + StackLink link = self->links[i]; + if (link.subtree.ptr) ts_subtree_release(subtree_pool, link.subtree); + stack_node_release(link.node, pool, subtree_pool); + } + StackLink link = self->links[0]; + if (link.subtree.ptr) ts_subtree_release(subtree_pool, link.subtree); + first_predecessor = self->links[0].node; + } + + if (pool->size < MAX_NODE_POOL_SIZE) { + array_push(pool, self); + } else { + ts_free(self); + } + + if (first_predecessor) { + self = first_predecessor; + goto recur; + } +} + +/// Get the number of nodes in the subtree, for the purpose of measuring +/// how much progress has been made by a given version of the stack. +static uint32_t stack__subtree_node_count(Subtree subtree) { + uint32_t count = ts_subtree_visible_descendant_count(subtree); + if (ts_subtree_visible(subtree)) count++; + + // Count intermediate error nodes even though they are not visible, + // because a stack version's node count is used to check whether it + // has made any progress since the last time it encountered an error. + if (ts_subtree_symbol(subtree) == ts_builtin_sym_error_repeat) count++; + + return count; +} + +static StackNode *stack_node_new( + StackNode *previous_node, + Subtree subtree, + bool is_pending, + TSStateId state, + StackNodeArray *pool +) { + StackNode *node = pool->size > 0 + ? array_pop(pool) + : ts_malloc(sizeof(StackNode)); + *node = (StackNode) { + .ref_count = 1, + .link_count = 0, + .state = state + }; + + if (previous_node) { + node->link_count = 1; + node->links[0] = (StackLink) { + .node = previous_node, + .subtree = subtree, + .is_pending = is_pending, + }; + + node->position = previous_node->position; + node->error_cost = previous_node->error_cost; + node->dynamic_precedence = previous_node->dynamic_precedence; + node->node_count = previous_node->node_count; + + if (subtree.ptr) { + node->error_cost += ts_subtree_error_cost(subtree); + node->position = length_add(node->position, ts_subtree_total_size(subtree)); + node->node_count += stack__subtree_node_count(subtree); + node->dynamic_precedence += ts_subtree_dynamic_precedence(subtree); + } + } else { + node->position = length_zero(); + node->error_cost = 0; + } + + return node; +} + +static bool stack__subtree_is_equivalent(Subtree left, Subtree right) { + if (left.ptr == right.ptr) return true; + if (!left.ptr || !right.ptr) return false; + + // Symbols must match + if (ts_subtree_symbol(left) != ts_subtree_symbol(right)) return false; + + // If both have errors, don't bother keeping both. + if (ts_subtree_error_cost(left) > 0 && ts_subtree_error_cost(right) > 0) return true; + + return ( + ts_subtree_padding(left).bytes == ts_subtree_padding(right).bytes && + ts_subtree_size(left).bytes == ts_subtree_size(right).bytes && + ts_subtree_child_count(left) == ts_subtree_child_count(right) && + ts_subtree_extra(left) == ts_subtree_extra(right) && + ts_subtree_external_scanner_state_eq(left, right) + ); +} + +static void stack_node_add_link( + StackNode *self, + StackLink link, + SubtreePool *subtree_pool +) { + if (link.node == self) return; + + for (int i = 0; i < self->link_count; i++) { + StackLink *existing_link = &self->links[i]; + if (stack__subtree_is_equivalent(existing_link->subtree, link.subtree)) { + // In general, we preserve ambiguities until they are removed from the stack + // during a pop operation where multiple paths lead to the same node. But in + // the special case where two links directly connect the same pair of nodes, + // we can safely remove the ambiguity ahead of time without changing behavior. + if (existing_link->node == link.node) { + if ( + ts_subtree_dynamic_precedence(link.subtree) > + ts_subtree_dynamic_precedence(existing_link->subtree) + ) { + ts_subtree_retain(link.subtree); + ts_subtree_release(subtree_pool, existing_link->subtree); + existing_link->subtree = link.subtree; + self->dynamic_precedence = + link.node->dynamic_precedence + ts_subtree_dynamic_precedence(link.subtree); + } + return; + } + + // If the previous nodes are mergeable, merge them recursively. + if ( + existing_link->node->state == link.node->state && + existing_link->node->position.bytes == link.node->position.bytes && + existing_link->node->error_cost == link.node->error_cost + ) { + for (int j = 0; j < link.node->link_count; j++) { + stack_node_add_link(existing_link->node, link.node->links[j], subtree_pool); + } + int32_t dynamic_precedence = link.node->dynamic_precedence; + if (link.subtree.ptr) { + dynamic_precedence += ts_subtree_dynamic_precedence(link.subtree); + } + if (dynamic_precedence > self->dynamic_precedence) { + self->dynamic_precedence = dynamic_precedence; + } + return; + } + } + } + + if (self->link_count == MAX_LINK_COUNT) return; + + stack_node_retain(link.node); + unsigned node_count = link.node->node_count; + int dynamic_precedence = link.node->dynamic_precedence; + self->links[self->link_count++] = link; + + if (link.subtree.ptr) { + ts_subtree_retain(link.subtree); + node_count += stack__subtree_node_count(link.subtree); + dynamic_precedence += ts_subtree_dynamic_precedence(link.subtree); + } + + if (node_count > self->node_count) self->node_count = node_count; + if (dynamic_precedence > self->dynamic_precedence) self->dynamic_precedence = dynamic_precedence; +} + +static void stack_head_delete( + StackHead *self, + StackNodeArray *pool, + SubtreePool *subtree_pool +) { + if (self->node) { + if (self->last_external_token.ptr) { + ts_subtree_release(subtree_pool, self->last_external_token); + } + if (self->lookahead_when_paused.ptr) { + ts_subtree_release(subtree_pool, self->lookahead_when_paused); + } + if (self->summary) { + array_delete(self->summary); + ts_free(self->summary); + } + stack_node_release(self->node, pool, subtree_pool); + } +} + +static StackVersion ts_stack__add_version( + Stack *self, + StackVersion original_version, + StackNode *node +) { + StackHead head = { + .node = node, + .node_count_at_last_error = array_get(&self->heads, original_version)->node_count_at_last_error, + .last_external_token = array_get(&self->heads, original_version)->last_external_token, + .status = StackStatusActive, + .lookahead_when_paused = NULL_SUBTREE, + }; + array_push(&self->heads, head); + stack_node_retain(node); + if (head.last_external_token.ptr) ts_subtree_retain(head.last_external_token); + return (StackVersion)(self->heads.size - 1); +} + +static void ts_stack__add_slice( + Stack *self, + StackVersion original_version, + StackNode *node, + SubtreeArray *subtrees +) { + for (uint32_t i = self->slices.size - 1; i + 1 > 0; i--) { + StackVersion version = array_get(&self->slices, i)->version; + if (array_get(&self->heads, version)->node == node) { + StackSlice slice = {*subtrees, version}; + array_insert(&self->slices, i + 1, slice); + return; + } + } + + StackVersion version = ts_stack__add_version(self, original_version, node); + StackSlice slice = { *subtrees, version }; + array_push(&self->slices, slice); +} + +static StackSliceArray stack__iter( + Stack *self, + StackVersion version, + StackCallback callback, + void *payload, + int goal_subtree_count +) { + array_clear(&self->slices); + array_clear(&self->iterators); + + StackHead *head = array_get(&self->heads, version); + StackIterator new_iterator = { + .node = head->node, + .subtrees = array_new(), + .subtree_count = 0, + .is_pending = true, + }; + + bool include_subtrees = false; + if (goal_subtree_count >= 0) { + include_subtrees = true; + array_reserve(&new_iterator.subtrees, (uint32_t)ts_subtree_alloc_size(goal_subtree_count) / sizeof(Subtree)); + } + + array_push(&self->iterators, new_iterator); + + while (self->iterators.size > 0) { + for (uint32_t i = 0, size = self->iterators.size; i < size; i++) { + StackIterator *iterator = array_get(&self->iterators, i); + StackNode *node = iterator->node; + + StackAction action = callback(payload, iterator); + bool should_pop = action & StackActionPop; + bool should_stop = action & StackActionStop || node->link_count == 0; + + if (should_pop) { + SubtreeArray subtrees = iterator->subtrees; + if (!should_stop) { + ts_subtree_array_copy(subtrees, &subtrees); + } + ts_subtree_array_reverse(&subtrees); + ts_stack__add_slice( + self, + version, + node, + &subtrees + ); + } + + if (should_stop) { + if (!should_pop) { + ts_subtree_array_delete(self->subtree_pool, &iterator->subtrees); + } + array_erase(&self->iterators, i); + i--, size--; + continue; + } + + for (uint32_t j = 1; j <= node->link_count; j++) { + StackIterator *next_iterator; + StackLink link; + if (j == node->link_count) { + link = node->links[0]; + next_iterator = array_get(&self->iterators, i); + } else { + if (self->iterators.size >= MAX_ITERATOR_COUNT) continue; + link = node->links[j]; + StackIterator current_iterator = *array_get(&self->iterators, i); + array_push(&self->iterators, current_iterator); + next_iterator = array_back(&self->iterators); + ts_subtree_array_copy(next_iterator->subtrees, &next_iterator->subtrees); + } + + next_iterator->node = link.node; + if (link.subtree.ptr) { + if (include_subtrees) { + array_push(&next_iterator->subtrees, link.subtree); + ts_subtree_retain(link.subtree); + } + + if (!ts_subtree_extra(link.subtree)) { + next_iterator->subtree_count++; + if (!link.is_pending) { + next_iterator->is_pending = false; + } + } + } else { + next_iterator->subtree_count++; + next_iterator->is_pending = false; + } + } + } + } + + return self->slices; +} + +Stack *ts_stack_new(SubtreePool *subtree_pool) { + Stack *self = ts_calloc(1, sizeof(Stack)); + + array_init(&self->heads); + array_init(&self->slices); + array_init(&self->iterators); + array_init(&self->node_pool); + array_reserve(&self->heads, 4); + array_reserve(&self->slices, 4); + array_reserve(&self->iterators, 4); + array_reserve(&self->node_pool, MAX_NODE_POOL_SIZE); + + self->subtree_pool = subtree_pool; + self->base_node = stack_node_new(NULL, NULL_SUBTREE, false, 1, &self->node_pool); + ts_stack_clear(self); + + return self; +} + +void ts_stack_delete(Stack *self) { + if (self->slices.contents) + array_delete(&self->slices); + if (self->iterators.contents) + array_delete(&self->iterators); + stack_node_release(self->base_node, &self->node_pool, self->subtree_pool); + for (uint32_t i = 0; i < self->heads.size; i++) { + stack_head_delete(array_get(&self->heads, i), &self->node_pool, self->subtree_pool); + } + array_clear(&self->heads); + if (self->node_pool.contents) { + for (uint32_t i = 0; i < self->node_pool.size; i++) + ts_free(*array_get(&self->node_pool, i)); + array_delete(&self->node_pool); + } + array_delete(&self->heads); + ts_free(self); +} + +uint32_t ts_stack_version_count(const Stack *self) { + return self->heads.size; +} + +uint32_t ts_stack_halted_version_count(Stack *self) { + uint32_t count = 0; + for (uint32_t i = 0; i < self->heads.size; i++) { + StackHead *head = array_get(&self->heads, i); + if (head->status == StackStatusHalted) { + count++; + } + } + return count; +} + +TSStateId ts_stack_state(const Stack *self, StackVersion version) { + return array_get(&self->heads, version)->node->state; +} + +Length ts_stack_position(const Stack *self, StackVersion version) { + return array_get(&self->heads, version)->node->position; +} + +Subtree ts_stack_last_external_token(const Stack *self, StackVersion version) { + return array_get(&self->heads, version)->last_external_token; +} + +void ts_stack_set_last_external_token(Stack *self, StackVersion version, Subtree token) { + StackHead *head = array_get(&self->heads, version); + if (token.ptr) ts_subtree_retain(token); + if (head->last_external_token.ptr) ts_subtree_release(self->subtree_pool, head->last_external_token); + head->last_external_token = token; +} + +unsigned ts_stack_error_cost(const Stack *self, StackVersion version) { + StackHead *head = array_get(&self->heads, version); + unsigned result = head->node->error_cost; + if ( + head->status == StackStatusPaused || + (head->node->state == ERROR_STATE && !head->node->links[0].subtree.ptr)) { + result += ERROR_COST_PER_RECOVERY; + } + return result; +} + +unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version) { + StackHead *head = array_get(&self->heads, version); + if (head->node->node_count < head->node_count_at_last_error) { + head->node_count_at_last_error = head->node->node_count; + } + return head->node->node_count - head->node_count_at_last_error; +} + +void ts_stack_push( + Stack *self, + StackVersion version, + Subtree subtree, + bool pending, + TSStateId state +) { + StackHead *head = array_get(&self->heads, version); + StackNode *new_node = stack_node_new(head->node, subtree, pending, state, &self->node_pool); + if (!subtree.ptr) head->node_count_at_last_error = new_node->node_count; + head->node = new_node; +} + +forceinline StackAction pop_count_callback(void *payload, const StackIterator *iterator) { + unsigned *goal_subtree_count = payload; + if (iterator->subtree_count == *goal_subtree_count) { + return StackActionPop | StackActionStop; + } else { + return StackActionNone; + } +} + +StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, uint32_t count) { + return stack__iter(self, version, pop_count_callback, &count, (int)count); +} + + +forceinline StackAction pop_pending_callback(void *payload, const StackIterator *iterator) { + (void)payload; + if (iterator->subtree_count >= 1) { + if (iterator->is_pending) { + return StackActionPop | StackActionStop; + } else { + return StackActionStop; + } + } else { + return StackActionNone; + } +} + +StackSliceArray ts_stack_pop_pending(Stack *self, StackVersion version) { + StackSliceArray pop = stack__iter(self, version, pop_pending_callback, NULL, 0); + if (pop.size > 0) { + ts_stack_renumber_version(self, array_get(&pop, 0)->version, version); + array_get(&pop, 0)->version = version; + } + return pop; +} + +forceinline StackAction pop_error_callback(void *payload, const StackIterator *iterator) { + if (iterator->subtrees.size > 0) { + bool *found_error = payload; + if (!*found_error && ts_subtree_is_error(*array_get(&iterator->subtrees, 0))) { + *found_error = true; + return StackActionPop | StackActionStop; + } else { + return StackActionStop; + } + } else { + return StackActionNone; + } +} + +SubtreeArray ts_stack_pop_error(Stack *self, StackVersion version) { + StackNode *node = array_get(&self->heads, version)->node; + for (unsigned i = 0; i < node->link_count; i++) { + if (node->links[i].subtree.ptr && ts_subtree_is_error(node->links[i].subtree)) { + bool found_error = false; + StackSliceArray pop = stack__iter(self, version, pop_error_callback, &found_error, 1); + if (pop.size > 0) { + ts_assert(pop.size == 1); + ts_stack_renumber_version(self, array_get(&pop, 0)->version, version); + return array_get(&pop, 0)->subtrees; + } + break; + } + } + return (SubtreeArray) {.size = 0}; +} + +forceinline StackAction pop_all_callback(void *payload, const StackIterator *iterator) { + (void)payload; + return iterator->node->link_count == 0 ? StackActionPop : StackActionNone; +} + +StackSliceArray ts_stack_pop_all(Stack *self, StackVersion version) { + return stack__iter(self, version, pop_all_callback, NULL, 0); +} + +typedef struct { + StackSummary *summary; + unsigned max_depth; +} SummarizeStackSession; + +forceinline StackAction summarize_stack_callback(void *payload, const StackIterator *iterator) { + SummarizeStackSession *session = payload; + TSStateId state = iterator->node->state; + unsigned depth = iterator->subtree_count; + if (depth > session->max_depth) return StackActionStop; + for (unsigned i = session->summary->size - 1; i + 1 > 0; i--) { + StackSummaryEntry entry = *array_get(session->summary, i); + if (entry.depth < depth) break; + if (entry.depth == depth && entry.state == state) return StackActionNone; + } + array_push(session->summary, ((StackSummaryEntry) { + .position = iterator->node->position, + .depth = depth, + .state = state, + })); + return StackActionNone; +} + +void ts_stack_record_summary(Stack *self, StackVersion version, unsigned max_depth) { + SummarizeStackSession session = { + .summary = ts_malloc(sizeof(StackSummary)), + .max_depth = max_depth + }; + array_init(session.summary); + stack__iter(self, version, summarize_stack_callback, &session, -1); + StackHead *head = array_get(&self->heads, version); + if (head->summary) { + array_delete(head->summary); + ts_free(head->summary); + } + head->summary = session.summary; +} + +StackSummary *ts_stack_get_summary(Stack *self, StackVersion version) { + return array_get(&self->heads, version)->summary; +} + +int ts_stack_dynamic_precedence(Stack *self, StackVersion version) { + return array_get(&self->heads, version)->node->dynamic_precedence; +} + +bool ts_stack_has_advanced_since_error(const Stack *self, StackVersion version) { + const StackHead *head = array_get(&self->heads, version); + const StackNode *node = head->node; + if (node->error_cost == 0) return true; + while (node) { + if (node->link_count > 0) { + Subtree subtree = node->links[0].subtree; + if (subtree.ptr) { + if (ts_subtree_total_bytes(subtree) > 0) { + return true; + } else if ( + node->node_count > head->node_count_at_last_error && + ts_subtree_error_cost(subtree) == 0 + ) { + node = node->links[0].node; + continue; + } + } + } + break; + } + return false; +} + +void ts_stack_remove_version(Stack *self, StackVersion version) { + stack_head_delete(array_get(&self->heads, version), &self->node_pool, self->subtree_pool); + array_erase(&self->heads, version); +} + +void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) { + if (v1 == v2) return; + ts_assert(v2 < v1); + ts_assert((uint32_t)v1 < self->heads.size); + StackHead *source_head = array_get(&self->heads, v1); + StackHead *target_head = array_get(&self->heads, v2); + if (target_head->summary && !source_head->summary) { + source_head->summary = target_head->summary; + target_head->summary = NULL; + } + stack_head_delete(target_head, &self->node_pool, self->subtree_pool); + *target_head = *source_head; + array_erase(&self->heads, v1); +} + +void ts_stack_swap_versions(Stack *self, StackVersion v1, StackVersion v2) { + StackHead temporary_head = *array_get(&self->heads, v1); + *array_get(&self->heads, v1) = *array_get(&self->heads, v2); + *array_get(&self->heads, v2) = temporary_head; +} + +StackVersion ts_stack_copy_version(Stack *self, StackVersion version) { + ts_assert(version < self->heads.size); + StackHead version_head = *array_get(&self->heads, version); + array_push(&self->heads, version_head); + StackHead *head = array_back(&self->heads); + stack_node_retain(head->node); + if (head->last_external_token.ptr) ts_subtree_retain(head->last_external_token); + head->summary = NULL; + return self->heads.size - 1; +} + +bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2) { + if (!ts_stack_can_merge(self, version1, version2)) return false; + StackHead *head1 = array_get(&self->heads, version1); + StackHead *head2 = array_get(&self->heads, version2); + for (uint32_t i = 0; i < head2->node->link_count; i++) { + stack_node_add_link(head1->node, head2->node->links[i], self->subtree_pool); + } + if (head1->node->state == ERROR_STATE) { + head1->node_count_at_last_error = head1->node->node_count; + } + ts_stack_remove_version(self, version2); + return true; +} + +bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version2) { + StackHead *head1 = array_get(&self->heads, version1); + StackHead *head2 = array_get(&self->heads, version2); + return + head1->status == StackStatusActive && + head2->status == StackStatusActive && + head1->node->state == head2->node->state && + head1->node->position.bytes == head2->node->position.bytes && + head1->node->error_cost == head2->node->error_cost && + ts_subtree_external_scanner_state_eq(head1->last_external_token, head2->last_external_token); +} + +void ts_stack_halt(Stack *self, StackVersion version) { + array_get(&self->heads, version)->status = StackStatusHalted; +} + +void ts_stack_pause(Stack *self, StackVersion version, Subtree lookahead) { + StackHead *head = array_get(&self->heads, version); + head->status = StackStatusPaused; + head->lookahead_when_paused = lookahead; + head->node_count_at_last_error = head->node->node_count; +} + +bool ts_stack_is_active(const Stack *self, StackVersion version) { + return array_get(&self->heads, version)->status == StackStatusActive; +} + +bool ts_stack_is_halted(const Stack *self, StackVersion version) { + return array_get(&self->heads, version)->status == StackStatusHalted; +} + +bool ts_stack_is_paused(const Stack *self, StackVersion version) { + return array_get(&self->heads, version)->status == StackStatusPaused; +} + +Subtree ts_stack_resume(Stack *self, StackVersion version) { + StackHead *head = array_get(&self->heads, version); + ts_assert(head->status == StackStatusPaused); + Subtree result = head->lookahead_when_paused; + head->status = StackStatusActive; + head->lookahead_when_paused = NULL_SUBTREE; + return result; +} + +void ts_stack_clear(Stack *self) { + stack_node_retain(self->base_node); + for (uint32_t i = 0; i < self->heads.size; i++) { + stack_head_delete(array_get(&self->heads, i), &self->node_pool, self->subtree_pool); + } + array_clear(&self->heads); + array_push(&self->heads, ((StackHead) { + .node = self->base_node, + .status = StackStatusActive, + .last_external_token = NULL_SUBTREE, + .lookahead_when_paused = NULL_SUBTREE, + })); +} + +bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) { + array_reserve(&self->iterators, 32); + if (!f) f = stderr; + + fprintf(f, "digraph stack {\n"); + fprintf(f, "rankdir=\"RL\";\n"); + fprintf(f, "edge [arrowhead=none]\n"); + + Array(StackNode *) visited_nodes = array_new(); + + array_clear(&self->iterators); + for (uint32_t i = 0; i < self->heads.size; i++) { + StackHead *head = array_get(&self->heads, i); + if (head->status == StackStatusHalted) continue; + + fprintf(f, "node_head_%u [shape=none, label=\"\"]\n", i); + fprintf(f, "node_head_%u -> node_%p [", i, (void *)head->node); + + if (head->status == StackStatusPaused) { + fprintf(f, "color=red "); + } + fprintf(f, + "label=%u, fontcolor=blue, weight=10000, labeltooltip=\"node_count: %u\nerror_cost: %u", + i, + ts_stack_node_count_since_error(self, i), + ts_stack_error_cost(self, i) + ); + + if (head->summary) { + fprintf(f, "\nsummary:"); + for (uint32_t j = 0; j < head->summary->size; j++) fprintf(f, " %u", array_get(head->summary, j)->state); + } + + if (head->last_external_token.ptr) { + const ExternalScannerState *state = &head->last_external_token.ptr->external_scanner_state; + const char *data = ts_external_scanner_state_data(state); + fprintf(f, "\nexternal_scanner_state:"); + for (uint32_t j = 0; j < state->length; j++) fprintf(f, " %2X", data[j]); + } + + fprintf(f, "\"]\n"); + array_push(&self->iterators, ((StackIterator) { + .node = head->node + })); + } + + bool all_iterators_done = false; + while (!all_iterators_done) { + all_iterators_done = true; + + for (uint32_t i = 0; i < self->iterators.size; i++) { + StackIterator iterator = *array_get(&self->iterators, i); + StackNode *node = iterator.node; + + for (uint32_t j = 0; j < visited_nodes.size; j++) { + if (*array_get(&visited_nodes, j) == node) { + node = NULL; + break; + } + } + + if (!node) continue; + all_iterators_done = false; + + fprintf(f, "node_%p [", (void *)node); + if (node->state == ERROR_STATE) { + fprintf(f, "label=\"?\""); + } else if ( + node->link_count == 1 && + node->links[0].subtree.ptr && + ts_subtree_extra(node->links[0].subtree) + ) { + fprintf(f, "shape=point margin=0 label=\"\""); + } else { + fprintf(f, "label=\"%d\"", node->state); + } + + fprintf( + f, + " tooltip=\"position: %u,%u\nnode_count:%u\nerror_cost: %u\ndynamic_precedence: %d\"];\n", + node->position.extent.row + 1, + node->position.extent.column, + node->node_count, + node->error_cost, + node->dynamic_precedence + ); + + for (int j = 0; j < node->link_count; j++) { + StackLink link = node->links[j]; + fprintf(f, "node_%p -> node_%p [", (void *)node, (void *)link.node); + if (link.is_pending) fprintf(f, "style=dashed "); + if (link.subtree.ptr && ts_subtree_extra(link.subtree)) fprintf(f, "fontcolor=gray "); + + if (!link.subtree.ptr) { + fprintf(f, "color=red"); + } else { + fprintf(f, "label=\""); + bool quoted = ts_subtree_visible(link.subtree) && !ts_subtree_named(link.subtree); + if (quoted) fprintf(f, "'"); + ts_language_write_symbol_as_dot_string(language, f, ts_subtree_symbol(link.subtree)); + if (quoted) fprintf(f, "'"); + fprintf(f, "\""); + fprintf( + f, + "labeltooltip=\"error_cost: %u\ndynamic_precedence: %" PRId32 "\"", + ts_subtree_error_cost(link.subtree), + ts_subtree_dynamic_precedence(link.subtree) + ); + } + + fprintf(f, "];\n"); + + StackIterator *next_iterator; + if (j == 0) { + next_iterator = array_get(&self->iterators, i); + } else { + array_push(&self->iterators, iterator); + next_iterator = array_back(&self->iterators); + } + next_iterator->node = link.node; + } + + array_push(&visited_nodes, node); + } + } + + fprintf(f, "}\n"); + + array_delete(&visited_nodes); + return true; +} + +#undef forceinline diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/stack.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/stack.h new file mode 100644 index 00000000000..2619f1e8a54 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/stack.h @@ -0,0 +1,133 @@ +#ifndef TREE_SITTER_PARSE_STACK_H_ +#define TREE_SITTER_PARSE_STACK_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "./array.h" +#include "./subtree.h" +#include + +typedef struct Stack Stack; + +typedef unsigned StackVersion; +#define STACK_VERSION_NONE ((StackVersion)-1) + +typedef struct { + SubtreeArray subtrees; + StackVersion version; +} StackSlice; +typedef Array(StackSlice) StackSliceArray; + +typedef struct { + Length position; + unsigned depth; + TSStateId state; +} StackSummaryEntry; +typedef Array(StackSummaryEntry) StackSummary; + +// Create a stack. +Stack *ts_stack_new(SubtreePool *subtree_pool); + +// Release the memory reserved for a given stack. +void ts_stack_delete(Stack *self); + +// Get the stack's current number of versions. +uint32_t ts_stack_version_count(const Stack *self); + +// Get the stack's current number of halted versions. +uint32_t ts_stack_halted_version_count(Stack *self); + +// Get the state at the top of the given version of the stack. If the stack is +// empty, this returns the initial state, 0. +TSStateId ts_stack_state(const Stack *self, StackVersion version); + +// Get the last external token associated with a given version of the stack. +Subtree ts_stack_last_external_token(const Stack *self, StackVersion version); + +// Set the last external token associated with a given version of the stack. +void ts_stack_set_last_external_token(Stack *self, StackVersion version, Subtree token); + +// Get the position of the given version of the stack within the document. +Length ts_stack_position(const Stack *, StackVersion); + +// Push a tree and state onto the given version of the stack. +// +// This transfers ownership of the tree to the Stack. Callers that +// need to retain ownership of the tree for their own purposes should +// first retain the tree. +void ts_stack_push(Stack *self, StackVersion version, Subtree subtree, bool pending, TSStateId state); + +// Pop the given number of entries from the given version of the stack. This +// operation can increase the number of stack versions by revealing multiple +// versions which had previously been merged. It returns an array that +// specifies the index of each revealed version and the trees that were +// removed from that version. +StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, uint32_t count); + +// Remove an error at the top of the given version of the stack. +SubtreeArray ts_stack_pop_error(Stack *self, StackVersion version); + +// Remove any pending trees from the top of the given version of the stack. +StackSliceArray ts_stack_pop_pending(Stack *self, StackVersion version); + +// Remove all trees from the given version of the stack. +StackSliceArray ts_stack_pop_all(Stack *self, StackVersion version); + +// Get the maximum number of tree nodes reachable from this version of the stack +// since the last error was detected. +unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version); + +int ts_stack_dynamic_precedence(Stack *self, StackVersion version); + +bool ts_stack_has_advanced_since_error(const Stack *self, StackVersion version); + +// Compute a summary of all the parse states near the top of the given +// version of the stack and store the summary for later retrieval. +void ts_stack_record_summary(Stack *self, StackVersion version, unsigned max_depth); + +// Retrieve a summary of all the parse states near the top of the +// given version of the stack. +StackSummary *ts_stack_get_summary(Stack *self, StackVersion version); + +// Get the total cost of all errors on the given version of the stack. +unsigned ts_stack_error_cost(const Stack *self, StackVersion version); + +// Merge the given two stack versions if possible, returning true +// if they were successfully merged and false otherwise. +bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2); + +// Determine whether the given two stack versions can be merged. +bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version2); + +Subtree ts_stack_resume(Stack *self, StackVersion version); + +void ts_stack_pause(Stack *self, StackVersion version, Subtree lookahead); + +void ts_stack_halt(Stack *self, StackVersion version); + +bool ts_stack_is_active(const Stack *self, StackVersion version); + +bool ts_stack_is_paused(const Stack *self, StackVersion version); + +bool ts_stack_is_halted(const Stack *self, StackVersion version); + +void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2); + +void ts_stack_swap_versions(Stack *, StackVersion v1, StackVersion v2); + +StackVersion ts_stack_copy_version(Stack *self, StackVersion version); + +// Remove the given version from the stack. +void ts_stack_remove_version(Stack *self, StackVersion version); + +void ts_stack_clear(Stack *self); + +bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f); + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_PARSE_STACK_H_ diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/subtree.c b/Externals/crystaledit/editlib/tree-sitter/lib/src/subtree.c new file mode 100644 index 00000000000..97d55c86112 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/subtree.c @@ -0,0 +1,1034 @@ +#include +#include +#include +#include +#include +#include "./alloc.h" +#include "./array.h" +#include "./atomic.h" +#include "./subtree.h" +#include "./length.h" +#include "./language.h" +#include "./error_costs.h" +#include "./ts_assert.h" +#include + +typedef struct { + Length start; + Length old_end; + Length new_end; +} Edit; + +#define TS_MAX_INLINE_TREE_LENGTH UINT8_MAX +#define TS_MAX_TREE_POOL_SIZE 32 + +// ExternalScannerState + +void ts_external_scanner_state_init(ExternalScannerState *self, const char *data, unsigned length) { + self->length = length; + if (length > sizeof(self->short_data)) { + self->long_data = ts_malloc(length); + memcpy(self->long_data, data, length); + } else { + memcpy(self->short_data, data, length); + } +} + +ExternalScannerState ts_external_scanner_state_copy(const ExternalScannerState *self) { + ExternalScannerState result = *self; + if (self->length > sizeof(self->short_data)) { + result.long_data = ts_malloc(self->length); + memcpy(result.long_data, self->long_data, self->length); + } + return result; +} + +void ts_external_scanner_state_delete(ExternalScannerState *self) { + if (self->length > sizeof(self->short_data)) { + ts_free(self->long_data); + } +} + +const char *ts_external_scanner_state_data(const ExternalScannerState *self) { + if (self->length > sizeof(self->short_data)) { + return self->long_data; + } else { + return self->short_data; + } +} + +bool ts_external_scanner_state_eq(const ExternalScannerState *self, const char *buffer, unsigned length) { + return + self->length == length && + memcmp(ts_external_scanner_state_data(self), buffer, length) == 0; +} + +// SubtreeArray + +void ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest) { + dest->size = self.size; + dest->capacity = self.capacity; + dest->contents = self.contents; + if (self.capacity > 0) { + dest->contents = ts_calloc(self.capacity, sizeof(Subtree)); + memcpy(dest->contents, self.contents, self.size * sizeof(Subtree)); + for (uint32_t i = 0; i < self.size; i++) { + ts_subtree_retain(*array_get(dest, i)); + } + } +} + +void ts_subtree_array_clear(SubtreePool *pool, SubtreeArray *self) { + for (uint32_t i = 0; i < self->size; i++) { + ts_subtree_release(pool, *array_get(self, i)); + } + array_clear(self); +} + +void ts_subtree_array_delete(SubtreePool *pool, SubtreeArray *self) { + ts_subtree_array_clear(pool, self); + array_delete(self); +} + +void ts_subtree_array_remove_trailing_extras( + SubtreeArray *self, + SubtreeArray *destination +) { + array_clear(destination); + while (self->size > 0) { + Subtree last = *array_get(self, self->size - 1); + if (ts_subtree_extra(last)) { + self->size--; + array_push(destination, last); + } else { + break; + } + } + ts_subtree_array_reverse(destination); +} + +void ts_subtree_array_reverse(SubtreeArray *self) { + for (uint32_t i = 0, limit = self->size / 2; i < limit; i++) { + size_t reverse_index = self->size - 1 - i; + Subtree swap = *array_get(self, i); + *array_get(self, i) = *array_get(self, reverse_index); + *array_get(self, reverse_index) = swap; + } +} + +// SubtreePool + +SubtreePool ts_subtree_pool_new(uint32_t capacity) { + SubtreePool self = {array_new(), array_new()}; + array_reserve(&self.free_trees, capacity); + return self; +} + +void ts_subtree_pool_delete(SubtreePool *self) { + if (self->free_trees.contents) { + for (unsigned i = 0; i < self->free_trees.size; i++) { + ts_free(array_get(&self->free_trees, i)->ptr); + } + array_delete(&self->free_trees); + } + if (self->tree_stack.contents) array_delete(&self->tree_stack); +} + +static SubtreeHeapData *ts_subtree_pool_allocate(SubtreePool *self) { + if (self->free_trees.size > 0) { + return array_pop(&self->free_trees).ptr; + } else { + return ts_malloc(sizeof(SubtreeHeapData)); + } +} + +static void ts_subtree_pool_free(SubtreePool *self, SubtreeHeapData *tree) { + if (self->free_trees.capacity > 0 && self->free_trees.size + 1 <= TS_MAX_TREE_POOL_SIZE) { + array_push(&self->free_trees, (MutableSubtree) {.ptr = tree}); + } else { + ts_free(tree); + } +} + +// Subtree + +static inline bool ts_subtree_can_inline(Length padding, Length size, uint32_t lookahead_bytes) { + return + padding.bytes < TS_MAX_INLINE_TREE_LENGTH && + padding.extent.row < 16 && + padding.extent.column < TS_MAX_INLINE_TREE_LENGTH && + size.bytes < TS_MAX_INLINE_TREE_LENGTH && + size.extent.row == 0 && + size.extent.column < TS_MAX_INLINE_TREE_LENGTH && + lookahead_bytes < 16; +} + +Subtree ts_subtree_new_leaf( + SubtreePool *pool, TSSymbol symbol, Length padding, Length size, + uint32_t lookahead_bytes, TSStateId parse_state, + bool has_external_tokens, bool depends_on_column, + bool is_keyword, const TSLanguage *language +) { + TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); + bool extra = symbol == ts_builtin_sym_end; + + bool is_inline = ( + symbol <= UINT8_MAX && + !has_external_tokens && + ts_subtree_can_inline(padding, size, lookahead_bytes) + ); + + if (is_inline) { + return (Subtree) {{ + .parse_state = parse_state, + .symbol = symbol, + .padding_bytes = padding.bytes, + .padding_rows = padding.extent.row, + .padding_columns = padding.extent.column, + .size_bytes = size.bytes, + .lookahead_bytes = lookahead_bytes, + .visible = metadata.visible, + .named = metadata.named, + .extra = extra, + .has_changes = false, + .is_missing = false, + .is_keyword = is_keyword, + .is_inline = true, + }}; + } else { + SubtreeHeapData *data = ts_subtree_pool_allocate(pool); + *data = (SubtreeHeapData) { + .ref_count = 1, + .padding = padding, + .size = size, + .lookahead_bytes = lookahead_bytes, + .error_cost = 0, + .child_count = 0, + .symbol = symbol, + .parse_state = parse_state, + .visible = metadata.visible, + .named = metadata.named, + .extra = extra, + .fragile_left = false, + .fragile_right = false, + .has_changes = false, + .has_external_tokens = has_external_tokens, + .has_external_scanner_state_change = false, + .depends_on_column = depends_on_column, + .is_missing = false, + .is_keyword = is_keyword, + {{.first_leaf = {.symbol = 0, .parse_state = 0}}} + }; + return (Subtree) {.ptr = data}; + } +} + +void ts_subtree_set_symbol( + MutableSubtree *self, + TSSymbol symbol, + const TSLanguage *language +) { + TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); + if (self->data.is_inline) { + ts_assert(symbol < UINT8_MAX); + self->data.symbol = symbol; + self->data.named = metadata.named; + self->data.visible = metadata.visible; + } else { + self->ptr->symbol = symbol; + self->ptr->named = metadata.named; + self->ptr->visible = metadata.visible; + } +} + +Subtree ts_subtree_new_error( + SubtreePool *pool, int32_t lookahead_char, Length padding, Length size, + uint32_t bytes_scanned, TSStateId parse_state, const TSLanguage *language +) { + Subtree result = ts_subtree_new_leaf( + pool, ts_builtin_sym_error, padding, size, bytes_scanned, + parse_state, false, false, false, language + ); + SubtreeHeapData *data = (SubtreeHeapData *)result.ptr; + data->fragile_left = true; + data->fragile_right = true; + data->lookahead_char = lookahead_char; + return result; +} + +// Clone a subtree. +MutableSubtree ts_subtree_clone(Subtree self) { + size_t alloc_size = ts_subtree_alloc_size(self.ptr->child_count); + Subtree *new_children = ts_malloc(alloc_size); + Subtree *old_children = ts_subtree_children(self); + memcpy(new_children, old_children, alloc_size); + SubtreeHeapData *result = (SubtreeHeapData *)&new_children[self.ptr->child_count]; + if (self.ptr->child_count > 0) { + for (uint32_t i = 0; i < self.ptr->child_count; i++) { + ts_subtree_retain(new_children[i]); + } + } else if (self.ptr->has_external_tokens) { + result->external_scanner_state = ts_external_scanner_state_copy( + &self.ptr->external_scanner_state + ); + } + result->ref_count = 1; + return (MutableSubtree) {.ptr = result}; +} + +// Get mutable version of a subtree. +// +// This takes ownership of the subtree. If the subtree has only one owner, +// this will directly convert it into a mutable version. Otherwise, it will +// perform a copy. +MutableSubtree ts_subtree_make_mut(SubtreePool *pool, Subtree self) { + if (self.data.is_inline) return (MutableSubtree) {self.data}; + if (self.ptr->ref_count == 1) return ts_subtree_to_mut_unsafe(self); + MutableSubtree result = ts_subtree_clone(self); + ts_subtree_release(pool, self); + return result; +} + +void ts_subtree_compress( + MutableSubtree self, + unsigned count, + const TSLanguage *language, + MutableSubtreeArray *stack +) { + unsigned initial_stack_size = stack->size; + + MutableSubtree tree = self; + TSSymbol symbol = tree.ptr->symbol; + for (unsigned i = 0; i < count; i++) { + if (tree.ptr->ref_count > 1 || tree.ptr->child_count < 2) break; + + MutableSubtree child = ts_subtree_to_mut_unsafe(ts_subtree_children(tree)[0]); + if ( + child.data.is_inline || + child.ptr->child_count < 2 || + child.ptr->ref_count > 1 || + child.ptr->symbol != symbol + ) break; + + MutableSubtree grandchild = ts_subtree_to_mut_unsafe(ts_subtree_children(child)[0]); + if ( + grandchild.data.is_inline || + grandchild.ptr->child_count < 2 || + grandchild.ptr->ref_count > 1 || + grandchild.ptr->symbol != symbol + ) break; + + ts_subtree_children(tree)[0] = ts_subtree_from_mut(grandchild); + ts_subtree_children(child)[0] = ts_subtree_children(grandchild)[grandchild.ptr->child_count - 1]; + ts_subtree_children(grandchild)[grandchild.ptr->child_count - 1] = ts_subtree_from_mut(child); + array_push(stack, tree); + tree = grandchild; + } + + while (stack->size > initial_stack_size) { + tree = array_pop(stack); + MutableSubtree child = ts_subtree_to_mut_unsafe(ts_subtree_children(tree)[0]); + MutableSubtree grandchild = ts_subtree_to_mut_unsafe(ts_subtree_children(child)[child.ptr->child_count - 1]); + ts_subtree_summarize_children(grandchild, language); + ts_subtree_summarize_children(child, language); + ts_subtree_summarize_children(tree, language); + } +} + +// Assign all of the node's properties that depend on its children. +void ts_subtree_summarize_children( + MutableSubtree self, + const TSLanguage *language +) { + ts_assert(!self.data.is_inline); + + self.ptr->named_child_count = 0; + self.ptr->visible_child_count = 0; + self.ptr->error_cost = 0; + self.ptr->repeat_depth = 0; + self.ptr->visible_descendant_count = 0; + self.ptr->has_external_tokens = false; + self.ptr->depends_on_column = false; + self.ptr->has_external_scanner_state_change = false; + self.ptr->dynamic_precedence = 0; + + uint32_t structural_index = 0; + const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self.ptr->production_id); + uint32_t lookahead_end_byte = 0; + + const Subtree *children = ts_subtree_children(self); + for (uint32_t i = 0; i < self.ptr->child_count; i++) { + Subtree child = children[i]; + + if ( + self.ptr->size.extent.row == 0 && + ts_subtree_depends_on_column(child) + ) { + self.ptr->depends_on_column = true; + } + + if (ts_subtree_has_external_scanner_state_change(child)) { + self.ptr->has_external_scanner_state_change = true; + } + + if (i == 0) { + self.ptr->padding = ts_subtree_padding(child); + self.ptr->size = ts_subtree_size(child); + } else { + self.ptr->size = length_add(self.ptr->size, ts_subtree_total_size(child)); + } + + uint32_t child_lookahead_end_byte = + self.ptr->padding.bytes + + self.ptr->size.bytes + + ts_subtree_lookahead_bytes(child); + if (child_lookahead_end_byte > lookahead_end_byte) { + lookahead_end_byte = child_lookahead_end_byte; + } + + if (ts_subtree_symbol(child) != ts_builtin_sym_error_repeat) { + self.ptr->error_cost += ts_subtree_error_cost(child); + } + + uint32_t grandchild_count = ts_subtree_child_count(child); + if ( + self.ptr->symbol == ts_builtin_sym_error || + self.ptr->symbol == ts_builtin_sym_error_repeat + ) { + if (!ts_subtree_extra(child) && !(ts_subtree_is_error(child) && grandchild_count == 0)) { + if (ts_subtree_visible(child)) { + self.ptr->error_cost += ERROR_COST_PER_SKIPPED_TREE; + } else if (grandchild_count > 0) { + self.ptr->error_cost += ERROR_COST_PER_SKIPPED_TREE * child.ptr->visible_child_count; + } + } + } + + self.ptr->dynamic_precedence += ts_subtree_dynamic_precedence(child); + self.ptr->visible_descendant_count += ts_subtree_visible_descendant_count(child); + + if ( + !ts_subtree_extra(child) && + ts_subtree_symbol(child) != 0 && + alias_sequence && + alias_sequence[structural_index] != 0 + ) { + self.ptr->visible_descendant_count++; + self.ptr->visible_child_count++; + if (ts_language_symbol_metadata(language, alias_sequence[structural_index]).named) { + self.ptr->named_child_count++; + } + } else if (ts_subtree_visible(child)) { + self.ptr->visible_descendant_count++; + self.ptr->visible_child_count++; + if (ts_subtree_named(child)) self.ptr->named_child_count++; + } else if (grandchild_count > 0) { + self.ptr->visible_child_count += child.ptr->visible_child_count; + self.ptr->named_child_count += child.ptr->named_child_count; + } + + if (ts_subtree_has_external_tokens(child)) self.ptr->has_external_tokens = true; + + if (ts_subtree_is_error(child)) { + self.ptr->fragile_left = self.ptr->fragile_right = true; + self.ptr->parse_state = TS_TREE_STATE_NONE; + } + + if (!ts_subtree_extra(child)) structural_index++; + } + + self.ptr->lookahead_bytes = lookahead_end_byte - self.ptr->size.bytes - self.ptr->padding.bytes; + + if ( + self.ptr->symbol == ts_builtin_sym_error || + self.ptr->symbol == ts_builtin_sym_error_repeat + ) { + self.ptr->error_cost += + ERROR_COST_PER_RECOVERY + + ERROR_COST_PER_SKIPPED_CHAR * self.ptr->size.bytes + + ERROR_COST_PER_SKIPPED_LINE * self.ptr->size.extent.row; + } + + if (self.ptr->child_count > 0) { + Subtree first_child = children[0]; + Subtree last_child = children[self.ptr->child_count - 1]; + + self.ptr->first_leaf.symbol = ts_subtree_leaf_symbol(first_child); + self.ptr->first_leaf.parse_state = ts_subtree_leaf_parse_state(first_child); + + if (ts_subtree_fragile_left(first_child)) self.ptr->fragile_left = true; + if (ts_subtree_fragile_right(last_child)) self.ptr->fragile_right = true; + + if ( + self.ptr->child_count >= 2 && + !self.ptr->visible && + !self.ptr->named && + ts_subtree_symbol(first_child) == self.ptr->symbol + ) { + if (ts_subtree_repeat_depth(first_child) > ts_subtree_repeat_depth(last_child)) { + self.ptr->repeat_depth = ts_subtree_repeat_depth(first_child) + 1; + } else { + self.ptr->repeat_depth = ts_subtree_repeat_depth(last_child) + 1; + } + } + } +} + +// Create a new parent node with the given children. +// +// This takes ownership of the children array. +MutableSubtree ts_subtree_new_node( + TSSymbol symbol, + SubtreeArray *children, + unsigned production_id, + const TSLanguage *language +) { + TSSymbolMetadata metadata = ts_language_symbol_metadata(language, symbol); + bool fragile = symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat; + + // Allocate the node's data at the end of the array of children. + size_t new_byte_size = ts_subtree_alloc_size(children->size); + if (children->capacity * sizeof(Subtree) < new_byte_size) { + children->contents = ts_realloc(children->contents, new_byte_size); + children->capacity = (uint32_t)(new_byte_size / sizeof(Subtree)); + } + SubtreeHeapData *data = (SubtreeHeapData *)&children->contents[children->size]; + + *data = (SubtreeHeapData) { + .ref_count = 1, + .symbol = symbol, + .child_count = children->size, + .visible = metadata.visible, + .named = metadata.named, + .has_changes = false, + .has_external_scanner_state_change = false, + .fragile_left = fragile, + .fragile_right = fragile, + .is_keyword = false, + {{ + .visible_descendant_count = 0, + .production_id = production_id, + .first_leaf = {.symbol = 0, .parse_state = 0}, + }} + }; + MutableSubtree result = {.ptr = data}; + ts_subtree_summarize_children(result, language); + return result; +} + +// Create a new error node containing the given children. +// +// This node is treated as 'extra'. Its children are prevented from having +// having any effect on the parse state. +Subtree ts_subtree_new_error_node( + SubtreeArray *children, + bool extra, + const TSLanguage *language +) { + MutableSubtree result = ts_subtree_new_node( + ts_builtin_sym_error, children, 0, language + ); + result.ptr->extra = extra; + return ts_subtree_from_mut(result); +} + +// Create a new 'missing leaf' node. +// +// This node is treated as 'extra'. Its children are prevented from having +// having any effect on the parse state. +Subtree ts_subtree_new_missing_leaf( + SubtreePool *pool, + TSSymbol symbol, + Length padding, + uint32_t lookahead_bytes, + const TSLanguage *language +) { + Subtree result = ts_subtree_new_leaf( + pool, symbol, padding, length_zero(), lookahead_bytes, + 0, false, false, false, language + ); + if (result.data.is_inline) { + result.data.is_missing = true; + } else { + ((SubtreeHeapData *)result.ptr)->is_missing = true; + } + return result; +} + +void ts_subtree_retain(Subtree self) { + if (self.data.is_inline) return; + ts_assert(self.ptr->ref_count > 0); + atomic_inc((volatile uint32_t *)&self.ptr->ref_count); + ts_assert(self.ptr->ref_count != 0); +} + +void ts_subtree_release(SubtreePool *pool, Subtree self) { + if (self.data.is_inline) return; + array_clear(&pool->tree_stack); + + ts_assert(self.ptr->ref_count > 0); + if (atomic_dec((volatile uint32_t *)&self.ptr->ref_count) == 0) { + array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(self)); + } + + while (pool->tree_stack.size > 0) { + MutableSubtree tree = array_pop(&pool->tree_stack); + if (tree.ptr->child_count > 0) { + Subtree *children = ts_subtree_children(tree); + for (uint32_t i = 0; i < tree.ptr->child_count; i++) { + Subtree child = children[i]; + if (child.data.is_inline) continue; + ts_assert(child.ptr->ref_count > 0); + if (atomic_dec((volatile uint32_t *)&child.ptr->ref_count) == 0) { + array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(child)); + } + } + ts_free(children); + } else { + if (tree.ptr->has_external_tokens) { + ts_external_scanner_state_delete(&tree.ptr->external_scanner_state); + } + ts_subtree_pool_free(pool, tree.ptr); + } + } +} + +int ts_subtree_compare(Subtree left, Subtree right, SubtreePool *pool) { + array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(left)); + array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(right)); + + while (pool->tree_stack.size > 0) { + right = ts_subtree_from_mut(array_pop(&pool->tree_stack)); + left = ts_subtree_from_mut(array_pop(&pool->tree_stack)); + + int result = 0; + if (ts_subtree_symbol(left) < ts_subtree_symbol(right)) result = -1; + else if (ts_subtree_symbol(right) < ts_subtree_symbol(left)) result = 1; + else if (ts_subtree_child_count(left) < ts_subtree_child_count(right)) result = -1; + else if (ts_subtree_child_count(right) < ts_subtree_child_count(left)) result = 1; + if (result != 0) { + array_clear(&pool->tree_stack); + return result; + } + + for (uint32_t i = ts_subtree_child_count(left); i > 0; i--) { + Subtree left_child = ts_subtree_children(left)[i - 1]; + Subtree right_child = ts_subtree_children(right)[i - 1]; + array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(left_child)); + array_push(&pool->tree_stack, ts_subtree_to_mut_unsafe(right_child)); + } + } + + return 0; +} + +static inline void ts_subtree_set_has_changes(MutableSubtree *self) { + if (self->data.is_inline) { + self->data.has_changes = true; + } else { + self->ptr->has_changes = true; + } +} + +Subtree ts_subtree_edit(Subtree self, const TSInputEdit *input_edit, SubtreePool *pool) { + typedef struct { + Subtree *tree; + Edit edit; + } EditEntry; + + Array(EditEntry) stack = array_new(); + array_push(&stack, ((EditEntry) { + .tree = &self, + .edit = (Edit) { + .start = {input_edit->start_byte, input_edit->start_point}, + .old_end = {input_edit->old_end_byte, input_edit->old_end_point}, + .new_end = {input_edit->new_end_byte, input_edit->new_end_point}, + }, + })); + + while (stack.size) { + EditEntry entry = array_pop(&stack); + Edit edit = entry.edit; + bool is_noop = edit.old_end.bytes == edit.start.bytes && edit.new_end.bytes == edit.start.bytes; + bool is_pure_insertion = edit.old_end.bytes == edit.start.bytes; + bool parent_depends_on_column = ts_subtree_depends_on_column(*entry.tree); + bool column_shifted = edit.new_end.extent.column != edit.old_end.extent.column; + + Length size = ts_subtree_size(*entry.tree); + Length padding = ts_subtree_padding(*entry.tree); + Length total_size = length_add(padding, size); + uint32_t lookahead_bytes = ts_subtree_lookahead_bytes(*entry.tree); + uint32_t end_byte = total_size.bytes + lookahead_bytes; + if (edit.start.bytes > end_byte || (is_noop && edit.start.bytes == end_byte)) continue; + + // If the edit is entirely within the space before this subtree, then shift this + // subtree over according to the edit without changing its size. + if (edit.old_end.bytes <= padding.bytes) { + padding = length_add(edit.new_end, length_sub(padding, edit.old_end)); + } + + // If the edit starts in the space before this subtree and extends into this subtree, + // shrink the subtree's content to compensate for the change in the space before it. + else if (edit.start.bytes < padding.bytes) { + size = length_saturating_sub(size, length_sub(edit.old_end, padding)); + padding = edit.new_end; + } + + // If the edit is within this subtree, resize the subtree to reflect the edit. + else if ( + edit.start.bytes < total_size.bytes || + (edit.start.bytes == total_size.bytes && is_pure_insertion) + ) { + size = length_add( + length_sub(edit.new_end, padding), + length_saturating_sub(total_size, edit.old_end) + ); + } + + MutableSubtree result = ts_subtree_make_mut(pool, *entry.tree); + + if (result.data.is_inline) { + if (ts_subtree_can_inline(padding, size, lookahead_bytes)) { + result.data.padding_bytes = padding.bytes; + result.data.padding_rows = padding.extent.row; + result.data.padding_columns = padding.extent.column; + result.data.size_bytes = size.bytes; + } else { + SubtreeHeapData *data = ts_subtree_pool_allocate(pool); + data->ref_count = 1; + data->padding = padding; + data->size = size; + data->lookahead_bytes = lookahead_bytes; + data->error_cost = 0; + data->child_count = 0; + data->symbol = result.data.symbol; + data->parse_state = result.data.parse_state; + data->visible = result.data.visible; + data->named = result.data.named; + data->extra = result.data.extra; + data->fragile_left = false; + data->fragile_right = false; + data->has_changes = false; + data->has_external_tokens = false; + data->depends_on_column = false; + data->is_missing = result.data.is_missing; + data->is_keyword = result.data.is_keyword; + result.ptr = data; + } + } else { + result.ptr->padding = padding; + result.ptr->size = size; + } + + ts_subtree_set_has_changes(&result); + *entry.tree = ts_subtree_from_mut(result); + + Length child_left, child_right = length_zero(); + for (uint32_t i = 0, n = ts_subtree_child_count(*entry.tree); i < n; i++) { + Subtree *child = &ts_subtree_children(*entry.tree)[i]; + Length child_size = ts_subtree_total_size(*child); + child_left = child_right; + child_right = length_add(child_left, child_size); + + // If this child ends before the edit, it is not affected. + if (child_right.bytes + ts_subtree_lookahead_bytes(*child) < edit.start.bytes) continue; + + // Keep editing child nodes until a node is reached that starts after the edit. + // Also, if this node's validity depends on its column position, then continue + // invalidating child nodes until reaching a line break. + if (( + (child_left.bytes > edit.old_end.bytes) || + (child_left.bytes == edit.old_end.bytes && child_size.bytes > 0 && i > 0) + ) && ( + !parent_depends_on_column || + child_left.extent.row > padding.extent.row + ) && ( + !ts_subtree_depends_on_column(*child) || + !column_shifted || + child_left.extent.row > edit.old_end.extent.row + )) { + break; + } + + // Transform edit into the child's coordinate space. + Edit child_edit = { + .start = length_saturating_sub(edit.start, child_left), + .old_end = length_saturating_sub(edit.old_end, child_left), + .new_end = length_saturating_sub(edit.new_end, child_left), + }; + + // Interpret all inserted text as applying to the *first* child that touches the edit. + // Subsequent children are only never have any text inserted into them; they are only + // shrunk to compensate for the edit. + if ( + child_right.bytes > edit.start.bytes || + (child_right.bytes == edit.start.bytes && is_pure_insertion) + ) { + edit.new_end = edit.start; + } + + // Children that occur before the edit are not reshaped by the edit. + else { + child_edit.old_end = child_edit.start; + child_edit.new_end = child_edit.start; + } + + // Queue processing of this child's subtree. + array_push(&stack, ((EditEntry) { + .tree = child, + .edit = child_edit, + })); + } + } + + array_delete(&stack); + return self; +} + +Subtree ts_subtree_last_external_token(Subtree tree) { + if (!ts_subtree_has_external_tokens(tree)) return NULL_SUBTREE; + while (tree.ptr->child_count > 0) { + for (uint32_t i = tree.ptr->child_count - 1; i + 1 > 0; i--) { + Subtree child = ts_subtree_children(tree)[i]; + if (ts_subtree_has_external_tokens(child)) { + tree = child; + break; + } + } + } + return tree; +} + +static size_t ts_subtree__write_char_to_string(char *str, size_t n, int32_t chr) { + if (chr == -1) + return snprintf(str, n, "INVALID"); + else if (chr == '\0') + return snprintf(str, n, "'\\0'"); + else if (chr == '\n') + return snprintf(str, n, "'\\n'"); + else if (chr == '\t') + return snprintf(str, n, "'\\t'"); + else if (chr == '\r') + return snprintf(str, n, "'\\r'"); + else if (0 < chr && chr < 128 && isprint(chr)) + return snprintf(str, n, "'%c'", chr); + else + return snprintf(str, n, "%d", chr); +} + +static const char *const ROOT_FIELD = "__ROOT__"; + +static size_t ts_subtree__write_to_string( + Subtree self, char *string, size_t limit, + const TSLanguage *language, bool include_all, + TSSymbol alias_symbol, bool alias_is_named, const char *field_name +) { + if (!self.ptr) return snprintf(string, limit, "(NULL)"); + + char *cursor = string; + char **writer = (limit > 1) ? &cursor : &string; + bool is_root = field_name == ROOT_FIELD; + bool is_visible = + include_all || + ts_subtree_missing(self) || + ( + alias_symbol + ? alias_is_named + : ts_subtree_visible(self) && ts_subtree_named(self) + ); + + if (is_visible) { + if (!is_root) { + cursor += snprintf(*writer, limit, " "); + if (field_name) { + cursor += snprintf(*writer, limit, "%s: ", field_name); + } + } + + if (ts_subtree_is_error(self) && ts_subtree_child_count(self) == 0 && self.ptr->size.bytes > 0) { + cursor += snprintf(*writer, limit, "(UNEXPECTED "); + cursor += ts_subtree__write_char_to_string(*writer, limit, self.ptr->lookahead_char); + } else { + TSSymbol symbol = alias_symbol ? alias_symbol : ts_subtree_symbol(self); + const char *symbol_name = ts_language_symbol_name(language, symbol); + if (ts_subtree_missing(self)) { + cursor += snprintf(*writer, limit, "(MISSING "); + if (alias_is_named || ts_subtree_named(self)) { + cursor += snprintf(*writer, limit, "%s", symbol_name); + } else { + cursor += snprintf(*writer, limit, "\"%s\"", symbol_name); + } + } else { + cursor += snprintf(*writer, limit, "(%s", symbol_name); + } + } + } else if (is_root) { + TSSymbol symbol = alias_symbol ? alias_symbol : ts_subtree_symbol(self); + const char *symbol_name = ts_language_symbol_name(language, symbol); + if (ts_subtree_child_count(self) > 0) { + cursor += snprintf(*writer, limit, "(%s", symbol_name); + } else if (ts_subtree_named(self)) { + cursor += snprintf(*writer, limit, "(%s)", symbol_name); + } else { + cursor += snprintf(*writer, limit, "(\"%s\")", symbol_name); + } + } + + if (ts_subtree_child_count(self)) { + const TSSymbol *alias_sequence = ts_language_alias_sequence(language, self.ptr->production_id); + const TSFieldMapEntry *field_map, *field_map_end; + ts_language_field_map( + language, + self.ptr->production_id, + &field_map, + &field_map_end + ); + + uint32_t structural_child_index = 0; + for (uint32_t i = 0; i < self.ptr->child_count; i++) { + Subtree child = ts_subtree_children(self)[i]; + if (ts_subtree_extra(child)) { + cursor += ts_subtree__write_to_string( + child, *writer, limit, + language, include_all, + 0, false, NULL + ); + } else { + TSSymbol subtree_alias_symbol = alias_sequence + ? alias_sequence[structural_child_index] + : 0; + bool subtree_alias_is_named = subtree_alias_symbol + ? ts_language_symbol_metadata(language, subtree_alias_symbol).named + : false; + + const char *child_field_name = is_visible ? NULL : field_name; + for (const TSFieldMapEntry *map = field_map; map < field_map_end; map++) { + if (!map->inherited && map->child_index == structural_child_index) { + child_field_name = language->field_names[map->field_id]; + break; + } + } + + cursor += ts_subtree__write_to_string( + child, *writer, limit, + language, include_all, + subtree_alias_symbol, subtree_alias_is_named, child_field_name + ); + structural_child_index++; + } + } + } + + if (is_visible) cursor += snprintf(*writer, limit, ")"); + + return cursor - string; +} + +char *ts_subtree_string( + Subtree self, + TSSymbol alias_symbol, + bool alias_is_named, + const TSLanguage *language, + bool include_all +) { + char scratch_string[1]; + size_t size = ts_subtree__write_to_string( + self, scratch_string, 1, + language, include_all, + alias_symbol, alias_is_named, ROOT_FIELD + ) + 1; + char *result = ts_malloc(size * sizeof(char)); + ts_subtree__write_to_string( + self, result, size, + language, include_all, + alias_symbol, alias_is_named, ROOT_FIELD + ); + return result; +} + +void ts_subtree__print_dot_graph(const Subtree *self, uint32_t start_offset, + const TSLanguage *language, TSSymbol alias_symbol, + FILE *f) { + TSSymbol subtree_symbol = ts_subtree_symbol(*self); + TSSymbol symbol = alias_symbol ? alias_symbol : subtree_symbol; + uint32_t end_offset = start_offset + ts_subtree_total_bytes(*self); + fprintf(f, "tree_%p [label=\"", (void *)self); + ts_language_write_symbol_as_dot_string(language, f, symbol); + fprintf(f, "\""); + + if (ts_subtree_child_count(*self) == 0) fprintf(f, ", shape=plaintext"); + if (ts_subtree_extra(*self)) fprintf(f, ", fontcolor=gray"); + if (ts_subtree_has_changes(*self)) fprintf(f, ", color=green, penwidth=2"); + + fprintf(f, ", tooltip=\"" + "range: %u - %u\n" + "state: %d\n" + "error-cost: %u\n" + "has-changes: %u\n" + "depends-on-column: %u\n" + "descendant-count: %u\n" + "repeat-depth: %u\n" + "lookahead-bytes: %u", + start_offset, end_offset, + ts_subtree_parse_state(*self), + ts_subtree_error_cost(*self), + ts_subtree_has_changes(*self), + ts_subtree_depends_on_column(*self), + ts_subtree_visible_descendant_count(*self), + ts_subtree_repeat_depth(*self), + ts_subtree_lookahead_bytes(*self) + ); + + if (ts_subtree_is_error(*self) && ts_subtree_child_count(*self) == 0 && self->ptr->lookahead_char != 0) { + fprintf(f, "\ncharacter: '%c'", self->ptr->lookahead_char); + } + + fprintf(f, "\"]\n"); + + uint32_t child_start_offset = start_offset; + uint32_t child_info_offset = + language->max_alias_sequence_length * + ts_subtree_production_id(*self); + for (uint32_t i = 0, n = ts_subtree_child_count(*self); i < n; i++) { + const Subtree *child = &ts_subtree_children(*self)[i]; + TSSymbol subtree_alias_symbol = 0; + if (!ts_subtree_extra(*child) && child_info_offset) { + subtree_alias_symbol = language->alias_sequences[child_info_offset]; + child_info_offset++; + } + ts_subtree__print_dot_graph(child, child_start_offset, language, subtree_alias_symbol, f); + fprintf(f, "tree_%p -> tree_%p [tooltip=%u]\n", (void *)self, (void *)child, i); + child_start_offset += ts_subtree_total_bytes(*child); + } +} + +void ts_subtree_print_dot_graph(Subtree self, const TSLanguage *language, FILE *f) { + fprintf(f, "digraph tree {\n"); + fprintf(f, "edge [arrowhead=none]\n"); + ts_subtree__print_dot_graph(&self, 0, language, 0, f); + fprintf(f, "}\n"); +} + +const ExternalScannerState *ts_subtree_external_scanner_state(Subtree self) { + static const ExternalScannerState empty_state = {{.short_data = {0}}, .length = 0}; + if ( + self.ptr && + !self.data.is_inline && + self.ptr->has_external_tokens && + self.ptr->child_count == 0 + ) { + return &self.ptr->external_scanner_state; + } else { + return &empty_state; + } +} + +bool ts_subtree_external_scanner_state_eq(Subtree self, Subtree other) { + const ExternalScannerState *state_self = ts_subtree_external_scanner_state(self); + const ExternalScannerState *state_other = ts_subtree_external_scanner_state(other); + return ts_external_scanner_state_eq( + state_self, + ts_external_scanner_state_data(state_other), + state_other->length + ); +} diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/subtree.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/subtree.h new file mode 100644 index 00000000000..ffc5fb7a3bf --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/subtree.h @@ -0,0 +1,399 @@ +#ifndef TREE_SITTER_SUBTREE_H_ +#define TREE_SITTER_SUBTREE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "./length.h" +#include "./array.h" +#include "./error_costs.h" +#include "./host.h" +#include "tree_sitter/api.h" +#include "./parser.h" + +#define TS_TREE_STATE_NONE USHRT_MAX +#define NULL_SUBTREE ((Subtree) {.ptr = NULL}) + +// The serialized state of an external scanner. +// +// Every time an external token subtree is created after a call to an +// external scanner, the scanner's `serialize` function is called to +// retrieve a serialized copy of its state. The bytes are then copied +// onto the subtree itself so that the scanner's state can later be +// restored using its `deserialize` function. +// +// Small byte arrays are stored inline, and long ones are allocated +// separately on the heap. +typedef struct { + union { + char *long_data; + char short_data[24]; + }; + uint32_t length; +} ExternalScannerState; + +// A compact representation of a subtree. +// +// This representation is used for small leaf nodes that are not +// errors, and were not created by an external scanner. +// +// The idea behind the layout of this struct is that the `is_inline` +// bit will fall exactly into the same location as the least significant +// bit of the pointer in `Subtree` or `MutableSubtree`, respectively. +// Because of alignment, for any valid pointer this will be 0, giving +// us the opportunity to make use of this bit to signify whether to use +// the pointer or the inline struct. +typedef struct SubtreeInlineData SubtreeInlineData; + +#define SUBTREE_BITS \ + bool visible : 1; \ + bool named : 1; \ + bool extra : 1; \ + bool has_changes : 1; \ + bool is_missing : 1; \ + bool is_keyword : 1; + +#define SUBTREE_SIZE \ + uint8_t padding_columns; \ + uint8_t padding_rows : 4; \ + uint8_t lookahead_bytes : 4; \ + uint8_t padding_bytes; \ + uint8_t size_bytes; + +#if TS_BIG_ENDIAN +#if TS_PTR_SIZE == 32 + +struct SubtreeInlineData { + uint16_t parse_state; + uint8_t symbol; + SUBTREE_BITS + bool unused : 1; + bool is_inline : 1; + SUBTREE_SIZE +}; + +#else + +struct SubtreeInlineData { + SUBTREE_SIZE + uint16_t parse_state; + uint8_t symbol; + SUBTREE_BITS + bool unused : 1; + bool is_inline : 1; +}; + +#endif +#else + +struct SubtreeInlineData { + bool is_inline : 1; + SUBTREE_BITS + uint8_t symbol; + uint16_t parse_state; + SUBTREE_SIZE +}; + +#endif + +#undef SUBTREE_BITS +#undef SUBTREE_SIZE + +// A heap-allocated representation of a subtree. +// +// This representation is used for parent nodes, external tokens, +// errors, and other leaf nodes whose data is too large to fit into +// the inline representation. +typedef struct { + volatile uint32_t ref_count; + Length padding; + Length size; + uint32_t lookahead_bytes; + uint32_t error_cost; + uint32_t child_count; + TSSymbol symbol; + TSStateId parse_state; + + bool visible : 1; + bool named : 1; + bool extra : 1; + bool fragile_left : 1; + bool fragile_right : 1; + bool has_changes : 1; + bool has_external_tokens : 1; + bool has_external_scanner_state_change : 1; + bool depends_on_column: 1; + bool is_missing : 1; + bool is_keyword : 1; + + union { + // Non-terminal subtrees (`child_count > 0`) + struct { + uint32_t visible_child_count; + uint32_t named_child_count; + uint32_t visible_descendant_count; + int32_t dynamic_precedence; + uint16_t repeat_depth; + uint16_t production_id; + struct { + TSSymbol symbol; + TSStateId parse_state; + } first_leaf; + }; + + // External terminal subtrees (`child_count == 0 && has_external_tokens`) + ExternalScannerState external_scanner_state; + + // Error terminal subtrees (`child_count == 0 && symbol == ts_builtin_sym_error`) + int32_t lookahead_char; + }; +} SubtreeHeapData; + +// The fundamental building block of a syntax tree. +typedef union { + SubtreeInlineData data; + const SubtreeHeapData *ptr; +} Subtree; + +// Like Subtree, but mutable. +typedef union { + SubtreeInlineData data; + SubtreeHeapData *ptr; +} MutableSubtree; + +typedef Array(Subtree) SubtreeArray; +typedef Array(MutableSubtree) MutableSubtreeArray; + +typedef struct { + MutableSubtreeArray free_trees; + MutableSubtreeArray tree_stack; +} SubtreePool; + +void ts_external_scanner_state_init(ExternalScannerState *self, const char *data, unsigned length); +const char *ts_external_scanner_state_data(const ExternalScannerState *self); +bool ts_external_scanner_state_eq(const ExternalScannerState *self, const char *buffer, unsigned length); +void ts_external_scanner_state_delete(ExternalScannerState *self); + +void ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest); +void ts_subtree_array_clear(SubtreePool *pool, SubtreeArray *self); +void ts_subtree_array_delete(SubtreePool *pool, SubtreeArray *self); +void ts_subtree_array_remove_trailing_extras(SubtreeArray *self, SubtreeArray *destination); +void ts_subtree_array_reverse(SubtreeArray *self); + +SubtreePool ts_subtree_pool_new(uint32_t capacity); +void ts_subtree_pool_delete(SubtreePool *self); + +Subtree ts_subtree_new_leaf( + SubtreePool *pool, TSSymbol symbol, Length padding, Length size, + uint32_t lookahead_bytes, TSStateId parse_state, + bool has_external_tokens, bool depends_on_column, + bool is_keyword, const TSLanguage *language +); +Subtree ts_subtree_new_error( + SubtreePool *pool, int32_t lookahead_char, Length padding, Length size, + uint32_t bytes_scanned, TSStateId parse_state, const TSLanguage *language +); +MutableSubtree ts_subtree_new_node( + TSSymbol symbol, + SubtreeArray *chiildren, + unsigned production_id, + const TSLanguage *language +); +Subtree ts_subtree_new_error_node( + SubtreeArray *children, + bool extra, + const TSLanguage * language +); +Subtree ts_subtree_new_missing_leaf( + SubtreePool *pool, + TSSymbol symbol, + Length padding, + uint32_t lookahead_bytes, + const TSLanguage *language +); +MutableSubtree ts_subtree_make_mut(SubtreePool *pool, Subtree self); +void ts_subtree_retain(Subtree self); +void ts_subtree_release(SubtreePool *pool, Subtree self); +int ts_subtree_compare(Subtree left, Subtree right, SubtreePool *pool); +void ts_subtree_set_symbol(MutableSubtree *self, TSSymbol symbol, const TSLanguage *language); +void ts_subtree_compress(MutableSubtree self, unsigned count, const TSLanguage *language, MutableSubtreeArray *stack); +void ts_subtree_summarize_children(MutableSubtree self, const TSLanguage *language); +Subtree ts_subtree_edit(Subtree self, const TSInputEdit *edit, SubtreePool *pool); +char *ts_subtree_string(Subtree self, TSSymbol alias_symbol, bool alias_is_named, const TSLanguage *language, bool include_all); +void ts_subtree_print_dot_graph(Subtree self, const TSLanguage *language, FILE *f); +Subtree ts_subtree_last_external_token(Subtree tree); +const ExternalScannerState *ts_subtree_external_scanner_state(Subtree self); +bool ts_subtree_external_scanner_state_eq(Subtree self, Subtree other); + +#define SUBTREE_GET(self, name) ((self).data.is_inline ? (self).data.name : (self).ptr->name) + +static inline TSSymbol ts_subtree_symbol(Subtree self) { return SUBTREE_GET(self, symbol); } +static inline bool ts_subtree_visible(Subtree self) { return SUBTREE_GET(self, visible); } +static inline bool ts_subtree_named(Subtree self) { return SUBTREE_GET(self, named); } +static inline bool ts_subtree_extra(Subtree self) { return SUBTREE_GET(self, extra); } +static inline bool ts_subtree_has_changes(Subtree self) { return SUBTREE_GET(self, has_changes); } +static inline bool ts_subtree_missing(Subtree self) { return SUBTREE_GET(self, is_missing); } +static inline bool ts_subtree_is_keyword(Subtree self) { return SUBTREE_GET(self, is_keyword); } +static inline TSStateId ts_subtree_parse_state(Subtree self) { return SUBTREE_GET(self, parse_state); } +static inline uint32_t ts_subtree_lookahead_bytes(Subtree self) { return SUBTREE_GET(self, lookahead_bytes); } + +#undef SUBTREE_GET + +// Get the size needed to store a heap-allocated subtree with the given +// number of children. +static inline size_t ts_subtree_alloc_size(uint32_t child_count) { + return child_count * sizeof(Subtree) + sizeof(SubtreeHeapData); +} + +// Get a subtree's children, which are allocated immediately before the +// tree's own heap data. +#define ts_subtree_children(self) \ + ((self).data.is_inline ? NULL : (Subtree *)((self).ptr) - (self).ptr->child_count) + +static inline void ts_subtree_set_extra(MutableSubtree *self, bool is_extra) { + if (self->data.is_inline) { + self->data.extra = is_extra; + } else { + self->ptr->extra = is_extra; + } +} + +static inline TSSymbol ts_subtree_leaf_symbol(Subtree self) { + if (self.data.is_inline) return self.data.symbol; + if (self.ptr->child_count == 0) return self.ptr->symbol; + return self.ptr->first_leaf.symbol; +} + +static inline TSStateId ts_subtree_leaf_parse_state(Subtree self) { + if (self.data.is_inline) return self.data.parse_state; + if (self.ptr->child_count == 0) return self.ptr->parse_state; + return self.ptr->first_leaf.parse_state; +} + +static inline Length ts_subtree_padding(Subtree self) { + if (self.data.is_inline) { + Length result = {self.data.padding_bytes, {self.data.padding_rows, self.data.padding_columns}}; + return result; + } else { + return self.ptr->padding; + } +} + +static inline Length ts_subtree_size(Subtree self) { + if (self.data.is_inline) { + Length result = {self.data.size_bytes, {0, self.data.size_bytes}}; + return result; + } else { + return self.ptr->size; + } +} + +static inline Length ts_subtree_total_size(Subtree self) { + return length_add(ts_subtree_padding(self), ts_subtree_size(self)); +} + +static inline uint32_t ts_subtree_total_bytes(Subtree self) { + return ts_subtree_total_size(self).bytes; +} + +static inline uint32_t ts_subtree_child_count(Subtree self) { + return self.data.is_inline ? 0 : self.ptr->child_count; +} + +static inline uint32_t ts_subtree_repeat_depth(Subtree self) { + return self.data.is_inline ? 0 : self.ptr->repeat_depth; +} + +static inline uint32_t ts_subtree_is_repetition(Subtree self) { + return self.data.is_inline + ? 0 + : !self.ptr->named && !self.ptr->visible && self.ptr->child_count != 0; +} + +static inline uint32_t ts_subtree_visible_descendant_count(Subtree self) { + return (self.data.is_inline || self.ptr->child_count == 0) + ? 0 + : self.ptr->visible_descendant_count; +} + +static inline uint32_t ts_subtree_visible_child_count(Subtree self) { + if (ts_subtree_child_count(self) > 0) { + return self.ptr->visible_child_count; + } else { + return 0; + } +} + +static inline uint32_t ts_subtree_error_cost(Subtree self) { + if (ts_subtree_missing(self)) { + return ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY; + } else { + return self.data.is_inline ? 0 : self.ptr->error_cost; + } +} + +static inline int32_t ts_subtree_dynamic_precedence(Subtree self) { + return (self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->dynamic_precedence; +} + +static inline uint16_t ts_subtree_production_id(Subtree self) { + if (ts_subtree_child_count(self) > 0) { + return self.ptr->production_id; + } else { + return 0; + } +} + +static inline bool ts_subtree_fragile_left(Subtree self) { + return self.data.is_inline ? false : self.ptr->fragile_left; +} + +static inline bool ts_subtree_fragile_right(Subtree self) { + return self.data.is_inline ? false : self.ptr->fragile_right; +} + +static inline bool ts_subtree_has_external_tokens(Subtree self) { + return self.data.is_inline ? false : self.ptr->has_external_tokens; +} + +static inline bool ts_subtree_has_external_scanner_state_change(Subtree self) { + return self.data.is_inline ? false : self.ptr->has_external_scanner_state_change; +} + +static inline bool ts_subtree_depends_on_column(Subtree self) { + return self.data.is_inline ? false : self.ptr->depends_on_column; +} + +static inline bool ts_subtree_is_fragile(Subtree self) { + return self.data.is_inline ? false : (self.ptr->fragile_left || self.ptr->fragile_right); +} + +static inline bool ts_subtree_is_error(Subtree self) { + return ts_subtree_symbol(self) == ts_builtin_sym_error; +} + +static inline bool ts_subtree_is_eof(Subtree self) { + return ts_subtree_symbol(self) == ts_builtin_sym_end; +} + +static inline Subtree ts_subtree_from_mut(MutableSubtree self) { + Subtree result; + result.data = self.data; + return result; +} + +static inline MutableSubtree ts_subtree_to_mut_unsafe(Subtree self) { + MutableSubtree result; + result.data = self.data; + return result; +} + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_SUBTREE_H_ diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/tree.c b/Externals/crystaledit/editlib/tree-sitter/lib/src/tree.c new file mode 100644 index 00000000000..bb451180006 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/tree.c @@ -0,0 +1,170 @@ +#include "tree_sitter/api.h" +#include "./array.h" +#include "./get_changed_ranges.h" +#include "./length.h" +#include "./subtree.h" +#include "./tree_cursor.h" +#include "./tree.h" + +TSTree *ts_tree_new( + Subtree root, const TSLanguage *language, + const TSRange *included_ranges, unsigned included_range_count +) { + TSTree *result = ts_malloc(sizeof(TSTree)); + result->root = root; + result->language = ts_language_copy(language); + result->included_ranges = ts_calloc(included_range_count, sizeof(TSRange)); + memcpy(result->included_ranges, included_ranges, included_range_count * sizeof(TSRange)); + result->included_range_count = included_range_count; + return result; +} + +TSTree *ts_tree_copy(const TSTree *self) { + ts_subtree_retain(self->root); + return ts_tree_new(self->root, self->language, self->included_ranges, self->included_range_count); +} + +void ts_tree_delete(TSTree *self) { + if (!self) return; + + SubtreePool pool = ts_subtree_pool_new(0); + ts_subtree_release(&pool, self->root); + ts_subtree_pool_delete(&pool); + ts_language_delete(self->language); + ts_free(self->included_ranges); + ts_free(self); +} + +TSNode ts_tree_root_node(const TSTree *self) { + return ts_node_new(self, &self->root, ts_subtree_padding(self->root), 0); +} + +TSNode ts_tree_root_node_with_offset( + const TSTree *self, + uint32_t offset_bytes, + TSPoint offset_extent +) { + Length offset = {offset_bytes, offset_extent}; + return ts_node_new(self, &self->root, length_add(offset, ts_subtree_padding(self->root)), 0); +} + +const TSLanguage *ts_tree_language(const TSTree *self) { + return self->language; +} + +void ts_tree_edit(TSTree *self, const TSInputEdit *edit) { + for (unsigned i = 0; i < self->included_range_count; i++) { + TSRange *range = &self->included_ranges[i]; + if (range->end_byte >= edit->old_end_byte) { + if (range->end_byte != UINT32_MAX) { + range->end_byte = edit->new_end_byte + (range->end_byte - edit->old_end_byte); + range->end_point = point_add( + edit->new_end_point, + point_sub(range->end_point, edit->old_end_point) + ); + if (range->end_byte < edit->new_end_byte) { + range->end_byte = UINT32_MAX; + range->end_point = POINT_MAX; + } + } + } else if (range->end_byte > edit->start_byte) { + range->end_byte = edit->start_byte; + range->end_point = edit->start_point; + } + if (range->start_byte >= edit->old_end_byte) { + range->start_byte = edit->new_end_byte + (range->start_byte - edit->old_end_byte); + range->start_point = point_add( + edit->new_end_point, + point_sub(range->start_point, edit->old_end_point) + ); + if (range->start_byte < edit->new_end_byte) { + range->start_byte = UINT32_MAX; + range->start_point = POINT_MAX; + } + } else if (range->start_byte > edit->start_byte) { + range->start_byte = edit->start_byte; + range->start_point = edit->start_point; + } + } + + SubtreePool pool = ts_subtree_pool_new(0); + self->root = ts_subtree_edit(self->root, edit, &pool); + ts_subtree_pool_delete(&pool); +} + +TSRange *ts_tree_included_ranges(const TSTree *self, uint32_t *length) { + *length = self->included_range_count; + TSRange *ranges = ts_calloc(self->included_range_count, sizeof(TSRange)); + memcpy(ranges, self->included_ranges, self->included_range_count * sizeof(TSRange)); + return ranges; +} + +TSRange *ts_tree_get_changed_ranges(const TSTree *old_tree, const TSTree *new_tree, uint32_t *length) { + TreeCursor cursor1 = {NULL, array_new(), 0}; + TreeCursor cursor2 = {NULL, array_new(), 0}; + ts_tree_cursor_init(&cursor1, ts_tree_root_node(old_tree)); + ts_tree_cursor_init(&cursor2, ts_tree_root_node(new_tree)); + + TSRangeArray included_range_differences = array_new(); + ts_range_array_get_changed_ranges( + old_tree->included_ranges, old_tree->included_range_count, + new_tree->included_ranges, new_tree->included_range_count, + &included_range_differences + ); + + TSRange *result; + *length = ts_subtree_get_changed_ranges( + &old_tree->root, &new_tree->root, &cursor1, &cursor2, + old_tree->language, &included_range_differences, &result + ); + + array_delete(&included_range_differences); + array_delete(&cursor1.stack); + array_delete(&cursor2.stack); + return result; +} + +#ifdef _WIN32 + +#include +#include + +int _ts_dup(HANDLE handle) { + HANDLE dup_handle; + if (!DuplicateHandle( + GetCurrentProcess(), handle, + GetCurrentProcess(), &dup_handle, + 0, FALSE, DUPLICATE_SAME_ACCESS + )) return -1; + + return _open_osfhandle((intptr_t)dup_handle, 0); +} + +void ts_tree_print_dot_graph(const TSTree *self, int fd) { + FILE *file = _fdopen(_ts_dup((HANDLE)_get_osfhandle(fd)), "a"); + ts_subtree_print_dot_graph(self->root, self->language, file); + fclose(file); +} + +#elif !defined(__wasi__) // WASI doesn't support dup + +#include + +int _ts_dup(int file_descriptor) { + return dup(file_descriptor); +} + +void ts_tree_print_dot_graph(const TSTree *self, int file_descriptor) { + FILE *file = fdopen(_ts_dup(file_descriptor), "a"); + ts_subtree_print_dot_graph(self->root, self->language, file); + fclose(file); +} + +#else + +void ts_tree_print_dot_graph(const TSTree *self, int file_descriptor) { + (void)self; + (void)file_descriptor; +} + +#endif diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/tree.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/tree.h new file mode 100644 index 00000000000..9328f55a5bf --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/tree.h @@ -0,0 +1,31 @@ +#ifndef TREE_SITTER_TREE_H_ +#define TREE_SITTER_TREE_H_ + +#include "./subtree.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + const Subtree *child; + const Subtree *parent; + Length position; + TSSymbol alias_symbol; +} ParentCacheEntry; + +struct TSTree { + Subtree root; + const TSLanguage *language; + TSRange *included_ranges; + unsigned included_range_count; +}; + +TSTree *ts_tree_new(Subtree root, const TSLanguage *language, const TSRange *included_ranges, unsigned included_range_count); +TSNode ts_node_new(const TSTree *tree, const Subtree *subtree, Length position, TSSymbol alias); + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_TREE_H_ diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/tree_cursor.c b/Externals/crystaledit/editlib/tree-sitter/lib/src/tree_cursor.c new file mode 100644 index 00000000000..70ef5e39aac --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/tree_cursor.c @@ -0,0 +1,716 @@ +#include "tree_sitter/api.h" +#include "./tree_cursor.h" +#include "./language.h" +#include "./tree.h" + +typedef struct { + Subtree parent; + const TSTree *tree; + Length position; + uint32_t child_index; + uint32_t structural_child_index; + uint32_t descendant_index; + const TSSymbol *alias_sequence; +} CursorChildIterator; + +// CursorChildIterator + +static inline bool ts_tree_cursor_is_entry_visible(const TreeCursor *self, uint32_t index) { + TreeCursorEntry *entry = array_get(&self->stack, index); + if (index == 0 || ts_subtree_visible(*entry->subtree)) { + return true; + } else if (!ts_subtree_extra(*entry->subtree)) { + TreeCursorEntry *parent_entry = array_get(&self->stack, index - 1); + return ts_language_alias_at( + self->tree->language, + parent_entry->subtree->ptr->production_id, + entry->structural_child_index + ); + } else { + return false; + } +} + +static inline CursorChildIterator ts_tree_cursor_iterate_children(const TreeCursor *self) { + TreeCursorEntry *last_entry = array_back(&self->stack); + if (ts_subtree_child_count(*last_entry->subtree) == 0) { + return (CursorChildIterator) {NULL_SUBTREE, self->tree, length_zero(), 0, 0, 0, NULL}; + } + const TSSymbol *alias_sequence = ts_language_alias_sequence( + self->tree->language, + last_entry->subtree->ptr->production_id + ); + + uint32_t descendant_index = last_entry->descendant_index; + if (ts_tree_cursor_is_entry_visible(self, self->stack.size - 1)) { + descendant_index += 1; + } + + return (CursorChildIterator) { + .tree = self->tree, + .parent = *last_entry->subtree, + .position = last_entry->position, + .child_index = 0, + .structural_child_index = 0, + .descendant_index = descendant_index, + .alias_sequence = alias_sequence, + }; +} + +static inline bool ts_tree_cursor_child_iterator_next( + CursorChildIterator *self, + TreeCursorEntry *result, + bool *visible +) { + if (!self->parent.ptr || self->child_index == self->parent.ptr->child_count) return false; + const Subtree *child = &ts_subtree_children(self->parent)[self->child_index]; + *result = (TreeCursorEntry) { + .subtree = child, + .position = self->position, + .child_index = self->child_index, + .structural_child_index = self->structural_child_index, + .descendant_index = self->descendant_index, + }; + *visible = ts_subtree_visible(*child); + bool extra = ts_subtree_extra(*child); + if (!extra) { + if (self->alias_sequence) { + *visible |= self->alias_sequence[self->structural_child_index]; + } + self->structural_child_index++; + } + + self->descendant_index += ts_subtree_visible_descendant_count(*child); + if (*visible) { + self->descendant_index += 1; + } + + self->position = length_add(self->position, ts_subtree_size(*child)); + self->child_index++; + + if (self->child_index < self->parent.ptr->child_count) { + Subtree next_child = ts_subtree_children(self->parent)[self->child_index]; + self->position = length_add(self->position, ts_subtree_padding(next_child)); + } + + return true; +} + +// Return a position that, when `b` is added to it, yields `a`. This +// can only be computed if `b` has zero rows. Otherwise, this function +// returns `LENGTH_UNDEFINED`, and the caller needs to recompute +// the position some other way. +static inline Length length_backtrack(Length a, Length b) { + if (length_is_undefined(a) || b.extent.row != 0) { + return LENGTH_UNDEFINED; + } + + Length result; + result.bytes = a.bytes - b.bytes; + result.extent.row = a.extent.row; + result.extent.column = a.extent.column - b.extent.column; + return result; +} + +static inline bool ts_tree_cursor_child_iterator_previous( + CursorChildIterator *self, + TreeCursorEntry *result, + bool *visible +) { + // this is mostly a reverse `ts_tree_cursor_child_iterator_next` taking into + // account unsigned underflow + if (!self->parent.ptr || (int8_t)self->child_index == -1) return false; + const Subtree *child = &ts_subtree_children(self->parent)[self->child_index]; + *result = (TreeCursorEntry) { + .subtree = child, + .position = self->position, + .child_index = self->child_index, + .structural_child_index = self->structural_child_index, + }; + *visible = ts_subtree_visible(*child); + bool extra = ts_subtree_extra(*child); + + self->position = length_backtrack(self->position, ts_subtree_padding(*child)); + self->child_index--; + + if (!extra && self->alias_sequence) { + *visible |= self->alias_sequence[self->structural_child_index]; + if (self->structural_child_index > 0) { + self->structural_child_index--; + } + } + + // unsigned can underflow so compare it to child_count + if (self->child_index < self->parent.ptr->child_count) { + Subtree previous_child = ts_subtree_children(self->parent)[self->child_index]; + Length size = ts_subtree_size(previous_child); + self->position = length_backtrack(self->position, size); + } + + return true; +} + +// TSTreeCursor - lifecycle + +TSTreeCursor ts_tree_cursor_new(TSNode node) { + TSTreeCursor self = {NULL, NULL, {0, 0, 0}}; + ts_tree_cursor_init((TreeCursor *)&self, node); + return self; +} + +void ts_tree_cursor_reset(TSTreeCursor *_self, TSNode node) { + ts_tree_cursor_init((TreeCursor *)_self, node); +} + +void ts_tree_cursor_init(TreeCursor *self, TSNode node) { + self->tree = node.tree; + self->root_alias_symbol = node.context[3]; + array_clear(&self->stack); + array_push(&self->stack, ((TreeCursorEntry) { + .subtree = (const Subtree *)node.id, + .position = { + ts_node_start_byte(node), + ts_node_start_point(node) + }, + .child_index = 0, + .structural_child_index = 0, + .descendant_index = 0, + })); +} + +void ts_tree_cursor_delete(TSTreeCursor *_self) { + TreeCursor *self = (TreeCursor *)_self; + array_delete(&self->stack); +} + +// TSTreeCursor - walking the tree + +TreeCursorStep ts_tree_cursor_goto_first_child_internal(TSTreeCursor *_self) { + TreeCursor *self = (TreeCursor *)_self; + bool visible; + TreeCursorEntry entry; + CursorChildIterator iterator = ts_tree_cursor_iterate_children(self); + while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) { + if (visible) { + array_push(&self->stack, entry); + return TreeCursorStepVisible; + } + if (ts_subtree_visible_child_count(*entry.subtree) > 0) { + array_push(&self->stack, entry); + return TreeCursorStepHidden; + } + } + return TreeCursorStepNone; +} + +bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) { + for (;;) { + switch (ts_tree_cursor_goto_first_child_internal(self)) { + case TreeCursorStepHidden: + continue; + case TreeCursorStepVisible: + return true; + default: + return false; + } + } +} + +TreeCursorStep ts_tree_cursor_goto_last_child_internal(TSTreeCursor *_self) { + TreeCursor *self = (TreeCursor *)_self; + bool visible; + TreeCursorEntry entry; + CursorChildIterator iterator = ts_tree_cursor_iterate_children(self); + if (!iterator.parent.ptr || iterator.parent.ptr->child_count == 0) return TreeCursorStepNone; + + TreeCursorEntry last_entry = {0}; + TreeCursorStep last_step = TreeCursorStepNone; + while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) { + if (visible) { + last_entry = entry; + last_step = TreeCursorStepVisible; + } + else if (ts_subtree_visible_child_count(*entry.subtree) > 0) { + last_entry = entry; + last_step = TreeCursorStepHidden; + } + } + if (last_entry.subtree) { + array_push(&self->stack, last_entry); + return last_step; + } + + return TreeCursorStepNone; +} + +bool ts_tree_cursor_goto_last_child(TSTreeCursor *self) { + for (;;) { + switch (ts_tree_cursor_goto_last_child_internal(self)) { + case TreeCursorStepHidden: + continue; + case TreeCursorStepVisible: + return true; + default: + return false; + } + } +} + +static inline int64_t ts_tree_cursor_goto_first_child_for_byte_and_point( + TSTreeCursor *_self, + uint32_t goal_byte, + TSPoint goal_point +) { + TreeCursor *self = (TreeCursor *)_self; + uint32_t initial_size = self->stack.size; + uint32_t visible_child_index = 0; + + bool did_descend; + do { + did_descend = false; + + bool visible; + TreeCursorEntry entry; + CursorChildIterator iterator = ts_tree_cursor_iterate_children(self); + while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) { + Length entry_end = length_add(entry.position, ts_subtree_size(*entry.subtree)); + bool at_goal = entry_end.bytes > goal_byte && point_gt(entry_end.extent, goal_point); + uint32_t visible_child_count = ts_subtree_visible_child_count(*entry.subtree); + if (at_goal) { + if (visible) { + array_push(&self->stack, entry); + return visible_child_index; + } + if (visible_child_count > 0) { + array_push(&self->stack, entry); + did_descend = true; + break; + } + } else if (visible) { + visible_child_index++; + } else { + visible_child_index += visible_child_count; + } + } + } while (did_descend); + + self->stack.size = initial_size; + return -1; +} + +int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *self, uint32_t goal_byte) { + return ts_tree_cursor_goto_first_child_for_byte_and_point(self, goal_byte, POINT_ZERO); +} + +int64_t ts_tree_cursor_goto_first_child_for_point(TSTreeCursor *self, TSPoint goal_point) { + return ts_tree_cursor_goto_first_child_for_byte_and_point(self, 0, goal_point); +} + +TreeCursorStep ts_tree_cursor_goto_sibling_internal( + TSTreeCursor *_self, + bool (*advance)(CursorChildIterator *, TreeCursorEntry *, bool *) +) { + TreeCursor *self = (TreeCursor *)_self; + uint32_t initial_size = self->stack.size; + + while (self->stack.size > 1) { + TreeCursorEntry entry = array_pop(&self->stack); + CursorChildIterator iterator = ts_tree_cursor_iterate_children(self); + iterator.child_index = entry.child_index; + iterator.structural_child_index = entry.structural_child_index; + iterator.position = entry.position; + iterator.descendant_index = entry.descendant_index; + + bool visible = false; + advance(&iterator, &entry, &visible); + if (visible && self->stack.size + 1 < initial_size) break; + + while (advance(&iterator, &entry, &visible)) { + if (visible) { + array_push(&self->stack, entry); + return TreeCursorStepVisible; + } + + if (ts_subtree_visible_child_count(*entry.subtree)) { + array_push(&self->stack, entry); + return TreeCursorStepHidden; + } + } + } + + self->stack.size = initial_size; + return TreeCursorStepNone; +} + +TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(TSTreeCursor *_self) { + return ts_tree_cursor_goto_sibling_internal(_self, ts_tree_cursor_child_iterator_next); +} + +bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) { + switch (ts_tree_cursor_goto_next_sibling_internal(self)) { + case TreeCursorStepHidden: + ts_tree_cursor_goto_first_child(self); + return true; + case TreeCursorStepVisible: + return true; + default: + return false; + } +} + +TreeCursorStep ts_tree_cursor_goto_previous_sibling_internal(TSTreeCursor *_self) { + // since subtracting across row loses column information, we may have to + // restore it + TreeCursor *self = (TreeCursor *)_self; + + // for that, save current position before traversing + TreeCursorStep step = ts_tree_cursor_goto_sibling_internal( + _self, ts_tree_cursor_child_iterator_previous); + if (step == TreeCursorStepNone) + return step; + + // if length is already valid, there's no need to recompute it + if (!length_is_undefined(array_back(&self->stack)->position)) + return step; + + // restore position from the parent node + const TreeCursorEntry *parent = array_get(&self->stack, self->stack.size - 2); + Length position = parent->position; + uint32_t child_index = array_back(&self->stack)->child_index; + const Subtree *children = ts_subtree_children((*(parent->subtree))); + + if (child_index > 0) { + // skip first child padding since its position should match the position of the parent + position = length_add(position, ts_subtree_size(children[0])); + for (uint32_t i = 1; i < child_index; ++i) { + position = length_add(position, ts_subtree_total_size(children[i])); + } + position = length_add(position, ts_subtree_padding(children[child_index])); + } + + array_back(&self->stack)->position = position; + + return step; +} + +bool ts_tree_cursor_goto_previous_sibling(TSTreeCursor *self) { + switch (ts_tree_cursor_goto_previous_sibling_internal(self)) { + case TreeCursorStepHidden: + ts_tree_cursor_goto_last_child(self); + return true; + case TreeCursorStepVisible: + return true; + default: + return false; + } +} + +bool ts_tree_cursor_goto_parent(TSTreeCursor *_self) { + TreeCursor *self = (TreeCursor *)_self; + for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) { + if (ts_tree_cursor_is_entry_visible(self, i)) { + self->stack.size = i + 1; + return true; + } + } + return false; +} + +void ts_tree_cursor_goto_descendant( + TSTreeCursor *_self, + uint32_t goal_descendant_index +) { + TreeCursor *self = (TreeCursor *)_self; + + // Ascend to the lowest ancestor that contains the goal node. + for (;;) { + uint32_t i = self->stack.size - 1; + TreeCursorEntry *entry = array_get(&self->stack, i); + uint32_t next_descendant_index = + entry->descendant_index + + (ts_tree_cursor_is_entry_visible(self, i) ? 1 : 0) + + ts_subtree_visible_descendant_count(*entry->subtree); + if ( + (entry->descendant_index <= goal_descendant_index) && + (next_descendant_index > goal_descendant_index) + ) { + break; + } else if (self->stack.size <= 1) { + return; + } else { + self->stack.size--; + } + } + + // Descend to the goal node. + bool did_descend = true; + do { + did_descend = false; + bool visible; + TreeCursorEntry entry; + CursorChildIterator iterator = ts_tree_cursor_iterate_children(self); + if (iterator.descendant_index > goal_descendant_index) { + return; + } + + while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) { + if (iterator.descendant_index > goal_descendant_index) { + array_push(&self->stack, entry); + if (visible && entry.descendant_index == goal_descendant_index) { + return; + } else { + did_descend = true; + break; + } + } + } + } while (did_descend); +} + +uint32_t ts_tree_cursor_current_descendant_index(const TSTreeCursor *_self) { + const TreeCursor *self = (const TreeCursor *)_self; + TreeCursorEntry *last_entry = array_back(&self->stack); + return last_entry->descendant_index; +} + +TSNode ts_tree_cursor_current_node(const TSTreeCursor *_self) { + const TreeCursor *self = (const TreeCursor *)_self; + TreeCursorEntry *last_entry = array_back(&self->stack); + bool is_extra = ts_subtree_extra(*last_entry->subtree); + TSSymbol alias_symbol = is_extra ? 0 : self->root_alias_symbol; + if (self->stack.size > 1 && !is_extra) { + TreeCursorEntry *parent_entry = array_get(&self->stack, self->stack.size - 2); + alias_symbol = ts_language_alias_at( + self->tree->language, + parent_entry->subtree->ptr->production_id, + last_entry->structural_child_index + ); + } + return ts_node_new( + self->tree, + last_entry->subtree, + last_entry->position, + alias_symbol + ); +} + +// Private - Get various facts about the current node that are needed +// when executing tree queries. +void ts_tree_cursor_current_status( + const TSTreeCursor *_self, + TSFieldId *field_id, + bool *has_later_siblings, + bool *has_later_named_siblings, + bool *can_have_later_siblings_with_this_field, + TSSymbol *supertypes, + unsigned *supertype_count +) { + const TreeCursor *self = (const TreeCursor *)_self; + unsigned max_supertypes = *supertype_count; + *field_id = 0; + *supertype_count = 0; + *has_later_siblings = false; + *has_later_named_siblings = false; + *can_have_later_siblings_with_this_field = false; + + // Walk up the tree, visiting the current node and its invisible ancestors, + // because fields can refer to nodes through invisible *wrapper* nodes, + for (unsigned i = self->stack.size - 1; i > 0; i--) { + TreeCursorEntry *entry = array_get(&self->stack, i); + TreeCursorEntry *parent_entry = array_get(&self->stack, i - 1); + + const TSSymbol *alias_sequence = ts_language_alias_sequence( + self->tree->language, + parent_entry->subtree->ptr->production_id + ); + + #define subtree_symbol(subtree, structural_child_index) \ + (( \ + !ts_subtree_extra(subtree) && \ + alias_sequence && \ + alias_sequence[structural_child_index] \ + ) ? \ + alias_sequence[structural_child_index] : \ + ts_subtree_symbol(subtree)) + + // Stop walking up when a visible ancestor is found. + TSSymbol entry_symbol = subtree_symbol( + *entry->subtree, + entry->structural_child_index + ); + TSSymbolMetadata entry_metadata = ts_language_symbol_metadata( + self->tree->language, + entry_symbol + ); + if (i != self->stack.size - 1 && entry_metadata.visible) break; + + // Record any supertypes + if (entry_metadata.supertype && *supertype_count < max_supertypes) { + supertypes[*supertype_count] = entry_symbol; + (*supertype_count)++; + } + + // Determine if the current node has later siblings. + if (!*has_later_siblings) { + unsigned sibling_count = parent_entry->subtree->ptr->child_count; + unsigned structural_child_index = entry->structural_child_index; + if (!ts_subtree_extra(*entry->subtree)) structural_child_index++; + for (unsigned j = entry->child_index + 1; j < sibling_count; j++) { + Subtree sibling = ts_subtree_children(*parent_entry->subtree)[j]; + TSSymbolMetadata sibling_metadata = ts_language_symbol_metadata( + self->tree->language, + subtree_symbol(sibling, structural_child_index) + ); + if (sibling_metadata.visible) { + *has_later_siblings = true; + if (*has_later_named_siblings) break; + if (sibling_metadata.named) { + *has_later_named_siblings = true; + break; + } + } else if (ts_subtree_visible_child_count(sibling) > 0) { + *has_later_siblings = true; + if (*has_later_named_siblings) break; + if (sibling.ptr->named_child_count > 0) { + *has_later_named_siblings = true; + break; + } + } + if (!ts_subtree_extra(sibling)) structural_child_index++; + } + } + + #undef subtree_symbol + + if (!ts_subtree_extra(*entry->subtree)) { + const TSFieldMapEntry *field_map, *field_map_end; + ts_language_field_map( + self->tree->language, + parent_entry->subtree->ptr->production_id, + &field_map, &field_map_end + ); + + // Look for a field name associated with the current node. + if (!*field_id) { + for (const TSFieldMapEntry *map = field_map; map < field_map_end; map++) { + if (!map->inherited && map->child_index == entry->structural_child_index) { + *field_id = map->field_id; + break; + } + } + } + + // Determine if the current node can have later siblings with the same field name. + if (*field_id) { + for (const TSFieldMapEntry *map = field_map; map < field_map_end; map++) { + if ( + map->field_id == *field_id && + map->child_index > entry->structural_child_index + ) { + *can_have_later_siblings_with_this_field = true; + break; + } + } + } + } + } +} + +uint32_t ts_tree_cursor_current_depth(const TSTreeCursor *_self) { + const TreeCursor *self = (const TreeCursor *)_self; + uint32_t depth = 0; + for (unsigned i = 1; i < self->stack.size; i++) { + if (ts_tree_cursor_is_entry_visible(self, i)) { + depth++; + } + } + return depth; +} + +TSNode ts_tree_cursor_parent_node(const TSTreeCursor *_self) { + const TreeCursor *self = (const TreeCursor *)_self; + for (int i = (int)self->stack.size - 2; i >= 0; i--) { + TreeCursorEntry *entry = array_get(&self->stack, i); + bool is_visible = true; + TSSymbol alias_symbol = 0; + if (i > 0) { + TreeCursorEntry *parent_entry = array_get(&self->stack, i - 1); + alias_symbol = ts_language_alias_at( + self->tree->language, + parent_entry->subtree->ptr->production_id, + entry->structural_child_index + ); + is_visible = (alias_symbol != 0) || ts_subtree_visible(*entry->subtree); + } + if (is_visible) { + return ts_node_new( + self->tree, + entry->subtree, + entry->position, + alias_symbol + ); + } + } + return ts_node_new(NULL, NULL, length_zero(), 0); +} + +TSFieldId ts_tree_cursor_current_field_id(const TSTreeCursor *_self) { + const TreeCursor *self = (const TreeCursor *)_self; + + // Walk up the tree, visiting the current node and its invisible ancestors. + for (unsigned i = self->stack.size - 1; i > 0; i--) { + TreeCursorEntry *entry = array_get(&self->stack, i); + TreeCursorEntry *parent_entry = array_get(&self->stack, i - 1); + + // Stop walking up when another visible node is found. + if ( + i != self->stack.size - 1 && + ts_tree_cursor_is_entry_visible(self, i) + ) break; + + if (ts_subtree_extra(*entry->subtree)) break; + + const TSFieldMapEntry *field_map, *field_map_end; + ts_language_field_map( + self->tree->language, + parent_entry->subtree->ptr->production_id, + &field_map, &field_map_end + ); + for (const TSFieldMapEntry *map = field_map; map < field_map_end; map++) { + if (!map->inherited && map->child_index == entry->structural_child_index) { + return map->field_id; + } + } + } + return 0; +} + +const char *ts_tree_cursor_current_field_name(const TSTreeCursor *_self) { + TSFieldId id = ts_tree_cursor_current_field_id(_self); + if (id) { + const TreeCursor *self = (const TreeCursor *)_self; + return self->tree->language->field_names[id]; + } else { + return NULL; + } +} + +TSTreeCursor ts_tree_cursor_copy(const TSTreeCursor *_cursor) { + const TreeCursor *cursor = (const TreeCursor *)_cursor; + TSTreeCursor res = {NULL, NULL, {0, 0}}; + TreeCursor *copy = (TreeCursor *)&res; + copy->tree = cursor->tree; + copy->root_alias_symbol = cursor->root_alias_symbol; + array_init(©->stack); + array_push_all(©->stack, &cursor->stack); + return res; +} + +void ts_tree_cursor_reset_to(TSTreeCursor *_dst, const TSTreeCursor *_src) { + const TreeCursor *cursor = (const TreeCursor *)_src; + TreeCursor *copy = (TreeCursor *)_dst; + copy->tree = cursor->tree; + copy->root_alias_symbol = cursor->root_alias_symbol; + array_clear(©->stack); + array_push_all(©->stack, &cursor->stack); +} diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/tree_cursor.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/tree_cursor.h new file mode 100644 index 00000000000..7d4e7ef0a67 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/tree_cursor.h @@ -0,0 +1,48 @@ +#ifndef TREE_SITTER_TREE_CURSOR_H_ +#define TREE_SITTER_TREE_CURSOR_H_ + +#include "./subtree.h" + +typedef struct { + const Subtree *subtree; + Length position; + uint32_t child_index; + uint32_t structural_child_index; + uint32_t descendant_index; +} TreeCursorEntry; + +typedef struct { + const TSTree *tree; + Array(TreeCursorEntry) stack; + TSSymbol root_alias_symbol; +} TreeCursor; + +typedef enum { + TreeCursorStepNone, + TreeCursorStepHidden, + TreeCursorStepVisible, +} TreeCursorStep; + +void ts_tree_cursor_init(TreeCursor *self, TSNode node); +void ts_tree_cursor_current_status( + const TSTreeCursor *_self, + TSFieldId *field_id, + bool *has_later_siblings, + bool *has_later_named_siblings, + bool *can_have_later_siblings_with_this_field, + TSSymbol *supertypes, + unsigned *supertype_count +); + +TreeCursorStep ts_tree_cursor_goto_first_child_internal(TSTreeCursor *_self); +TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(TSTreeCursor *_self); + +static inline Subtree ts_tree_cursor_current_subtree(const TSTreeCursor *_self) { + const TreeCursor *self = (const TreeCursor *)_self; + TreeCursorEntry *last_entry = array_back(&self->stack); + return *last_entry->subtree; +} + +TSNode ts_tree_cursor_parent_node(const TSTreeCursor *_self); + +#endif // TREE_SITTER_TREE_CURSOR_H_ diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/ts_assert.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/ts_assert.h new file mode 100644 index 00000000000..4cb8f36a122 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/ts_assert.h @@ -0,0 +1,11 @@ +#ifndef TREE_SITTER_ASSERT_H_ +#define TREE_SITTER_ASSERT_H_ + +#ifdef NDEBUG +#define ts_assert(e) ((void)(e)) +#else +#include +#define ts_assert(e) assert(e) +#endif + +#endif // TREE_SITTER_ASSERT_H_ diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode.h new file mode 100644 index 00000000000..0fba3f29614 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode.h @@ -0,0 +1,75 @@ +#ifndef TREE_SITTER_UNICODE_H_ +#define TREE_SITTER_UNICODE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +#define U_EXPORT +#define U_EXPORT2 +#include "unicode/utf8.h" +#include "unicode/utf16.h" +#include "portable/endian.h" + +#define U16_NEXT_LE(s, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \ + (c)=le16toh((s)[(i)++]); \ + if(U16_IS_LEAD(c)) { \ + uint16_t __c2; \ + if((i)!=(length) && U16_IS_TRAIL(__c2=(s)[(i)])) { \ + ++(i); \ + (c)=U16_GET_SUPPLEMENTARY((c), __c2); \ + } \ + } \ +} UPRV_BLOCK_MACRO_END + +#define U16_NEXT_BE(s, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \ + (c)=be16toh((s)[(i)++]); \ + if(U16_IS_LEAD(c)) { \ + uint16_t __c2; \ + if((i)!=(length) && U16_IS_TRAIL(__c2=(s)[(i)])) { \ + ++(i); \ + (c)=U16_GET_SUPPLEMENTARY((c), __c2); \ + } \ + } \ +} UPRV_BLOCK_MACRO_END + +static const int32_t TS_DECODE_ERROR = U_SENTINEL; + +static inline uint32_t ts_decode_utf8( + const uint8_t *string, + uint32_t length, + int32_t *code_point +) { + uint32_t i = 0; + U8_NEXT(string, i, length, *code_point); + return i; +} + +static inline uint32_t ts_decode_utf16_le( + const uint8_t *string, + uint32_t length, + int32_t *code_point +) { + uint32_t i = 0; + U16_NEXT_LE(((uint16_t *)string), i, length, *code_point); + return i * 2; +} + +static inline uint32_t ts_decode_utf16_be( + const uint8_t *string, + uint32_t length, + int32_t *code_point +) { + uint32_t i = 0; + U16_NEXT_BE(((uint16_t *)string), i, length, *code_point); + return i * 2; +} + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_UNICODE_H_ diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/ICU_SHA b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/ICU_SHA new file mode 100644 index 00000000000..3622283ba35 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/ICU_SHA @@ -0,0 +1 @@ +552b01f61127d30d6589aa4bf99468224979b661 diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/LICENSE b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/LICENSE new file mode 100644 index 00000000000..2e01e368768 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/LICENSE @@ -0,0 +1,414 @@ +COPYRIGHT AND PERMISSION NOTICE (ICU 58 and later) + +Copyright © 1991-2019 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in https://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation +(the "Data Files") or Unicode software and any associated documentation +(the "Software") to deal in the Data Files or Software +without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of +the Data Files or Software, and to permit persons to whom the Data Files +or Software are furnished to do so, provided that either +(a) this copyright and permission notice appear with all copies +of the Data Files or Software, or +(b) this copyright and permission notice appear in associated +Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, +use or other dealings in these Data Files or Software without prior +written authorization of the copyright holder. + +--------------------- + +Third-Party Software Licenses + +This section contains third-party software notices and/or additional +terms for licensed third-party software components included within ICU +libraries. + +1. ICU License - ICU 1.8.1 to ICU 57.1 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1995-2016 International Business Machines Corporation and others +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, provided that the above +copyright notice(s) and this permission notice appear in all copies of +the Software and that both the above copyright notice(s) and this +permission notice appear in supporting documentation. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY +SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER +RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, use +or other dealings in this Software without prior written authorization +of the copyright holder. + +All trademarks and registered trademarks mentioned herein are the +property of their respective owners. + +2. Chinese/Japanese Word Break Dictionary Data (cjdict.txt) + + # The Google Chrome software developed by Google is licensed under + # the BSD license. Other software included in this distribution is + # provided under other licenses, as set forth below. + # + # The BSD License + # http://opensource.org/licenses/bsd-license.php + # Copyright (C) 2006-2008, Google Inc. + # + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are met: + # + # Redistributions of source code must retain the above copyright notice, + # this list of conditions and the following disclaimer. + # Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided with + # the distribution. + # Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + # BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + # + # + # The word list in cjdict.txt are generated by combining three word lists + # listed below with further processing for compound word breaking. The + # frequency is generated with an iterative training against Google web + # corpora. + # + # * Libtabe (Chinese) + # - https://sourceforge.net/project/?group_id=1519 + # - Its license terms and conditions are shown below. + # + # * IPADIC (Japanese) + # - http://chasen.aist-nara.ac.jp/chasen/distribution.html + # - Its license terms and conditions are shown below. + # + # ---------COPYING.libtabe ---- BEGIN-------------------- + # + # /* + # * Copyright (c) 1999 TaBE Project. + # * Copyright (c) 1999 Pai-Hsiang Hsiao. + # * All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the TaBE Project nor the names of its + # * contributors may be used to endorse or promote products derived + # * from this software without specific prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # /* + # * Copyright (c) 1999 Computer Systems and Communication Lab, + # * Institute of Information Science, Academia + # * Sinica. All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the Computer Systems and Communication Lab + # * nor the names of its contributors may be used to endorse or + # * promote products derived from this software without specific + # * prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # Copyright 1996 Chih-Hao Tsai @ Beckman Institute, + # University of Illinois + # c-tsai4@uiuc.edu http://casper.beckman.uiuc.edu/~c-tsai4 + # + # ---------------COPYING.libtabe-----END-------------------------------- + # + # + # ---------------COPYING.ipadic-----BEGIN------------------------------- + # + # Copyright 2000, 2001, 2002, 2003 Nara Institute of Science + # and Technology. All Rights Reserved. + # + # Use, reproduction, and distribution of this software is permitted. + # Any copy of this software, whether in its original form or modified, + # must include both the above copyright notice and the following + # paragraphs. + # + # Nara Institute of Science and Technology (NAIST), + # the copyright holders, disclaims all warranties with regard to this + # software, including all implied warranties of merchantability and + # fitness, in no event shall NAIST be liable for + # any special, indirect or consequential damages or any damages + # whatsoever resulting from loss of use, data or profits, whether in an + # action of contract, negligence or other tortuous action, arising out + # of or in connection with the use or performance of this software. + # + # A large portion of the dictionary entries + # originate from ICOT Free Software. The following conditions for ICOT + # Free Software applies to the current dictionary as well. + # + # Each User may also freely distribute the Program, whether in its + # original form or modified, to any third party or parties, PROVIDED + # that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear + # on, or be attached to, the Program, which is distributed substantially + # in the same form as set out herein and that such intended + # distribution, if actually made, will neither violate or otherwise + # contravene any of the laws and regulations of the countries having + # jurisdiction over the User or the intended distribution itself. + # + # NO WARRANTY + # + # The program was produced on an experimental basis in the course of the + # research and development conducted during the project and is provided + # to users as so produced on an experimental basis. Accordingly, the + # program is provided without any warranty whatsoever, whether express, + # implied, statutory or otherwise. The term "warranty" used herein + # includes, but is not limited to, any warranty of the quality, + # performance, merchantability and fitness for a particular purpose of + # the program and the nonexistence of any infringement or violation of + # any right of any third party. + # + # Each user of the program will agree and understand, and be deemed to + # have agreed and understood, that there is no warranty whatsoever for + # the program and, accordingly, the entire risk arising from or + # otherwise connected with the program is assumed by the user. + # + # Therefore, neither ICOT, the copyright holder, or any other + # organization that participated in or was otherwise related to the + # development of the program and their respective officials, directors, + # officers and other employees shall be held liable for any and all + # damages, including, without limitation, general, special, incidental + # and consequential damages, arising out of or otherwise in connection + # with the use or inability to use the program or any product, material + # or result produced or otherwise obtained by using the program, + # regardless of whether they have been advised of, or otherwise had + # knowledge of, the possibility of such damages at any time during the + # project or thereafter. Each user will be deemed to have agreed to the + # foregoing by his or her commencement of use of the program. The term + # "use" as used herein includes, but is not limited to, the use, + # modification, copying and distribution of the program and the + # production of secondary products from the program. + # + # In the case where the program, whether in its original form or + # modified, was distributed or delivered to or received by a user from + # any person, organization or entity other than ICOT, unless it makes or + # grants independently of ICOT any specific warranty to the user in + # writing, such person, organization or entity, will also be exempted + # from and not be held liable to the user for any such damages as noted + # above as far as the program is concerned. + # + # ---------------COPYING.ipadic-----END---------------------------------- + +3. Lao Word Break Dictionary Data (laodict.txt) + + # Copyright (c) 2013 International Business Machines Corporation + # and others. All Rights Reserved. + # + # Project: http://code.google.com/p/lao-dictionary/ + # Dictionary: http://lao-dictionary.googlecode.com/git/Lao-Dictionary.txt + # License: http://lao-dictionary.googlecode.com/git/Lao-Dictionary-LICENSE.txt + # (copied below) + # + # This file is derived from the above dictionary, with slight + # modifications. + # ---------------------------------------------------------------------- + # Copyright (C) 2013 Brian Eugene Wilson, Robert Martin Campbell. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, + # are permitted provided that the following conditions are met: + # + # + # Redistributions of source code must retain the above copyright notice, this + # list of conditions and the following disclaimer. Redistributions in + # binary form must reproduce the above copyright notice, this list of + # conditions and the following disclaimer in the documentation and/or + # other materials provided with the distribution. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # OF THE POSSIBILITY OF SUCH DAMAGE. + # -------------------------------------------------------------------------- + +4. Burmese Word Break Dictionary Data (burmesedict.txt) + + # Copyright (c) 2014 International Business Machines Corporation + # and others. All Rights Reserved. + # + # This list is part of a project hosted at: + # github.com/kanyawtech/myanmar-karen-word-lists + # + # -------------------------------------------------------------------------- + # Copyright (c) 2013, LeRoy Benjamin Sharon + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions + # are met: Redistributions of source code must retain the above + # copyright notice, this list of conditions and the following + # disclaimer. Redistributions in binary form must reproduce the + # above copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided + # with the distribution. + # + # Neither the name Myanmar Karen Word Lists, nor the names of its + # contributors may be used to endorse or promote products derived + # from this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS + # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + # TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR + # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + # SUCH DAMAGE. + # -------------------------------------------------------------------------- + +5. Time Zone Database + + ICU uses the public domain data and code derived from Time Zone +Database for its time zone support. The ownership of the TZ database +is explained in BCP 175: Procedure for Maintaining the Time Zone +Database section 7. + + # 7. Database Ownership + # + # The TZ database itself is not an IETF Contribution or an IETF + # document. Rather it is a pre-existing and regularly updated work + # that is in the public domain, and is intended to remain in the + # public domain. Therefore, BCPs 78 [RFC5378] and 79 [RFC3979] do + # not apply to the TZ Database or contributions that individuals make + # to it. Should any claims be made and substantiated against the TZ + # Database, the organization that is providing the IANA + # Considerations defined in this RFC, under the memorandum of + # understanding with the IETF, currently ICANN, may act in accordance + # with all competent court orders. No ownership claims will be made + # by ICANN or the IETF Trust on the database or the code. Any person + # making a contribution to the database or code waives all rights to + # future claims in that contribution or in the TZ Database. + +6. Google double-conversion + +Copyright 2006-2011, the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/README.md b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/README.md new file mode 100644 index 00000000000..623b8e38430 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/README.md @@ -0,0 +1,29 @@ +# ICU Parts + +This directory contains a small subset of files from the Unicode organization's [ICU repository](https://github.com/unicode-org/icu). + +### License + +The license for these files is contained in the `LICENSE` file within this directory. + +### Contents + +* Source files taken from the [`icu4c/source/common/unicode`](https://github.com/unicode-org/icu/tree/552b01f61127d30d6589aa4bf99468224979b661/icu4c/source/common/unicode) directory: + * `utf8.h` + * `utf16.h` + * `umachine.h` +* Empty source files that are referenced by the above source files, but whose original contents in `libicu` are not needed: + * `ptypes.h` + * `urename.h` + * `utf.h` +* `ICU_SHA` - File containing the Git SHA of the commit in the `icu` repository from which the files were obtained. +* `LICENSE` - The license file from the [`icu4c`](https://github.com/unicode-org/icu/tree/552b01f61127d30d6589aa4bf99468224979b661/icu4c) directory of the `icu` repository. +* `README.md` - This text file. + +### Updating ICU + +To incorporate changes from the upstream `icu` repository: + +* Update `ICU_SHA` with the new Git SHA. +* Update `LICENSE` with the license text from the directory mentioned above. +* Update `utf8.h`, `utf16.h`, and `umachine.h` with their new contents in the `icu` repository. diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/ptypes.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/ptypes.h new file mode 100644 index 00000000000..ac79ad0f983 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/ptypes.h @@ -0,0 +1 @@ +// This file must exist in order for `utf8.h` and `utf16.h` to be used. diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/umachine.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/umachine.h new file mode 100644 index 00000000000..9195824d5b9 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/umachine.h @@ -0,0 +1,448 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +****************************************************************************** +* +* Copyright (C) 1999-2015, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: umachine.h +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999sep13 +* created by: Markus W. Scherer +* +* This file defines basic types and constants for ICU to be +* platform-independent. umachine.h and utf.h are included into +* utypes.h to provide all the general definitions for ICU. +* All of these definitions used to be in utypes.h before +* the UTF-handling macros made this unmaintainable. +*/ + +#ifndef __UMACHINE_H__ +#define __UMACHINE_H__ + + +/** + * \file + * \brief Basic types and constants for UTF + * + *

Basic types and constants for UTF

+ * This file defines basic types and constants for utf.h to be + * platform-independent. umachine.h and utf.h are included into + * utypes.h to provide all the general definitions for ICU. + * All of these definitions used to be in utypes.h before + * the UTF-handling macros made this unmaintainable. + * + */ +/*==========================================================================*/ +/* Include platform-dependent definitions */ +/* which are contained in the platform-specific file platform.h */ +/*==========================================================================*/ + +#include "unicode/ptypes.h" /* platform.h is included in ptypes.h */ + +/* + * ANSI C headers: + * stddef.h defines wchar_t + */ +#include + +/*==========================================================================*/ +/* For C wrappers, we use the symbol U_STABLE. */ +/* This works properly if the includer is C or C++. */ +/* Functions are declared U_STABLE return-type U_EXPORT2 function-name()... */ +/*==========================================================================*/ + +/** + * \def U_CFUNC + * This is used in a declaration of a library private ICU C function. + * @stable ICU 2.4 + */ + +/** + * \def U_CDECL_BEGIN + * This is used to begin a declaration of a library private ICU C API. + * @stable ICU 2.4 + */ + +/** + * \def U_CDECL_END + * This is used to end a declaration of a library private ICU C API + * @stable ICU 2.4 + */ + +#ifdef __cplusplus +# define U_CFUNC extern "C" +# define U_CDECL_BEGIN extern "C" { +# define U_CDECL_END } +#else +# define U_CFUNC extern +# define U_CDECL_BEGIN +# define U_CDECL_END +#endif + +#ifndef U_ATTRIBUTE_DEPRECATED +/** + * \def U_ATTRIBUTE_DEPRECATED + * This is used for GCC specific attributes + * @internal + */ +#if U_GCC_MAJOR_MINOR >= 302 +# define U_ATTRIBUTE_DEPRECATED __attribute__ ((deprecated)) +/** + * \def U_ATTRIBUTE_DEPRECATED + * This is used for Visual C++ specific attributes + * @internal + */ +#elif defined(_MSC_VER) && (_MSC_VER >= 1400) +# define U_ATTRIBUTE_DEPRECATED __declspec(deprecated) +#else +# define U_ATTRIBUTE_DEPRECATED +#endif +#endif + +/** This is used to declare a function as a public ICU C API @stable ICU 2.0*/ +#define U_CAPI U_CFUNC U_EXPORT +/** This is used to declare a function as a stable public ICU C API*/ +#define U_STABLE U_CAPI +/** This is used to declare a function as a draft public ICU C API */ +#define U_DRAFT U_CAPI +/** This is used to declare a function as a deprecated public ICU C API */ +#define U_DEPRECATED U_CAPI U_ATTRIBUTE_DEPRECATED +/** This is used to declare a function as an obsolete public ICU C API */ +#define U_OBSOLETE U_CAPI +/** This is used to declare a function as an internal ICU C API */ +#define U_INTERNAL U_CAPI + +/** + * \def U_OVERRIDE + * Defined to the C++11 "override" keyword if available. + * Denotes a class or member which is an override of the base class. + * May result in an error if it applied to something not an override. + * @internal + */ +#ifndef U_OVERRIDE +#define U_OVERRIDE override +#endif + +/** + * \def U_FINAL + * Defined to the C++11 "final" keyword if available. + * Denotes a class or member which may not be overridden in subclasses. + * May result in an error if subclasses attempt to override. + * @internal + */ +#if !defined(U_FINAL) || defined(U_IN_DOXYGEN) +#define U_FINAL final +#endif + +// Before ICU 65, function-like, multi-statement ICU macros were just defined as +// series of statements wrapped in { } blocks and the caller could choose to +// either treat them as if they were actual functions and end the invocation +// with a trailing ; creating an empty statement after the block or else omit +// this trailing ; using the knowledge that the macro would expand to { }. +// +// But doing so doesn't work well with macros that look like functions and +// compiler warnings about empty statements (ICU-20601) and ICU 65 therefore +// switches to the standard solution of wrapping such macros in do { } while. +// +// This will however break existing code that depends on being able to invoke +// these macros without a trailing ; so to be able to remain compatible with +// such code the wrapper is itself defined as macros so that it's possible to +// build ICU 65 and later with the old macro behaviour, like this: +// +// CPPFLAGS='-DUPRV_BLOCK_MACRO_BEGIN="" -DUPRV_BLOCK_MACRO_END=""' +// runConfigureICU ... + +/** + * \def UPRV_BLOCK_MACRO_BEGIN + * Defined as the "do" keyword by default. + * @internal + */ +#ifndef UPRV_BLOCK_MACRO_BEGIN +#define UPRV_BLOCK_MACRO_BEGIN do +#endif + +/** + * \def UPRV_BLOCK_MACRO_END + * Defined as "while (FALSE)" by default. + * @internal + */ +#ifndef UPRV_BLOCK_MACRO_END +#define UPRV_BLOCK_MACRO_END while (FALSE) +#endif + +/*==========================================================================*/ +/* limits for int32_t etc., like in POSIX inttypes.h */ +/*==========================================================================*/ + +#ifndef INT8_MIN +/** The smallest value an 8 bit signed integer can hold @stable ICU 2.0 */ +# define INT8_MIN ((int8_t)(-128)) +#endif +#ifndef INT16_MIN +/** The smallest value a 16 bit signed integer can hold @stable ICU 2.0 */ +# define INT16_MIN ((int16_t)(-32767-1)) +#endif +#ifndef INT32_MIN +/** The smallest value a 32 bit signed integer can hold @stable ICU 2.0 */ +# define INT32_MIN ((int32_t)(-2147483647-1)) +#endif + +#ifndef INT8_MAX +/** The largest value an 8 bit signed integer can hold @stable ICU 2.0 */ +# define INT8_MAX ((int8_t)(127)) +#endif +#ifndef INT16_MAX +/** The largest value a 16 bit signed integer can hold @stable ICU 2.0 */ +# define INT16_MAX ((int16_t)(32767)) +#endif +#ifndef INT32_MAX +/** The largest value a 32 bit signed integer can hold @stable ICU 2.0 */ +# define INT32_MAX ((int32_t)(2147483647)) +#endif + +#ifndef UINT8_MAX +/** The largest value an 8 bit unsigned integer can hold @stable ICU 2.0 */ +# define UINT8_MAX ((uint8_t)(255U)) +#endif +#ifndef UINT16_MAX +/** The largest value a 16 bit unsigned integer can hold @stable ICU 2.0 */ +# define UINT16_MAX ((uint16_t)(65535U)) +#endif +#ifndef UINT32_MAX +/** The largest value a 32 bit unsigned integer can hold @stable ICU 2.0 */ +# define UINT32_MAX ((uint32_t)(4294967295U)) +#endif + +#if defined(U_INT64_T_UNAVAILABLE) +# error int64_t is required for decimal format and rule-based number format. +#else +# ifndef INT64_C +/** + * Provides a platform independent way to specify a signed 64-bit integer constant. + * note: may be wrong for some 64 bit platforms - ensure your compiler provides INT64_C + * @stable ICU 2.8 + */ +# define INT64_C(c) c ## LL +# endif +# ifndef UINT64_C +/** + * Provides a platform independent way to specify an unsigned 64-bit integer constant. + * note: may be wrong for some 64 bit platforms - ensure your compiler provides UINT64_C + * @stable ICU 2.8 + */ +# define UINT64_C(c) c ## ULL +# endif +# ifndef U_INT64_MIN +/** The smallest value a 64 bit signed integer can hold @stable ICU 2.8 */ +# define U_INT64_MIN ((int64_t)(INT64_C(-9223372036854775807)-1)) +# endif +# ifndef U_INT64_MAX +/** The largest value a 64 bit signed integer can hold @stable ICU 2.8 */ +# define U_INT64_MAX ((int64_t)(INT64_C(9223372036854775807))) +# endif +# ifndef U_UINT64_MAX +/** The largest value a 64 bit unsigned integer can hold @stable ICU 2.8 */ +# define U_UINT64_MAX ((uint64_t)(UINT64_C(18446744073709551615))) +# endif +#endif + +/*==========================================================================*/ +/* Boolean data type */ +/*==========================================================================*/ + +/** The ICU boolean type @stable ICU 2.0 */ +typedef int8_t UBool; + +#ifndef TRUE +/** The TRUE value of a UBool @stable ICU 2.0 */ +# define TRUE 1 +#endif +#ifndef FALSE +/** The FALSE value of a UBool @stable ICU 2.0 */ +# define FALSE 0 +#endif + + +/*==========================================================================*/ +/* Unicode data types */ +/*==========================================================================*/ + +/* wchar_t-related definitions -------------------------------------------- */ + +/* + * \def U_WCHAR_IS_UTF16 + * Defined if wchar_t uses UTF-16. + * + * @stable ICU 2.0 + */ +/* + * \def U_WCHAR_IS_UTF32 + * Defined if wchar_t uses UTF-32. + * + * @stable ICU 2.0 + */ +#if !defined(U_WCHAR_IS_UTF16) && !defined(U_WCHAR_IS_UTF32) +# ifdef __STDC_ISO_10646__ +# if (U_SIZEOF_WCHAR_T==2) +# define U_WCHAR_IS_UTF16 +# elif (U_SIZEOF_WCHAR_T==4) +# define U_WCHAR_IS_UTF32 +# endif +# elif defined __UCS2__ +# if (U_PF_OS390 <= U_PLATFORM && U_PLATFORM <= U_PF_OS400) && (U_SIZEOF_WCHAR_T==2) +# define U_WCHAR_IS_UTF16 +# endif +# elif defined(__UCS4__) || (U_PLATFORM == U_PF_OS400 && defined(__UTF32__)) +# if (U_SIZEOF_WCHAR_T==4) +# define U_WCHAR_IS_UTF32 +# endif +# elif U_PLATFORM_IS_DARWIN_BASED || (U_SIZEOF_WCHAR_T==4 && U_PLATFORM_IS_LINUX_BASED) +# define U_WCHAR_IS_UTF32 +# elif U_PLATFORM_HAS_WIN32_API +# define U_WCHAR_IS_UTF16 +# endif +#endif + +/* UChar and UChar32 definitions -------------------------------------------- */ + +/** Number of bytes in a UChar. @stable ICU 2.0 */ +#define U_SIZEOF_UCHAR 2 + +/** + * \def U_CHAR16_IS_TYPEDEF + * If 1, then char16_t is a typedef and not a real type (yet) + * @internal + */ +#if (U_PLATFORM == U_PF_AIX) && defined(__cplusplus) &&(U_CPLUSPLUS_VERSION < 11) +// for AIX, uchar.h needs to be included +# include +# define U_CHAR16_IS_TYPEDEF 1 +#elif defined(_MSC_VER) && (_MSC_VER < 1900) +// Versions of Visual Studio/MSVC below 2015 do not support char16_t as a real type, +// and instead use a typedef. https://msdn.microsoft.com/library/bb531344.aspx +# define U_CHAR16_IS_TYPEDEF 1 +#else +# define U_CHAR16_IS_TYPEDEF 0 +#endif + + +/** + * \var UChar + * + * The base type for UTF-16 code units and pointers. + * Unsigned 16-bit integer. + * Starting with ICU 59, C++ API uses char16_t directly, while C API continues to use UChar. + * + * UChar is configurable by defining the macro UCHAR_TYPE + * on the preprocessor or compiler command line: + * -DUCHAR_TYPE=uint16_t or -DUCHAR_TYPE=wchar_t (if U_SIZEOF_WCHAR_T==2) etc. + * (The UCHAR_TYPE can also be \#defined earlier in this file, for outside the ICU library code.) + * This is for transitional use from application code that uses uint16_t or wchar_t for UTF-16. + * + * The default is UChar=char16_t. + * + * C++11 defines char16_t as bit-compatible with uint16_t, but as a distinct type. + * + * In C, char16_t is a simple typedef of uint_least16_t. + * ICU requires uint_least16_t=uint16_t for data memory mapping. + * On macOS, char16_t is not available because the uchar.h standard header is missing. + * + * @stable ICU 4.4 + */ + +#if 1 + // #if 1 is normal. UChar defaults to char16_t in C++. + // For configuration testing of UChar=uint16_t temporarily change this to #if 0. + // The intltest Makefile #defines UCHAR_TYPE=char16_t, + // so we only #define it to uint16_t if it is undefined so far. +#elif !defined(UCHAR_TYPE) +# define UCHAR_TYPE uint16_t +#endif + +#if defined(U_COMBINED_IMPLEMENTATION) || defined(U_COMMON_IMPLEMENTATION) || \ + defined(U_I18N_IMPLEMENTATION) || defined(U_IO_IMPLEMENTATION) + // Inside the ICU library code, never configurable. + typedef char16_t UChar; +#elif defined(UCHAR_TYPE) + typedef UCHAR_TYPE UChar; +#elif defined(__cplusplus) + typedef char16_t UChar; +#else + typedef uint16_t UChar; +#endif + +/** + * \var OldUChar + * Default ICU 58 definition of UChar. + * A base type for UTF-16 code units and pointers. + * Unsigned 16-bit integer. + * + * Define OldUChar to be wchar_t if that is 16 bits wide. + * If wchar_t is not 16 bits wide, then define UChar to be uint16_t. + * + * This makes the definition of OldUChar platform-dependent + * but allows direct string type compatibility with platforms with + * 16-bit wchar_t types. + * + * This is how UChar was defined in ICU 58, for transition convenience. + * Exception: ICU 58 UChar was defined to UCHAR_TYPE if that macro was defined. + * The current UChar responds to UCHAR_TYPE but OldUChar does not. + * + * @stable ICU 59 + */ +#if U_SIZEOF_WCHAR_T==2 + typedef wchar_t OldUChar; +#elif defined(__CHAR16_TYPE__) + typedef __CHAR16_TYPE__ OldUChar; +#else + typedef uint16_t OldUChar; +#endif + +/** + * Define UChar32 as a type for single Unicode code points. + * UChar32 is a signed 32-bit integer (same as int32_t). + * + * The Unicode code point range is 0..0x10ffff. + * All other values (negative or >=0x110000) are illegal as Unicode code points. + * They may be used as sentinel values to indicate "done", "error" + * or similar non-code point conditions. + * + * Before ICU 2.4 (Jitterbug 2146), UChar32 was defined + * to be wchar_t if that is 32 bits wide (wchar_t may be signed or unsigned) + * or else to be uint32_t. + * That is, the definition of UChar32 was platform-dependent. + * + * @see U_SENTINEL + * @stable ICU 2.4 + */ +typedef int32_t UChar32; + +/** + * This value is intended for sentinel values for APIs that + * (take or) return single code points (UChar32). + * It is outside of the Unicode code point range 0..0x10ffff. + * + * For example, a "done" or "error" value in a new API + * could be indicated with U_SENTINEL. + * + * ICU APIs designed before ICU 2.4 usually define service-specific "done" + * values, mostly 0xffff. + * Those may need to be distinguished from + * actual U+ffff text contents by calling functions like + * CharacterIterator::hasNext() or UnicodeString::length(). + * + * @return -1 + * @see UChar32 + * @stable ICU 2.4 + */ +#define U_SENTINEL (-1) + +#include "unicode/urename.h" + +#endif diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/urename.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/urename.h new file mode 100644 index 00000000000..ac79ad0f983 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/urename.h @@ -0,0 +1 @@ +// This file must exist in order for `utf8.h` and `utf16.h` to be used. diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/utf.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/utf.h new file mode 100644 index 00000000000..ac79ad0f983 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/utf.h @@ -0,0 +1 @@ +// This file must exist in order for `utf8.h` and `utf16.h` to be used. diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/utf16.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/utf16.h new file mode 100644 index 00000000000..9fd7d5c8a77 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/utf16.h @@ -0,0 +1,733 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* +* Copyright (C) 1999-2012, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: utf16.h +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999sep09 +* created by: Markus W. Scherer +*/ + +/** + * \file + * \brief C API: 16-bit Unicode handling macros + * + * This file defines macros to deal with 16-bit Unicode (UTF-16) code units and strings. + * + * For more information see utf.h and the ICU User Guide Strings chapter + * (http://userguide.icu-project.org/strings). + * + * Usage: + * ICU coding guidelines for if() statements should be followed when using these macros. + * Compound statements (curly braces {}) must be used for if-else-while... + * bodies and all macro statements should be terminated with semicolon. + */ + +#ifndef __UTF16_H__ +#define __UTF16_H__ + +#include "unicode/umachine.h" +#ifndef __UTF_H__ +# include "unicode/utf.h" +#endif + +/* single-code point definitions -------------------------------------------- */ + +/** + * Does this code unit alone encode a code point (BMP, not a surrogate)? + * @param c 16-bit code unit + * @return TRUE or FALSE + * @stable ICU 2.4 + */ +#define U16_IS_SINGLE(c) !U_IS_SURROGATE(c) + +/** + * Is this code unit a lead surrogate (U+d800..U+dbff)? + * @param c 16-bit code unit + * @return TRUE or FALSE + * @stable ICU 2.4 + */ +#define U16_IS_LEAD(c) (((c)&0xfffffc00)==0xd800) + +/** + * Is this code unit a trail surrogate (U+dc00..U+dfff)? + * @param c 16-bit code unit + * @return TRUE or FALSE + * @stable ICU 2.4 + */ +#define U16_IS_TRAIL(c) (((c)&0xfffffc00)==0xdc00) + +/** + * Is this code unit a surrogate (U+d800..U+dfff)? + * @param c 16-bit code unit + * @return TRUE or FALSE + * @stable ICU 2.4 + */ +#define U16_IS_SURROGATE(c) U_IS_SURROGATE(c) + +/** + * Assuming c is a surrogate code point (U16_IS_SURROGATE(c)), + * is it a lead surrogate? + * @param c 16-bit code unit + * @return TRUE or FALSE + * @stable ICU 2.4 + */ +#define U16_IS_SURROGATE_LEAD(c) (((c)&0x400)==0) + +/** + * Assuming c is a surrogate code point (U16_IS_SURROGATE(c)), + * is it a trail surrogate? + * @param c 16-bit code unit + * @return TRUE or FALSE + * @stable ICU 4.2 + */ +#define U16_IS_SURROGATE_TRAIL(c) (((c)&0x400)!=0) + +/** + * Helper constant for U16_GET_SUPPLEMENTARY. + * @internal + */ +#define U16_SURROGATE_OFFSET ((0xd800<<10UL)+0xdc00-0x10000) + +/** + * Get a supplementary code point value (U+10000..U+10ffff) + * from its lead and trail surrogates. + * The result is undefined if the input values are not + * lead and trail surrogates. + * + * @param lead lead surrogate (U+d800..U+dbff) + * @param trail trail surrogate (U+dc00..U+dfff) + * @return supplementary code point (U+10000..U+10ffff) + * @stable ICU 2.4 + */ +#define U16_GET_SUPPLEMENTARY(lead, trail) \ + (((UChar32)(lead)<<10UL)+(UChar32)(trail)-U16_SURROGATE_OFFSET) + + +/** + * Get the lead surrogate (0xd800..0xdbff) for a + * supplementary code point (0x10000..0x10ffff). + * @param supplementary 32-bit code point (U+10000..U+10ffff) + * @return lead surrogate (U+d800..U+dbff) for supplementary + * @stable ICU 2.4 + */ +#define U16_LEAD(supplementary) (UChar)(((supplementary)>>10)+0xd7c0) + +/** + * Get the trail surrogate (0xdc00..0xdfff) for a + * supplementary code point (0x10000..0x10ffff). + * @param supplementary 32-bit code point (U+10000..U+10ffff) + * @return trail surrogate (U+dc00..U+dfff) for supplementary + * @stable ICU 2.4 + */ +#define U16_TRAIL(supplementary) (UChar)(((supplementary)&0x3ff)|0xdc00) + +/** + * How many 16-bit code units are used to encode this Unicode code point? (1 or 2) + * The result is not defined if c is not a Unicode code point (U+0000..U+10ffff). + * @param c 32-bit code point + * @return 1 or 2 + * @stable ICU 2.4 + */ +#define U16_LENGTH(c) ((uint32_t)(c)<=0xffff ? 1 : 2) + +/** + * The maximum number of 16-bit code units per Unicode code point (U+0000..U+10ffff). + * @return 2 + * @stable ICU 2.4 + */ +#define U16_MAX_LENGTH 2 + +/** + * Get a code point from a string at a random-access offset, + * without changing the offset. + * "Unsafe" macro, assumes well-formed UTF-16. + * + * The offset may point to either the lead or trail surrogate unit + * for a supplementary code point, in which case the macro will read + * the adjacent matching surrogate as well. + * The result is undefined if the offset points to a single, unpaired surrogate. + * Iteration through a string is more efficient with U16_NEXT_UNSAFE or U16_NEXT. + * + * @param s const UChar * string + * @param i string offset + * @param c output UChar32 variable + * @see U16_GET + * @stable ICU 2.4 + */ +#define U16_GET_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \ + (c)=(s)[i]; \ + if(U16_IS_SURROGATE(c)) { \ + if(U16_IS_SURROGATE_LEAD(c)) { \ + (c)=U16_GET_SUPPLEMENTARY((c), (s)[(i)+1]); \ + } else { \ + (c)=U16_GET_SUPPLEMENTARY((s)[(i)-1], (c)); \ + } \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Get a code point from a string at a random-access offset, + * without changing the offset. + * "Safe" macro, handles unpaired surrogates and checks for string boundaries. + * + * The offset may point to either the lead or trail surrogate unit + * for a supplementary code point, in which case the macro will read + * the adjacent matching surrogate as well. + * + * The length can be negative for a NUL-terminated string. + * + * If the offset points to a single, unpaired surrogate, then + * c is set to that unpaired surrogate. + * Iteration through a string is more efficient with U16_NEXT_UNSAFE or U16_NEXT. + * + * @param s const UChar * string + * @param start starting string offset (usually 0) + * @param i string offset, must be start<=i(start) && U16_IS_LEAD(__c2=(s)[(i)-1])) { \ + (c)=U16_GET_SUPPLEMENTARY(__c2, (c)); \ + } \ + } \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Get a code point from a string at a random-access offset, + * without changing the offset. + * "Safe" macro, handles unpaired surrogates and checks for string boundaries. + * + * The offset may point to either the lead or trail surrogate unit + * for a supplementary code point, in which case the macro will read + * the adjacent matching surrogate as well. + * + * The length can be negative for a NUL-terminated string. + * + * If the offset points to a single, unpaired surrogate, then + * c is set to U+FFFD. + * Iteration through a string is more efficient with U16_NEXT_UNSAFE or U16_NEXT_OR_FFFD. + * + * @param s const UChar * string + * @param start starting string offset (usually 0) + * @param i string offset, must be start<=i(start) && U16_IS_LEAD(__c2=(s)[(i)-1])) { \ + (c)=U16_GET_SUPPLEMENTARY(__c2, (c)); \ + } else { \ + (c)=0xfffd; \ + } \ + } \ + } \ +} UPRV_BLOCK_MACRO_END + +/* definitions with forward iteration --------------------------------------- */ + +/** + * Get a code point from a string at a code point boundary offset, + * and advance the offset to the next code point boundary. + * (Post-incrementing forward iteration.) + * "Unsafe" macro, assumes well-formed UTF-16. + * + * The offset may point to the lead surrogate unit + * for a supplementary code point, in which case the macro will read + * the following trail surrogate as well. + * If the offset points to a trail surrogate, then that itself + * will be returned as the code point. + * The result is undefined if the offset points to a single, unpaired lead surrogate. + * + * @param s const UChar * string + * @param i string offset + * @param c output UChar32 variable + * @see U16_NEXT + * @stable ICU 2.4 + */ +#define U16_NEXT_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \ + (c)=(s)[(i)++]; \ + if(U16_IS_LEAD(c)) { \ + (c)=U16_GET_SUPPLEMENTARY((c), (s)[(i)++]); \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Get a code point from a string at a code point boundary offset, + * and advance the offset to the next code point boundary. + * (Post-incrementing forward iteration.) + * "Safe" macro, handles unpaired surrogates and checks for string boundaries. + * + * The length can be negative for a NUL-terminated string. + * + * The offset may point to the lead surrogate unit + * for a supplementary code point, in which case the macro will read + * the following trail surrogate as well. + * If the offset points to a trail surrogate or + * to a single, unpaired lead surrogate, then c is set to that unpaired surrogate. + * + * @param s const UChar * string + * @param i string offset, must be i>10)+0xd7c0); \ + (s)[(i)++]=(uint16_t)(((c)&0x3ff)|0xdc00); \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Append a code point to a string, overwriting 1 or 2 code units. + * The offset points to the current end of the string contents + * and is advanced (post-increment). + * "Safe" macro, checks for a valid code point. + * If a surrogate pair is written, checks for sufficient space in the string. + * If the code point is not valid or a trail surrogate does not fit, + * then isError is set to TRUE. + * + * @param s const UChar * string buffer + * @param i string offset, must be i>10)+0xd7c0); \ + (s)[(i)++]=(uint16_t)(((c)&0x3ff)|0xdc00); \ + } else /* c>0x10ffff or not enough space */ { \ + (isError)=TRUE; \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Advance the string offset from one code point boundary to the next. + * (Post-incrementing iteration.) + * "Unsafe" macro, assumes well-formed UTF-16. + * + * @param s const UChar * string + * @param i string offset + * @see U16_FWD_1 + * @stable ICU 2.4 + */ +#define U16_FWD_1_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \ + if(U16_IS_LEAD((s)[(i)++])) { \ + ++(i); \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Advance the string offset from one code point boundary to the next. + * (Post-incrementing iteration.) + * "Safe" macro, handles unpaired surrogates and checks for string boundaries. + * + * The length can be negative for a NUL-terminated string. + * + * @param s const UChar * string + * @param i string offset, must be i0) { \ + U16_FWD_1_UNSAFE(s, i); \ + --__N; \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Advance the string offset from one code point boundary to the n-th next one, + * i.e., move forward by n code points. + * (Post-incrementing iteration.) + * "Safe" macro, handles unpaired surrogates and checks for string boundaries. + * + * The length can be negative for a NUL-terminated string. + * + * @param s const UChar * string + * @param i int32_t string offset, must be i0 && ((i)<(length) || ((length)<0 && (s)[i]!=0))) { \ + U16_FWD_1(s, i, length); \ + --__N; \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Adjust a random-access offset to a code point boundary + * at the start of a code point. + * If the offset points to the trail surrogate of a surrogate pair, + * then the offset is decremented. + * Otherwise, it is not modified. + * "Unsafe" macro, assumes well-formed UTF-16. + * + * @param s const UChar * string + * @param i string offset + * @see U16_SET_CP_START + * @stable ICU 2.4 + */ +#define U16_SET_CP_START_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \ + if(U16_IS_TRAIL((s)[i])) { \ + --(i); \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Adjust a random-access offset to a code point boundary + * at the start of a code point. + * If the offset points to the trail surrogate of a surrogate pair, + * then the offset is decremented. + * Otherwise, it is not modified. + * "Safe" macro, handles unpaired surrogates and checks for string boundaries. + * + * @param s const UChar * string + * @param start starting string offset (usually 0) + * @param i string offset, must be start<=i + * @see U16_SET_CP_START_UNSAFE + * @stable ICU 2.4 + */ +#define U16_SET_CP_START(s, start, i) UPRV_BLOCK_MACRO_BEGIN { \ + if(U16_IS_TRAIL((s)[i]) && (i)>(start) && U16_IS_LEAD((s)[(i)-1])) { \ + --(i); \ + } \ +} UPRV_BLOCK_MACRO_END + +/* definitions with backward iteration -------------------------------------- */ + +/** + * Move the string offset from one code point boundary to the previous one + * and get the code point between them. + * (Pre-decrementing backward iteration.) + * "Unsafe" macro, assumes well-formed UTF-16. + * + * The input offset may be the same as the string length. + * If the offset is behind a trail surrogate unit + * for a supplementary code point, then the macro will read + * the preceding lead surrogate as well. + * If the offset is behind a lead surrogate, then that itself + * will be returned as the code point. + * The result is undefined if the offset is behind a single, unpaired trail surrogate. + * + * @param s const UChar * string + * @param i string offset + * @param c output UChar32 variable + * @see U16_PREV + * @stable ICU 2.4 + */ +#define U16_PREV_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \ + (c)=(s)[--(i)]; \ + if(U16_IS_TRAIL(c)) { \ + (c)=U16_GET_SUPPLEMENTARY((s)[--(i)], (c)); \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Move the string offset from one code point boundary to the previous one + * and get the code point between them. + * (Pre-decrementing backward iteration.) + * "Safe" macro, handles unpaired surrogates and checks for string boundaries. + * + * The input offset may be the same as the string length. + * If the offset is behind a trail surrogate unit + * for a supplementary code point, then the macro will read + * the preceding lead surrogate as well. + * If the offset is behind a lead surrogate or behind a single, unpaired + * trail surrogate, then c is set to that unpaired surrogate. + * + * @param s const UChar * string + * @param start starting string offset (usually 0) + * @param i string offset, must be start(start) && U16_IS_LEAD(__c2=(s)[(i)-1])) { \ + --(i); \ + (c)=U16_GET_SUPPLEMENTARY(__c2, (c)); \ + } \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Move the string offset from one code point boundary to the previous one + * and get the code point between them. + * (Pre-decrementing backward iteration.) + * "Safe" macro, handles unpaired surrogates and checks for string boundaries. + * + * The input offset may be the same as the string length. + * If the offset is behind a trail surrogate unit + * for a supplementary code point, then the macro will read + * the preceding lead surrogate as well. + * If the offset is behind a lead surrogate or behind a single, unpaired + * trail surrogate, then c is set to U+FFFD. + * + * @param s const UChar * string + * @param start starting string offset (usually 0) + * @param i string offset, must be start(start) && U16_IS_LEAD(__c2=(s)[(i)-1])) { \ + --(i); \ + (c)=U16_GET_SUPPLEMENTARY(__c2, (c)); \ + } else { \ + (c)=0xfffd; \ + } \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Move the string offset from one code point boundary to the previous one. + * (Pre-decrementing backward iteration.) + * The input offset may be the same as the string length. + * "Unsafe" macro, assumes well-formed UTF-16. + * + * @param s const UChar * string + * @param i string offset + * @see U16_BACK_1 + * @stable ICU 2.4 + */ +#define U16_BACK_1_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \ + if(U16_IS_TRAIL((s)[--(i)])) { \ + --(i); \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Move the string offset from one code point boundary to the previous one. + * (Pre-decrementing backward iteration.) + * The input offset may be the same as the string length. + * "Safe" macro, handles unpaired surrogates and checks for string boundaries. + * + * @param s const UChar * string + * @param start starting string offset (usually 0) + * @param i string offset, must be start(start) && U16_IS_LEAD((s)[(i)-1])) { \ + --(i); \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Move the string offset from one code point boundary to the n-th one before it, + * i.e., move backward by n code points. + * (Pre-decrementing backward iteration.) + * The input offset may be the same as the string length. + * "Unsafe" macro, assumes well-formed UTF-16. + * + * @param s const UChar * string + * @param i string offset + * @param n number of code points to skip + * @see U16_BACK_N + * @stable ICU 2.4 + */ +#define U16_BACK_N_UNSAFE(s, i, n) UPRV_BLOCK_MACRO_BEGIN { \ + int32_t __N=(n); \ + while(__N>0) { \ + U16_BACK_1_UNSAFE(s, i); \ + --__N; \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Move the string offset from one code point boundary to the n-th one before it, + * i.e., move backward by n code points. + * (Pre-decrementing backward iteration.) + * The input offset may be the same as the string length. + * "Safe" macro, handles unpaired surrogates and checks for string boundaries. + * + * @param s const UChar * string + * @param start start of string + * @param i string offset, must be start0 && (i)>(start)) { \ + U16_BACK_1(s, start, i); \ + --__N; \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Adjust a random-access offset to a code point boundary after a code point. + * If the offset is behind the lead surrogate of a surrogate pair, + * then the offset is incremented. + * Otherwise, it is not modified. + * The input offset may be the same as the string length. + * "Unsafe" macro, assumes well-formed UTF-16. + * + * @param s const UChar * string + * @param i string offset + * @see U16_SET_CP_LIMIT + * @stable ICU 2.4 + */ +#define U16_SET_CP_LIMIT_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \ + if(U16_IS_LEAD((s)[(i)-1])) { \ + ++(i); \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Adjust a random-access offset to a code point boundary after a code point. + * If the offset is behind the lead surrogate of a surrogate pair, + * then the offset is incremented. + * Otherwise, it is not modified. + * The input offset may be the same as the string length. + * "Safe" macro, handles unpaired surrogates and checks for string boundaries. + * + * The length can be negative for a NUL-terminated string. + * + * @param s const UChar * string + * @param start int32_t starting string offset (usually 0) + * @param i int32_t string offset, start<=i<=length + * @param length int32_t string length + * @see U16_SET_CP_LIMIT_UNSAFE + * @stable ICU 2.4 + */ +#define U16_SET_CP_LIMIT(s, start, i, length) UPRV_BLOCK_MACRO_BEGIN { \ + if((start)<(i) && ((i)<(length) || (length)<0) && U16_IS_LEAD((s)[(i)-1]) && U16_IS_TRAIL((s)[i])) { \ + ++(i); \ + } \ +} UPRV_BLOCK_MACRO_END + +#endif diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/utf8.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/utf8.h new file mode 100644 index 00000000000..bb001303747 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/unicode/utf8.h @@ -0,0 +1,881 @@ +// © 2016 and later: Unicode, Inc. and others. +// License & terms of use: http://www.unicode.org/copyright.html +/* +******************************************************************************* +* +* Copyright (C) 1999-2015, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: utf8.h +* encoding: UTF-8 +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999sep13 +* created by: Markus W. Scherer +*/ + +/** + * \file + * \brief C API: 8-bit Unicode handling macros + * + * This file defines macros to deal with 8-bit Unicode (UTF-8) code units (bytes) and strings. + * + * For more information see utf.h and the ICU User Guide Strings chapter + * (http://userguide.icu-project.org/strings). + * + * Usage: + * ICU coding guidelines for if() statements should be followed when using these macros. + * Compound statements (curly braces {}) must be used for if-else-while... + * bodies and all macro statements should be terminated with semicolon. + */ + +#ifndef __UTF8_H__ +#define __UTF8_H__ + +#include "unicode/umachine.h" +#ifndef __UTF_H__ +# include "unicode/utf.h" +#endif + +/* internal definitions ----------------------------------------------------- */ + +/** + * Counts the trail bytes for a UTF-8 lead byte. + * Returns 0 for 0..0xc1 as well as for 0xf5..0xff. + * leadByte might be evaluated multiple times. + * + * This is internal since it is not meant to be called directly by external clients; + * however it is called by public macros in this file and thus must remain stable. + * + * @param leadByte The first byte of a UTF-8 sequence. Must be 0..0xff. + * @internal + */ +#define U8_COUNT_TRAIL_BYTES(leadByte) \ + (U8_IS_LEAD(leadByte) ? \ + ((uint8_t)(leadByte)>=0xe0)+((uint8_t)(leadByte)>=0xf0)+1 : 0) + +/** + * Counts the trail bytes for a UTF-8 lead byte of a valid UTF-8 sequence. + * Returns 0 for 0..0xc1. Undefined for 0xf5..0xff. + * leadByte might be evaluated multiple times. + * + * This is internal since it is not meant to be called directly by external clients; + * however it is called by public macros in this file and thus must remain stable. + * + * @param leadByte The first byte of a UTF-8 sequence. Must be 0..0xff. + * @internal + */ +#define U8_COUNT_TRAIL_BYTES_UNSAFE(leadByte) \ + (((uint8_t)(leadByte)>=0xc2)+((uint8_t)(leadByte)>=0xe0)+((uint8_t)(leadByte)>=0xf0)) + +/** + * Mask a UTF-8 lead byte, leave only the lower bits that form part of the code point value. + * + * This is internal since it is not meant to be called directly by external clients; + * however it is called by public macros in this file and thus must remain stable. + * @internal + */ +#define U8_MASK_LEAD_BYTE(leadByte, countTrailBytes) ((leadByte)&=(1<<(6-(countTrailBytes)))-1) + +/** + * Internal bit vector for 3-byte UTF-8 validity check, for use in U8_IS_VALID_LEAD3_AND_T1. + * Each bit indicates whether one lead byte + first trail byte pair starts a valid sequence. + * Lead byte E0..EF bits 3..0 are used as byte index, + * first trail byte bits 7..5 are used as bit index into that byte. + * @see U8_IS_VALID_LEAD3_AND_T1 + * @internal + */ +#define U8_LEAD3_T1_BITS "\x20\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x10\x30\x30" + +/** + * Internal 3-byte UTF-8 validity check. + * Non-zero if lead byte E0..EF and first trail byte 00..FF start a valid sequence. + * @internal + */ +#define U8_IS_VALID_LEAD3_AND_T1(lead, t1) (U8_LEAD3_T1_BITS[(lead)&0xf]&(1<<((uint8_t)(t1)>>5))) + +/** + * Internal bit vector for 4-byte UTF-8 validity check, for use in U8_IS_VALID_LEAD4_AND_T1. + * Each bit indicates whether one lead byte + first trail byte pair starts a valid sequence. + * First trail byte bits 7..4 are used as byte index, + * lead byte F0..F4 bits 2..0 are used as bit index into that byte. + * @see U8_IS_VALID_LEAD4_AND_T1 + * @internal + */ +#define U8_LEAD4_T1_BITS "\x00\x00\x00\x00\x00\x00\x00\x00\x1E\x0F\x0F\x0F\x00\x00\x00\x00" + +/** + * Internal 4-byte UTF-8 validity check. + * Non-zero if lead byte F0..F4 and first trail byte 00..FF start a valid sequence. + * @internal + */ +#define U8_IS_VALID_LEAD4_AND_T1(lead, t1) (U8_LEAD4_T1_BITS[(uint8_t)(t1)>>4]&(1<<((lead)&7))) + +/** + * Function for handling "next code point" with error-checking. + * + * This is internal since it is not meant to be called directly by external clients; + * however it is U_STABLE (not U_INTERNAL) since it is called by public macros in this + * file and thus must remain stable, and should not be hidden when other internal + * functions are hidden (otherwise public macros would fail to compile). + * @internal + */ +U_STABLE UChar32 U_EXPORT2 +utf8_nextCharSafeBody(const uint8_t *s, int32_t *pi, int32_t length, UChar32 c, UBool strict); + +/** + * Function for handling "append code point" with error-checking. + * + * This is internal since it is not meant to be called directly by external clients; + * however it is U_STABLE (not U_INTERNAL) since it is called by public macros in this + * file and thus must remain stable, and should not be hidden when other internal + * functions are hidden (otherwise public macros would fail to compile). + * @internal + */ +U_STABLE int32_t U_EXPORT2 +utf8_appendCharSafeBody(uint8_t *s, int32_t i, int32_t length, UChar32 c, UBool *pIsError); + +/** + * Function for handling "previous code point" with error-checking. + * + * This is internal since it is not meant to be called directly by external clients; + * however it is U_STABLE (not U_INTERNAL) since it is called by public macros in this + * file and thus must remain stable, and should not be hidden when other internal + * functions are hidden (otherwise public macros would fail to compile). + * @internal + */ +U_STABLE UChar32 U_EXPORT2 +utf8_prevCharSafeBody(const uint8_t *s, int32_t start, int32_t *pi, UChar32 c, UBool strict); + +/** + * Function for handling "skip backward one code point" with error-checking. + * + * This is internal since it is not meant to be called directly by external clients; + * however it is U_STABLE (not U_INTERNAL) since it is called by public macros in this + * file and thus must remain stable, and should not be hidden when other internal + * functions are hidden (otherwise public macros would fail to compile). + * @internal + */ +U_STABLE int32_t U_EXPORT2 +utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i); + +/* single-code point definitions -------------------------------------------- */ + +/** + * Does this code unit (byte) encode a code point by itself (US-ASCII 0..0x7f)? + * @param c 8-bit code unit (byte) + * @return TRUE or FALSE + * @stable ICU 2.4 + */ +#define U8_IS_SINGLE(c) (((c)&0x80)==0) + +/** + * Is this code unit (byte) a UTF-8 lead byte? (0xC2..0xF4) + * @param c 8-bit code unit (byte) + * @return TRUE or FALSE + * @stable ICU 2.4 + */ +#define U8_IS_LEAD(c) ((uint8_t)((c)-0xc2)<=0x32) +// 0x32=0xf4-0xc2 + +/** + * Is this code unit (byte) a UTF-8 trail byte? (0x80..0xBF) + * @param c 8-bit code unit (byte) + * @return TRUE or FALSE + * @stable ICU 2.4 + */ +#define U8_IS_TRAIL(c) ((int8_t)(c)<-0x40) + +/** + * How many code units (bytes) are used for the UTF-8 encoding + * of this Unicode code point? + * @param c 32-bit code point + * @return 1..4, or 0 if c is a surrogate or not a Unicode code point + * @stable ICU 2.4 + */ +#define U8_LENGTH(c) \ + ((uint32_t)(c)<=0x7f ? 1 : \ + ((uint32_t)(c)<=0x7ff ? 2 : \ + ((uint32_t)(c)<=0xd7ff ? 3 : \ + ((uint32_t)(c)<=0xdfff || (uint32_t)(c)>0x10ffff ? 0 : \ + ((uint32_t)(c)<=0xffff ? 3 : 4)\ + ) \ + ) \ + ) \ + ) + +/** + * The maximum number of UTF-8 code units (bytes) per Unicode code point (U+0000..U+10ffff). + * @return 4 + * @stable ICU 2.4 + */ +#define U8_MAX_LENGTH 4 + +/** + * Get a code point from a string at a random-access offset, + * without changing the offset. + * The offset may point to either the lead byte or one of the trail bytes + * for a code point, in which case the macro will read all of the bytes + * for the code point. + * The result is undefined if the offset points to an illegal UTF-8 + * byte sequence. + * Iteration through a string is more efficient with U8_NEXT_UNSAFE or U8_NEXT. + * + * @param s const uint8_t * string + * @param i string offset + * @param c output UChar32 variable + * @see U8_GET + * @stable ICU 2.4 + */ +#define U8_GET_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \ + int32_t _u8_get_unsafe_index=(int32_t)(i); \ + U8_SET_CP_START_UNSAFE(s, _u8_get_unsafe_index); \ + U8_NEXT_UNSAFE(s, _u8_get_unsafe_index, c); \ +} UPRV_BLOCK_MACRO_END + +/** + * Get a code point from a string at a random-access offset, + * without changing the offset. + * The offset may point to either the lead byte or one of the trail bytes + * for a code point, in which case the macro will read all of the bytes + * for the code point. + * + * The length can be negative for a NUL-terminated string. + * + * If the offset points to an illegal UTF-8 byte sequence, then + * c is set to a negative value. + * Iteration through a string is more efficient with U8_NEXT_UNSAFE or U8_NEXT. + * + * @param s const uint8_t * string + * @param start int32_t starting string offset + * @param i int32_t string offset, must be start<=i=0xe0 ? \ + ((c)<0xf0 ? /* U+0800..U+FFFF except surrogates */ \ + U8_LEAD3_T1_BITS[(c)&=0xf]&(1<<((__t=(s)[i])>>5)) && \ + (__t&=0x3f, 1) \ + : /* U+10000..U+10FFFF */ \ + ((c)-=0xf0)<=4 && \ + U8_LEAD4_T1_BITS[(__t=(s)[i])>>4]&(1<<(c)) && \ + ((c)=((c)<<6)|(__t&0x3f), ++(i)!=(length)) && \ + (__t=(s)[i]-0x80)<=0x3f) && \ + /* valid second-to-last trail byte */ \ + ((c)=((c)<<6)|__t, ++(i)!=(length)) \ + : /* U+0080..U+07FF */ \ + (c)>=0xc2 && ((c)&=0x1f, 1)) && \ + /* last trail byte */ \ + (__t=(s)[i]-0x80)<=0x3f && \ + ((c)=((c)<<6)|__t, ++(i), 1)) { \ + } else { \ + (c)=(sub); /* ill-formed*/ \ + } \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Append a code point to a string, overwriting 1 to 4 bytes. + * The offset points to the current end of the string contents + * and is advanced (post-increment). + * "Unsafe" macro, assumes a valid code point and sufficient space in the string. + * Otherwise, the result is undefined. + * + * @param s const uint8_t * string buffer + * @param i string offset + * @param c code point to append + * @see U8_APPEND + * @stable ICU 2.4 + */ +#define U8_APPEND_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \ + uint32_t __uc=(c); \ + if(__uc<=0x7f) { \ + (s)[(i)++]=(uint8_t)__uc; \ + } else { \ + if(__uc<=0x7ff) { \ + (s)[(i)++]=(uint8_t)((__uc>>6)|0xc0); \ + } else { \ + if(__uc<=0xffff) { \ + (s)[(i)++]=(uint8_t)((__uc>>12)|0xe0); \ + } else { \ + (s)[(i)++]=(uint8_t)((__uc>>18)|0xf0); \ + (s)[(i)++]=(uint8_t)(((__uc>>12)&0x3f)|0x80); \ + } \ + (s)[(i)++]=(uint8_t)(((__uc>>6)&0x3f)|0x80); \ + } \ + (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Append a code point to a string, overwriting 1 to 4 bytes. + * The offset points to the current end of the string contents + * and is advanced (post-increment). + * "Safe" macro, checks for a valid code point. + * If a non-ASCII code point is written, checks for sufficient space in the string. + * If the code point is not valid or trail bytes do not fit, + * then isError is set to TRUE. + * + * @param s const uint8_t * string buffer + * @param i int32_t string offset, must be i>6)|0xc0); \ + (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \ + } else if((__uc<=0xd7ff || (0xe000<=__uc && __uc<=0xffff)) && (i)+2<(capacity)) { \ + (s)[(i)++]=(uint8_t)((__uc>>12)|0xe0); \ + (s)[(i)++]=(uint8_t)(((__uc>>6)&0x3f)|0x80); \ + (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \ + } else if(0xffff<__uc && __uc<=0x10ffff && (i)+3<(capacity)) { \ + (s)[(i)++]=(uint8_t)((__uc>>18)|0xf0); \ + (s)[(i)++]=(uint8_t)(((__uc>>12)&0x3f)|0x80); \ + (s)[(i)++]=(uint8_t)(((__uc>>6)&0x3f)|0x80); \ + (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \ + } else { \ + (isError)=TRUE; \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Advance the string offset from one code point boundary to the next. + * (Post-incrementing iteration.) + * "Unsafe" macro, assumes well-formed UTF-8. + * + * @param s const uint8_t * string + * @param i string offset + * @see U8_FWD_1 + * @stable ICU 2.4 + */ +#define U8_FWD_1_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \ + (i)+=1+U8_COUNT_TRAIL_BYTES_UNSAFE((s)[i]); \ +} UPRV_BLOCK_MACRO_END + +/** + * Advance the string offset from one code point boundary to the next. + * (Post-incrementing iteration.) + * "Safe" macro, checks for illegal sequences and for string boundaries. + * + * The length can be negative for a NUL-terminated string. + * + * @param s const uint8_t * string + * @param i int32_t string offset, must be i=0xf0 */ { \ + if(U8_IS_VALID_LEAD4_AND_T1(__b, __t1) && \ + ++(i)!=(length) && U8_IS_TRAIL((s)[i]) && \ + ++(i)!=(length) && U8_IS_TRAIL((s)[i])) { \ + ++(i); \ + } \ + } \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Advance the string offset from one code point boundary to the n-th next one, + * i.e., move forward by n code points. + * (Post-incrementing iteration.) + * "Unsafe" macro, assumes well-formed UTF-8. + * + * @param s const uint8_t * string + * @param i string offset + * @param n number of code points to skip + * @see U8_FWD_N + * @stable ICU 2.4 + */ +#define U8_FWD_N_UNSAFE(s, i, n) UPRV_BLOCK_MACRO_BEGIN { \ + int32_t __N=(n); \ + while(__N>0) { \ + U8_FWD_1_UNSAFE(s, i); \ + --__N; \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Advance the string offset from one code point boundary to the n-th next one, + * i.e., move forward by n code points. + * (Post-incrementing iteration.) + * "Safe" macro, checks for illegal sequences and for string boundaries. + * + * The length can be negative for a NUL-terminated string. + * + * @param s const uint8_t * string + * @param i int32_t string offset, must be i0 && ((i)<(length) || ((length)<0 && (s)[i]!=0))) { \ + U8_FWD_1(s, i, length); \ + --__N; \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Adjust a random-access offset to a code point boundary + * at the start of a code point. + * If the offset points to a UTF-8 trail byte, + * then the offset is moved backward to the corresponding lead byte. + * Otherwise, it is not modified. + * "Unsafe" macro, assumes well-formed UTF-8. + * + * @param s const uint8_t * string + * @param i string offset + * @see U8_SET_CP_START + * @stable ICU 2.4 + */ +#define U8_SET_CP_START_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \ + while(U8_IS_TRAIL((s)[i])) { --(i); } \ +} UPRV_BLOCK_MACRO_END + +/** + * Adjust a random-access offset to a code point boundary + * at the start of a code point. + * If the offset points to a UTF-8 trail byte, + * then the offset is moved backward to the corresponding lead byte. + * Otherwise, it is not modified. + * + * "Safe" macro, checks for illegal sequences and for string boundaries. + * Unlike U8_TRUNCATE_IF_INCOMPLETE(), this macro always reads s[i]. + * + * @param s const uint8_t * string + * @param start int32_t starting string offset (usually 0) + * @param i int32_t string offset, must be start<=i + * @see U8_SET_CP_START_UNSAFE + * @see U8_TRUNCATE_IF_INCOMPLETE + * @stable ICU 2.4 + */ +#define U8_SET_CP_START(s, start, i) UPRV_BLOCK_MACRO_BEGIN { \ + if(U8_IS_TRAIL((s)[(i)])) { \ + (i)=utf8_back1SafeBody(s, start, (i)); \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * If the string ends with a UTF-8 byte sequence that is valid so far + * but incomplete, then reduce the length of the string to end before + * the lead byte of that incomplete sequence. + * For example, if the string ends with E1 80, the length is reduced by 2. + * + * In all other cases (the string ends with a complete sequence, or it is not + * possible for any further trail byte to extend the trailing sequence) + * the length remains unchanged. + * + * Useful for processing text split across multiple buffers + * (save the incomplete sequence for later) + * and for optimizing iteration + * (check for string length only once per character). + * + * "Safe" macro, checks for illegal sequences and for string boundaries. + * Unlike U8_SET_CP_START(), this macro never reads s[length]. + * + * (In UTF-16, simply check for U16_IS_LEAD(last code unit).) + * + * @param s const uint8_t * string + * @param start int32_t starting string offset (usually 0) + * @param length int32_t string length (usually start<=length) + * @see U8_SET_CP_START + * @stable ICU 61 + */ +#define U8_TRUNCATE_IF_INCOMPLETE(s, start, length) UPRV_BLOCK_MACRO_BEGIN { \ + if((length)>(start)) { \ + uint8_t __b1=s[(length)-1]; \ + if(U8_IS_SINGLE(__b1)) { \ + /* common ASCII character */ \ + } else if(U8_IS_LEAD(__b1)) { \ + --(length); \ + } else if(U8_IS_TRAIL(__b1) && ((length)-2)>=(start)) { \ + uint8_t __b2=s[(length)-2]; \ + if(0xe0<=__b2 && __b2<=0xf4) { \ + if(__b2<0xf0 ? U8_IS_VALID_LEAD3_AND_T1(__b2, __b1) : \ + U8_IS_VALID_LEAD4_AND_T1(__b2, __b1)) { \ + (length)-=2; \ + } \ + } else if(U8_IS_TRAIL(__b2) && ((length)-3)>=(start)) { \ + uint8_t __b3=s[(length)-3]; \ + if(0xf0<=__b3 && __b3<=0xf4 && U8_IS_VALID_LEAD4_AND_T1(__b3, __b2)) { \ + (length)-=3; \ + } \ + } \ + } \ + } \ +} UPRV_BLOCK_MACRO_END + +/* definitions with backward iteration -------------------------------------- */ + +/** + * Move the string offset from one code point boundary to the previous one + * and get the code point between them. + * (Pre-decrementing backward iteration.) + * "Unsafe" macro, assumes well-formed UTF-8. + * + * The input offset may be the same as the string length. + * If the offset is behind a multi-byte sequence, then the macro will read + * the whole sequence. + * If the offset is behind a lead byte, then that itself + * will be returned as the code point. + * The result is undefined if the offset is behind an illegal UTF-8 sequence. + * + * @param s const uint8_t * string + * @param i string offset + * @param c output UChar32 variable + * @see U8_PREV + * @stable ICU 2.4 + */ +#define U8_PREV_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \ + (c)=(uint8_t)(s)[--(i)]; \ + if(U8_IS_TRAIL(c)) { \ + uint8_t __b, __count=1, __shift=6; \ +\ + /* c is a trail byte */ \ + (c)&=0x3f; \ + for(;;) { \ + __b=(s)[--(i)]; \ + if(__b>=0xc0) { \ + U8_MASK_LEAD_BYTE(__b, __count); \ + (c)|=(UChar32)__b<<__shift; \ + break; \ + } else { \ + (c)|=(UChar32)(__b&0x3f)<<__shift; \ + ++__count; \ + __shift+=6; \ + } \ + } \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Move the string offset from one code point boundary to the previous one + * and get the code point between them. + * (Pre-decrementing backward iteration.) + * "Safe" macro, checks for illegal sequences and for string boundaries. + * + * The input offset may be the same as the string length. + * If the offset is behind a multi-byte sequence, then the macro will read + * the whole sequence. + * If the offset is behind a lead byte, then that itself + * will be returned as the code point. + * If the offset is behind an illegal UTF-8 sequence, then c is set to a negative value. + * + * @param s const uint8_t * string + * @param start int32_t starting string offset (usually 0) + * @param i int32_t string offset, must be start0) { \ + U8_BACK_1_UNSAFE(s, i); \ + --__N; \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Move the string offset from one code point boundary to the n-th one before it, + * i.e., move backward by n code points. + * (Pre-decrementing backward iteration.) + * The input offset may be the same as the string length. + * "Safe" macro, checks for illegal sequences and for string boundaries. + * + * @param s const uint8_t * string + * @param start int32_t index of the start of the string + * @param i int32_t string offset, must be start0 && (i)>(start)) { \ + U8_BACK_1(s, start, i); \ + --__N; \ + } \ +} UPRV_BLOCK_MACRO_END + +/** + * Adjust a random-access offset to a code point boundary after a code point. + * If the offset is behind a partial multi-byte sequence, + * then the offset is incremented to behind the whole sequence. + * Otherwise, it is not modified. + * The input offset may be the same as the string length. + * "Unsafe" macro, assumes well-formed UTF-8. + * + * @param s const uint8_t * string + * @param i string offset + * @see U8_SET_CP_LIMIT + * @stable ICU 2.4 + */ +#define U8_SET_CP_LIMIT_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \ + U8_BACK_1_UNSAFE(s, i); \ + U8_FWD_1_UNSAFE(s, i); \ +} UPRV_BLOCK_MACRO_END + +/** + * Adjust a random-access offset to a code point boundary after a code point. + * If the offset is behind a partial multi-byte sequence, + * then the offset is incremented to behind the whole sequence. + * Otherwise, it is not modified. + * The input offset may be the same as the string length. + * "Safe" macro, checks for illegal sequences and for string boundaries. + * + * The length can be negative for a NUL-terminated string. + * + * @param s const uint8_t * string + * @param start int32_t starting string offset (usually 0) + * @param i int32_t string offset, must be start<=i<=length + * @param length int32_t string length + * @see U8_SET_CP_LIMIT_UNSAFE + * @stable ICU 2.4 + */ +#define U8_SET_CP_LIMIT(s, start, i, length) UPRV_BLOCK_MACRO_BEGIN { \ + if((start)<(i) && ((i)<(length) || (length)<0)) { \ + U8_BACK_1(s, start, i); \ + U8_FWD_1(s, i, length); \ + } \ +} UPRV_BLOCK_MACRO_END + +#endif diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/wasm_store.c b/Externals/crystaledit/editlib/tree-sitter/lib/src/wasm_store.c new file mode 100644 index 00000000000..e844c2a0d2e --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/wasm_store.c @@ -0,0 +1,1937 @@ +#include "tree_sitter/api.h" +#include "./parser.h" +#include + +#ifdef TREE_SITTER_FEATURE_WASM + +#include "./alloc.h" +#include "./array.h" +#include "./atomic.h" +#include "./language.h" +#include "./lexer.h" +#include "./wasm/wasm-stdlib.h" +#include "./wasm_store.h" + +#include +#include +#include + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4100) +#elif defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +#define array_len(a) (sizeof(a) / sizeof(a[0])) + +// The following symbols from the C and C++ standard libraries are available +// for external scanners to use. +const char *STDLIB_SYMBOLS[] = { + #include "./stdlib-symbols.txt" +}; + +// The contents of the `dylink.0` custom section of a wasm module, +// as specified by the current WebAssembly dynamic linking ABI proposal. +typedef struct { + uint32_t memory_size; + uint32_t memory_align; + uint32_t table_size; + uint32_t table_align; +} WasmDylinkInfo; + +// WasmLanguageId - A pointer used to identify a language. This language id is +// reference-counted, so that its ownership can be shared between the language +// itself and the instances of the language that are held in wasm stores. +typedef struct { + volatile uint32_t ref_count; + volatile uint32_t is_language_deleted; +} WasmLanguageId; + +// LanguageWasmModule - Additional data associated with a wasm-backed +// `TSLanguage`. This data is read-only and does not reference a particular +// wasm store, so it can be shared by all users of a `TSLanguage`. A pointer to +// this is stored on the language itself. +typedef struct { + volatile uint32_t ref_count; + WasmLanguageId *language_id; + wasmtime_module_t *module; + const char *name; + char *symbol_name_buffer; + char *field_name_buffer; + WasmDylinkInfo dylink_info; +} LanguageWasmModule; + +// LanguageWasmInstance - Additional data associated with an instantiation of +// a `TSLanguage` in a particular wasm store. The wasm store holds one of +// these structs for each language that it has instantiated. +typedef struct { + WasmLanguageId *language_id; + wasmtime_instance_t instance; + int32_t external_states_address; + int32_t lex_main_fn_index; + int32_t lex_keyword_fn_index; + int32_t scanner_create_fn_index; + int32_t scanner_destroy_fn_index; + int32_t scanner_serialize_fn_index; + int32_t scanner_deserialize_fn_index; + int32_t scanner_scan_fn_index; +} LanguageWasmInstance; + +typedef struct { + uint32_t reset_heap; + uint32_t proc_exit; + uint32_t abort; + uint32_t assert_fail; + uint32_t notify_memory_growth; + uint32_t debug_message; + uint32_t at_exit; + uint32_t args_get; + uint32_t args_sizes_get; +} BuiltinFunctionIndices; + +// TSWasmStore - A struct that allows a given `Parser` to use wasm-backed +// languages. This struct is mutable, and can only be used by one parser at a +// time. +struct TSWasmStore { + wasm_engine_t *engine; + wasmtime_store_t *store; + wasmtime_table_t function_table; + wasmtime_memory_t memory; + TSLexer *current_lexer; + LanguageWasmInstance *current_instance; + Array(LanguageWasmInstance) language_instances; + uint32_t current_memory_offset; + uint32_t current_function_table_offset; + uint32_t *stdlib_fn_indices; + BuiltinFunctionIndices builtin_fn_indices; + wasmtime_global_t stack_pointer_global; + wasm_globaltype_t *const_i32_type; + bool has_error; + uint32_t lexer_address; +}; + +typedef Array(char) StringData; + +// LanguageInWasmMemory - The memory layout of a `TSLanguage` when compiled to +// wasm32. This is used to copy static language data out of the wasm memory. +typedef struct { + uint32_t abi_version; + uint32_t symbol_count; + uint32_t alias_count; + uint32_t token_count; + uint32_t external_token_count; + uint32_t state_count; + uint32_t large_state_count; + uint32_t production_id_count; + uint32_t field_count; + uint16_t max_alias_sequence_length; + int32_t parse_table; + int32_t small_parse_table; + int32_t small_parse_table_map; + int32_t parse_actions; + int32_t symbol_names; + int32_t field_names; + int32_t field_map_slices; + int32_t field_map_entries; + int32_t symbol_metadata; + int32_t public_symbol_map; + int32_t alias_map; + int32_t alias_sequences; + int32_t lex_modes; + int32_t lex_fn; + int32_t keyword_lex_fn; + TSSymbol keyword_capture_token; + struct { + int32_t states; + int32_t symbol_map; + int32_t create; + int32_t destroy; + int32_t scan; + int32_t serialize; + int32_t deserialize; + } external_scanner; + int32_t primary_state_ids; + int32_t name; + int32_t reserved_words; + uint16_t max_reserved_word_set_size; + uint32_t supertype_count; + int32_t supertype_symbols; + int32_t supertype_map_slices; + int32_t supertype_map_entries; + TSLanguageMetadata metadata; +} LanguageInWasmMemory; + +// LexerInWasmMemory - The memory layout of a `TSLexer` when compiled to wasm32. +// This is used to copy mutable lexing state in and out of the wasm memory. +typedef struct { + int32_t lookahead; + TSSymbol result_symbol; + int32_t advance; + int32_t mark_end; + int32_t get_column; + int32_t is_at_included_range_start; + int32_t eof; +} LexerInWasmMemory; + +// Linear memory layout: +// [ <-- stack | stdlib statics | lexer | language statics --> | serialization_buffer | heap --> ] +#define MAX_MEMORY_SIZE (128 * 1024 * 1024 / MEMORY_PAGE_SIZE) + +/************************ + * WasmDylinkMemoryInfo + ***********************/ + +static uint8_t read_u8(const uint8_t **p) { + return *(*p)++; +} + +static inline uint64_t read_uleb128(const uint8_t **p, const uint8_t *end) { + uint64_t value = 0; + unsigned shift = 0; + do { + if (*p == end) return UINT64_MAX; + value += (uint64_t)(**p & 0x7f) << shift; + shift += 7; + } while (*((*p)++) >= 128); + return value; +} + +static bool wasm_dylink_info__parse( + const uint8_t *bytes, + size_t length, + WasmDylinkInfo *info +) { + const uint8_t WASM_MAGIC_NUMBER[4] = {0, 'a', 's', 'm'}; + const uint8_t WASM_VERSION[4] = {1, 0, 0, 0}; + const uint8_t WASM_CUSTOM_SECTION = 0x0; + const uint8_t WASM_DYLINK_MEM_INFO = 0x1; + + const uint8_t *p = bytes; + const uint8_t *end = bytes + length; + + if (length < 8) return false; + if (memcmp(p, WASM_MAGIC_NUMBER, 4) != 0) return false; + p += 4; + if (memcmp(p, WASM_VERSION, 4) != 0) return false; + p += 4; + + while (p < end) { + uint8_t section_id = read_u8(&p); + uint32_t section_length = read_uleb128(&p, end); + const uint8_t *section_end = p + section_length; + if (section_end > end) return false; + + if (section_id == WASM_CUSTOM_SECTION) { + uint32_t name_length = read_uleb128(&p, section_end); + const uint8_t *name_end = p + name_length; + if (name_end > section_end) return false; + + if (name_length == 8 && memcmp(p, "dylink.0", 8) == 0) { + p = name_end; + while (p < section_end) { + uint8_t subsection_type = read_u8(&p); + uint32_t subsection_size = read_uleb128(&p, section_end); + const uint8_t *subsection_end = p + subsection_size; + if (subsection_end > section_end) return false; + if (subsection_type == WASM_DYLINK_MEM_INFO) { + info->memory_size = read_uleb128(&p, subsection_end); + info->memory_align = read_uleb128(&p, subsection_end); + info->table_size = read_uleb128(&p, subsection_end); + info->table_align = read_uleb128(&p, subsection_end); + return true; + } + p = subsection_end; + } + } + } + p = section_end; + } + return false; +} + +/******************************************* + * Native callbacks exposed to wasm modules + *******************************************/ + + static wasm_trap_t *callback__abort( + void *env, + wasmtime_caller_t* caller, + wasmtime_val_raw_t *args_and_results, + size_t args_and_results_len +) { + return wasmtime_trap_new("wasm module called abort", 24); +} + +static wasm_trap_t *callback__debug_message( + void *env, + wasmtime_caller_t* caller, + wasmtime_val_raw_t *args_and_results, + size_t args_and_results_len +) { + wasmtime_context_t *context = wasmtime_caller_context(caller); + TSWasmStore *store = env; + ts_assert(args_and_results_len == 2); + uint32_t string_address = args_and_results[0].i32; + uint32_t value = args_and_results[1].i32; + uint8_t *memory = wasmtime_memory_data(context, &store->memory); + printf("DEBUG: %s %u\n", &memory[string_address], value); + return NULL; +} + +static wasm_trap_t *callback__noop( + void *env, + wasmtime_caller_t* caller, + wasmtime_val_raw_t *args_and_results, + size_t args_and_results_len +) { + return NULL; +} + +static wasm_trap_t *callback__lexer_advance( + void *env, + wasmtime_caller_t* caller, + wasmtime_val_raw_t *args_and_results, + size_t args_and_results_len +) { + wasmtime_context_t *context = wasmtime_caller_context(caller); + ts_assert(args_and_results_len == 2); + + TSWasmStore *store = env; + TSLexer *lexer = store->current_lexer; + bool skip = args_and_results[1].i32; + lexer->advance(lexer, skip); + + uint8_t *memory = wasmtime_memory_data(context, &store->memory); + memcpy(&memory[store->lexer_address], &lexer->lookahead, sizeof(lexer->lookahead)); + return NULL; +} + +static wasm_trap_t *callback__lexer_mark_end( + void *env, + wasmtime_caller_t* caller, + wasmtime_val_raw_t *args_and_results, + size_t args_and_results_len +) { + TSWasmStore *store = env; + TSLexer *lexer = store->current_lexer; + lexer->mark_end(lexer); + return NULL; +} + +static wasm_trap_t *callback__lexer_get_column( + void *env, + wasmtime_caller_t* caller, + wasmtime_val_raw_t *args_and_results, + size_t args_and_results_len +) { + TSWasmStore *store = env; + TSLexer *lexer = store->current_lexer; + uint32_t result = lexer->get_column(lexer); + args_and_results[0].i32 = result; + return NULL; +} + +static wasm_trap_t *callback__lexer_is_at_included_range_start( + void *env, + wasmtime_caller_t* caller, + wasmtime_val_raw_t *args_and_results, + size_t args_and_results_len +) { + TSWasmStore *store = env; + TSLexer *lexer = store->current_lexer; + bool result = lexer->is_at_included_range_start(lexer); + args_and_results[0].i32 = result; + return NULL; +} + +static wasm_trap_t *callback__lexer_eof( + void *env, + wasmtime_caller_t* caller, + wasmtime_val_raw_t *args_and_results, + size_t args_and_results_len +) { + TSWasmStore *store = env; + TSLexer *lexer = store->current_lexer; + bool result = lexer->eof(lexer); + args_and_results[0].i32 = result; + return NULL; +} + +typedef struct { + uint32_t *storage_location; + wasmtime_func_unchecked_callback_t callback; + wasm_functype_t *type; +} FunctionDefinition; + +static void *copy(const void *data, size_t size) { + void *result = ts_malloc(size); + memcpy(result, data, size); + return result; +} + +static void *copy_unsized_static_array( + const uint8_t *data, + int32_t start_address, + const int32_t all_addresses[], + size_t address_count +) { + int32_t end_address = 0; + for (unsigned i = 0; i < address_count; i++) { + if (all_addresses[i] > start_address) { + if (!end_address || all_addresses[i] < end_address) { + end_address = all_addresses[i]; + } + } + } + + if (!end_address) return NULL; + size_t size = end_address - start_address; + void *result = ts_malloc(size); + memcpy(result, &data[start_address], size); + return result; +} + +static void *copy_strings( + const uint8_t *data, + int32_t array_address, + size_t count, + StringData *string_data +) { + const char **result = ts_malloc(count * sizeof(char *)); + for (unsigned i = 0; i < count; i++) { + int32_t address; + memcpy(&address, &data[array_address + i * sizeof(address)], sizeof(address)); + if (address == 0) { + result[i] = (const char *)-1; + } else { + const uint8_t *string = &data[address]; + uint32_t len = strlen((const char *)string); + result[i] = (const char *)(uintptr_t)string_data->size; + array_extend(string_data, len + 1, string); + } + } + for (unsigned i = 0; i < count; i++) { + if (result[i] == (const char *)-1) { + result[i] = NULL; + } else { + result[i] = string_data->contents + (uintptr_t)result[i]; + } + } + return result; +} + +static void *copy_string( + const uint8_t *data, + int32_t address +) { + const char *string = (const char *)&data[address]; + size_t len = strlen(string); + char *result = ts_malloc(len + 1); + memcpy(result, string, len + 1); + return result; +} + +static bool name_eq(const wasm_name_t *name, const char *string) { + return strncmp(string, name->data, name->size) == 0; +} + +static inline wasm_functype_t* wasm_functype_new_4_0( + wasm_valtype_t* p1, + wasm_valtype_t* p2, + wasm_valtype_t* p3, + wasm_valtype_t* p4 +) { + wasm_valtype_t* ps[4] = {p1, p2, p3, p4}; + wasm_valtype_vec_t params, results; + wasm_valtype_vec_new(¶ms, 4, ps); + wasm_valtype_vec_new_empty(&results); + return wasm_functype_new(¶ms, &results); +} + +#define format(output, ...) \ + do { \ + size_t message_length = snprintf((char *)NULL, 0, __VA_ARGS__); \ + *output = ts_malloc(message_length + 1); \ + snprintf(*output, message_length + 1, __VA_ARGS__); \ + } while (0) + +WasmLanguageId *language_id_new(void) { + WasmLanguageId *self = ts_malloc(sizeof(WasmLanguageId)); + self->is_language_deleted = false; + self->ref_count = 1; + return self; +} + +WasmLanguageId *language_id_clone(WasmLanguageId *self) { + atomic_inc(&self->ref_count); + return self; +} + +void language_id_delete(WasmLanguageId *self) { + if (atomic_dec(&self->ref_count) == 0) { + ts_free(self); + } +} + +static wasmtime_extern_t get_builtin_extern( + wasmtime_table_t *table, + unsigned index +) { + return (wasmtime_extern_t) { + .kind = WASMTIME_EXTERN_FUNC, + .of.func = (wasmtime_func_t) { + .store_id = table->store_id, + .__private = index + } + }; +} + +static bool ts_wasm_store__provide_builtin_import( + TSWasmStore *self, + const wasm_name_t *import_name, + wasmtime_extern_t *import +) { + wasmtime_error_t *error = NULL; + wasmtime_context_t *context = wasmtime_store_context(self->store); + + // Dynamic linking parameters + if (name_eq(import_name, "__memory_base")) { + wasmtime_val_t value = WASM_I32_VAL(self->current_memory_offset); + wasmtime_global_t global; + error = wasmtime_global_new(context, self->const_i32_type, &value, &global); + ts_assert(!error); + *import = (wasmtime_extern_t) {.kind = WASMTIME_EXTERN_GLOBAL, .of.global = global}; + } else if (name_eq(import_name, "__table_base")) { + wasmtime_val_t value = WASM_I32_VAL(self->current_function_table_offset); + wasmtime_global_t global; + error = wasmtime_global_new(context, self->const_i32_type, &value, &global); + ts_assert(!error); + *import = (wasmtime_extern_t) {.kind = WASMTIME_EXTERN_GLOBAL, .of.global = global}; + } else if (name_eq(import_name, "__stack_pointer")) { + *import = (wasmtime_extern_t) {.kind = WASMTIME_EXTERN_GLOBAL, .of.global = self->stack_pointer_global}; + } else if (name_eq(import_name, "__indirect_function_table")) { + *import = (wasmtime_extern_t) {.kind = WASMTIME_EXTERN_TABLE, .of.table = self->function_table}; + } else if (name_eq(import_name, "memory")) { + *import = (wasmtime_extern_t) {.kind = WASMTIME_EXTERN_MEMORY, .of.memory = self->memory}; + } + + // Builtin functions + else if (name_eq(import_name, "__assert_fail")) { + *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.assert_fail); + } else if (name_eq(import_name, "__cxa_atexit")) { + *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.at_exit); + } else if (name_eq(import_name, "args_get")) { + *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.args_get); + } else if (name_eq(import_name, "args_sizes_get")) { + *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.args_sizes_get); + } else if (name_eq(import_name, "abort")) { + *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.abort); + } else if (name_eq(import_name, "proc_exit")) { + *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.proc_exit); + } else if (name_eq(import_name, "emscripten_notify_memory_growth")) { + *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.notify_memory_growth); + } else if (name_eq(import_name, "tree_sitter_debug_message")) { + *import = get_builtin_extern(&self->function_table, self->builtin_fn_indices.debug_message); + } else { + return false; + } + + return true; +} + +static bool ts_wasm_store__call_module_initializer( + TSWasmStore *self, + const wasm_name_t *export_name, + wasmtime_extern_t *export, + wasm_trap_t **trap +) { + if ( + name_eq(export_name, "_initialize") || + name_eq(export_name, "__wasm_apply_data_relocs") || + name_eq(export_name, "__wasm_call_ctors") + ) { + wasmtime_context_t *context = wasmtime_store_context(self->store); + wasmtime_func_t initialization_func = export->of.func; + wasmtime_error_t *error = wasmtime_func_call(context, &initialization_func, NULL, 0, NULL, 0, trap); + ts_assert(!error); + return true; + } else { + return false; + } +} + +TSWasmStore *ts_wasm_store_new(TSWasmEngine *engine, TSWasmError *wasm_error) { + TSWasmStore *self = ts_calloc(1, sizeof(TSWasmStore)); + wasmtime_store_t *store = wasmtime_store_new(engine, self, NULL); + wasmtime_context_t *context = wasmtime_store_context(store); + wasmtime_error_t *error = NULL; + wasm_trap_t *trap = NULL; + wasm_message_t message = WASM_EMPTY_VEC; + wasm_exporttype_vec_t export_types = WASM_EMPTY_VEC; + wasm_importtype_vec_t import_types = WASM_EMPTY_VEC; + wasmtime_extern_t *imports = NULL; + wasmtime_module_t *stdlib_module = NULL; + wasm_memorytype_t *memory_type = NULL; + wasm_tabletype_t *table_type = NULL; + + // Define functions called by scanners via function pointers on the lexer. + LexerInWasmMemory lexer = { + .lookahead = 0, + .result_symbol = 0, + }; + FunctionDefinition lexer_definitions[] = { + { + (uint32_t *)&lexer.advance, + callback__lexer_advance, + wasm_functype_new_2_0(wasm_valtype_new_i32(), wasm_valtype_new_i32()) + }, + { + (uint32_t *)&lexer.mark_end, + callback__lexer_mark_end, + wasm_functype_new_1_0(wasm_valtype_new_i32()) + }, + { + (uint32_t *)&lexer.get_column, + callback__lexer_get_column, + wasm_functype_new_1_1(wasm_valtype_new_i32(), wasm_valtype_new_i32()) + }, + { + (uint32_t *)&lexer.is_at_included_range_start, + callback__lexer_is_at_included_range_start, + wasm_functype_new_1_1(wasm_valtype_new_i32(), wasm_valtype_new_i32()) + }, + { + (uint32_t *)&lexer.eof, + callback__lexer_eof, + wasm_functype_new_1_1(wasm_valtype_new_i32(), wasm_valtype_new_i32()) + }, + }; + + // Define builtin functions that can be imported by scanners. + BuiltinFunctionIndices builtin_fn_indices; + FunctionDefinition builtin_definitions[] = { + { + &builtin_fn_indices.proc_exit, + callback__abort, + wasm_functype_new_1_0(wasm_valtype_new_i32()) + }, + { + &builtin_fn_indices.abort, + callback__abort, + wasm_functype_new_0_0() + }, + { + &builtin_fn_indices.assert_fail, + callback__abort, + wasm_functype_new_4_0(wasm_valtype_new_i32(), wasm_valtype_new_i32(), wasm_valtype_new_i32(), wasm_valtype_new_i32()) + }, + { + &builtin_fn_indices.notify_memory_growth, + callback__noop, + wasm_functype_new_1_0(wasm_valtype_new_i32()) + }, + { + &builtin_fn_indices.debug_message, + callback__debug_message, + wasm_functype_new_2_0(wasm_valtype_new_i32(), wasm_valtype_new_i32()) + }, + { + &builtin_fn_indices.at_exit, + callback__noop, + wasm_functype_new_3_1(wasm_valtype_new_i32(), wasm_valtype_new_i32(), wasm_valtype_new_i32(), wasm_valtype_new_i32()) + }, + { + &builtin_fn_indices.args_get, + callback__noop, + wasm_functype_new_2_1(wasm_valtype_new_i32(), wasm_valtype_new_i32(), wasm_valtype_new_i32()) + }, + { + &builtin_fn_indices.args_sizes_get, + callback__noop, + wasm_functype_new_2_1(wasm_valtype_new_i32(), wasm_valtype_new_i32(), wasm_valtype_new_i32()) + }, + }; + + // Create all of the wasm functions. + unsigned builtin_definitions_len = array_len(builtin_definitions); + unsigned lexer_definitions_len = array_len(lexer_definitions); + for (unsigned i = 0; i < builtin_definitions_len; i++) { + FunctionDefinition *definition = &builtin_definitions[i]; + wasmtime_func_t func; + wasmtime_func_new_unchecked(context, definition->type, definition->callback, self, NULL, &func); + *definition->storage_location = func.__private; + wasm_functype_delete(definition->type); + } + for (unsigned i = 0; i < lexer_definitions_len; i++) { + FunctionDefinition *definition = &lexer_definitions[i]; + wasmtime_func_t func; + wasmtime_func_new_unchecked(context, definition->type, definition->callback, self, NULL, &func); + *definition->storage_location = func.__private; + wasm_functype_delete(definition->type); + } + + // Compile the stdlib module. + error = wasmtime_module_new(engine, STDLIB_WASM, STDLIB_WASM_LEN, &stdlib_module); + if (error) { + wasmtime_error_message(error, &message); + wasm_error->kind = TSWasmErrorKindCompile; + format( + &wasm_error->message, + "failed to compile wasm stdlib: %.*s", + (int)message.size, message.data + ); + goto error; + } + + // Retrieve the stdlib module's imports. + wasmtime_module_imports(stdlib_module, &import_types); + + // Find the initial number of memory pages needed by the stdlib. + const wasm_memorytype_t *stdlib_memory_type = NULL; + for (unsigned i = 0; i < import_types.size; i++) { + wasm_importtype_t *import_type = import_types.data[i]; + const wasm_name_t *import_name = wasm_importtype_name(import_type); + if (name_eq(import_name, "memory")) { + const wasm_externtype_t *type = wasm_importtype_type(import_type); + stdlib_memory_type = wasm_externtype_as_memorytype_const(type); + } + } + if (!stdlib_memory_type) { + wasm_error->kind = TSWasmErrorKindCompile; + format( + &wasm_error->message, + "wasm stdlib is missing the 'memory' import" + ); + goto error; + } + + // Initialize store's memory + uint64_t initial_memory_pages = wasmtime_memorytype_minimum(stdlib_memory_type); + wasm_limits_t memory_limits = {.min = initial_memory_pages, .max = MAX_MEMORY_SIZE}; + memory_type = wasm_memorytype_new(&memory_limits); + wasmtime_memory_t memory; + error = wasmtime_memory_new(context, memory_type, &memory); + if (error) { + wasmtime_error_message(error, &message); + wasm_error->kind = TSWasmErrorKindAllocate; + format( + &wasm_error->message, + "failed to allocate wasm memory: %.*s", + (int)message.size, message.data + ); + goto error; + } + wasm_memorytype_delete(memory_type); + memory_type = NULL; + + // Initialize store's function table + wasm_limits_t table_limits = {.min = 1, .max = wasm_limits_max_default}; + table_type = wasm_tabletype_new(wasm_valtype_new(WASM_FUNCREF), &table_limits); + wasmtime_val_t initializer = {.kind = WASMTIME_FUNCREF}; + wasmtime_table_t function_table; + error = wasmtime_table_new(context, table_type, &initializer, &function_table); + if (error) { + wasmtime_error_message(error, &message); + wasm_error->kind = TSWasmErrorKindAllocate; + format( + &wasm_error->message, + "failed to allocate wasm table: %.*s", + (int)message.size, message.data + ); + goto error; + } + wasm_tabletype_delete(table_type); + table_type = NULL; + + unsigned stdlib_symbols_len = array_len(STDLIB_SYMBOLS); + + // Define globals for the stack and heap start addresses. + wasm_globaltype_t *const_i32_type = wasm_globaltype_new(wasm_valtype_new_i32(), WASM_CONST); + wasm_globaltype_t *var_i32_type = wasm_globaltype_new(wasm_valtype_new_i32(), WASM_VAR); + + wasmtime_val_t stack_pointer_value = WASM_I32_VAL(0); + wasmtime_global_t stack_pointer_global; + error = wasmtime_global_new(context, var_i32_type, &stack_pointer_value, &stack_pointer_global); + wasm_globaltype_delete(var_i32_type); + ts_assert(!error); + + *self = (TSWasmStore) { + .engine = wasmtime_engine_clone(engine), + .store = store, + .memory = memory, + .function_table = function_table, + .language_instances = array_new(), + .stdlib_fn_indices = ts_calloc(stdlib_symbols_len, sizeof(uint32_t)), + .builtin_fn_indices = builtin_fn_indices, + .stack_pointer_global = stack_pointer_global, + .current_memory_offset = 0, + .current_function_table_offset = 0, + .const_i32_type = const_i32_type, + }; + + // Set up the imports for the stdlib module. + imports = ts_calloc(import_types.size, sizeof(wasmtime_extern_t)); + for (unsigned i = 0; i < import_types.size; i++) { + wasm_importtype_t *type = import_types.data[i]; + const wasm_name_t *import_name = wasm_importtype_name(type); + if (!ts_wasm_store__provide_builtin_import(self, import_name, &imports[i])) { + wasm_error->kind = TSWasmErrorKindInstantiate; + format( + &wasm_error->message, + "unexpected import in wasm stdlib: %.*s\n", + (int)import_name->size, import_name->data + ); + goto error; + } + } + + // Instantiate the stdlib module. + wasmtime_instance_t instance; + error = wasmtime_instance_new(context, stdlib_module, imports, import_types.size, &instance, &trap); + ts_free(imports); + imports = NULL; + if (error) { + wasmtime_error_message(error, &message); + wasm_error->kind = TSWasmErrorKindInstantiate; + format( + &wasm_error->message, + "failed to instantiate wasm stdlib module: %.*s", + (int)message.size, message.data + ); + goto error; + } + if (trap) { + wasm_trap_message(trap, &message); + wasm_error->kind = TSWasmErrorKindInstantiate; + format( + &wasm_error->message, + "trapped when instantiating wasm stdlib module: %.*s", + (int)message.size, message.data + ); + goto error; + } + wasm_importtype_vec_delete(&import_types); + + // Process the stdlib module's exports. + for (unsigned i = 0; i < stdlib_symbols_len; i++) { + self->stdlib_fn_indices[i] = UINT32_MAX; + } + wasmtime_module_exports(stdlib_module, &export_types); + for (unsigned i = 0; i < export_types.size; i++) { + wasm_exporttype_t *export_type = export_types.data[i]; + const wasm_name_t *name = wasm_exporttype_name(export_type); + + char *export_name; + size_t name_len; + wasmtime_extern_t export = {.kind = WASM_EXTERN_GLOBAL}; + bool exists = wasmtime_instance_export_nth(context, &instance, i, &export_name, &name_len, &export); + ts_assert(exists); + + if (export.kind == WASMTIME_EXTERN_GLOBAL) { + if (name_eq(name, "__stack_pointer")) { + self->stack_pointer_global = export.of.global; + } + } + + if (export.kind == WASMTIME_EXTERN_FUNC) { + if (ts_wasm_store__call_module_initializer(self, name, &export, &trap)) { + if (trap) { + wasm_trap_message(trap, &message); + wasm_error->kind = TSWasmErrorKindInstantiate; + format( + &wasm_error->message, + "trap when calling stdlib relocation function: %.*s\n", + (int)message.size, message.data + ); + goto error; + } + continue; + } + + if (name_eq(name, "reset_heap")) { + self->builtin_fn_indices.reset_heap = export.of.func.__private; + continue; + } + + for (unsigned j = 0; j < stdlib_symbols_len; j++) { + if (name_eq(name, STDLIB_SYMBOLS[j])) { + self->stdlib_fn_indices[j] = export.of.func.__private; + break; + } + } + } + } + + if (self->builtin_fn_indices.reset_heap == UINT32_MAX) { + wasm_error->kind = TSWasmErrorKindInstantiate; + format( + &wasm_error->message, + "missing malloc reset function in wasm stdlib" + ); + goto error; + } + + for (unsigned i = 0; i < stdlib_symbols_len; i++) { + if (self->stdlib_fn_indices[i] == UINT32_MAX) { + wasm_error->kind = TSWasmErrorKindInstantiate; + format( + &wasm_error->message, + "missing exported symbol in wasm stdlib: %s", + STDLIB_SYMBOLS[i] + ); + goto error; + } + } + + wasm_exporttype_vec_delete(&export_types); + wasmtime_module_delete(stdlib_module); + + // Add all of the lexer callback functions to the function table. Store their function table + // indices on the in-memory lexer. + uint64_t table_index; + error = wasmtime_table_grow(context, &function_table, lexer_definitions_len, &initializer, &table_index); + if (error) { + wasmtime_error_message(error, &message); + wasm_error->kind = TSWasmErrorKindAllocate; + format( + &wasm_error->message, + "failed to grow wasm table to initial size: %.*s", + (int)message.size, message.data + ); + goto error; + } + for (unsigned i = 0; i < lexer_definitions_len; i++) { + FunctionDefinition *definition = &lexer_definitions[i]; + wasmtime_func_t func = {function_table.store_id, *definition->storage_location}; + wasmtime_val_t func_val = {.kind = WASMTIME_FUNCREF, .of.funcref = func}; + error = wasmtime_table_set(context, &function_table, table_index, &func_val); + ts_assert(!error); + *(int32_t *)(definition->storage_location) = table_index; + table_index++; + } + + self->current_function_table_offset = table_index; + self->lexer_address = initial_memory_pages * MEMORY_PAGE_SIZE; + self->current_memory_offset = self->lexer_address + sizeof(LexerInWasmMemory); + + // Grow the memory enough to hold the builtin lexer and serialization buffer. + uint32_t new_pages_needed = (self->current_memory_offset - self->lexer_address - 1) / MEMORY_PAGE_SIZE + 1; + uint64_t prev_memory_size; + wasmtime_memory_grow(context, &memory, new_pages_needed, &prev_memory_size); + + uint8_t *memory_data = wasmtime_memory_data(context, &memory); + memcpy(&memory_data[self->lexer_address], &lexer, sizeof(lexer)); + return self; + +error: + ts_free(self); + if (stdlib_module) wasmtime_module_delete(stdlib_module); + if (store) wasmtime_store_delete(store); + if (import_types.size) wasm_importtype_vec_delete(&import_types); + if (memory_type) wasm_memorytype_delete(memory_type); + if (table_type) wasm_tabletype_delete(table_type); + if (trap) wasm_trap_delete(trap); + if (error) wasmtime_error_delete(error); + if (message.size) wasm_byte_vec_delete(&message); + if (export_types.size) wasm_exporttype_vec_delete(&export_types); + if (imports) ts_free(imports); + return NULL; +} + +void ts_wasm_store_delete(TSWasmStore *self) { + if (!self) return; + ts_free(self->stdlib_fn_indices); + wasm_globaltype_delete(self->const_i32_type); + wasmtime_store_delete(self->store); + wasm_engine_delete(self->engine); + for (unsigned i = 0; i < self->language_instances.size; i++) { + LanguageWasmInstance *instance = array_get(&self->language_instances, i); + language_id_delete(instance->language_id); + } + array_delete(&self->language_instances); + ts_free(self); +} + +size_t ts_wasm_store_language_count(const TSWasmStore *self) { + size_t result = 0; + for (unsigned i = 0; i < self->language_instances.size; i++) { + const WasmLanguageId *id = array_get(&self->language_instances, i)->language_id; + if (!id->is_language_deleted) { + result++; + } + } + return result; +} + +static uint32_t ts_wasm_store__heap_address(TSWasmStore *self) { + return self->current_memory_offset + TREE_SITTER_SERIALIZATION_BUFFER_SIZE; +} + +static uint32_t ts_wasm_store__serialization_buffer_address(TSWasmStore *self) { + return self->current_memory_offset; +} + +static bool ts_wasm_store__instantiate( + TSWasmStore *self, + wasmtime_module_t *module, + const char *language_name, + const WasmDylinkInfo *dylink_info, + wasmtime_instance_t *result, + int32_t *language_address, + char **error_message +) { + wasmtime_error_t *error = NULL; + wasm_trap_t *trap = NULL; + wasm_message_t message = WASM_EMPTY_VEC; + char *language_function_name = NULL; + wasmtime_extern_t *imports = NULL; + wasmtime_context_t *context = wasmtime_store_context(self->store); + + // Grow the function table to make room for the new functions. + wasmtime_val_t initializer = {.kind = WASMTIME_FUNCREF}; + uint64_t prev_table_size; + error = wasmtime_table_grow(context, &self->function_table, dylink_info->table_size, &initializer, &prev_table_size); + if (error) { + format(error_message, "invalid function table size %u", dylink_info->table_size); + goto error; + } + + // Grow the memory to make room for the new data. + uint32_t needed_memory_size = ts_wasm_store__heap_address(self) + dylink_info->memory_size; + uint32_t current_memory_size = wasmtime_memory_data_size(context, &self->memory); + if (needed_memory_size > current_memory_size) { + uint32_t pages_to_grow = ( + needed_memory_size - current_memory_size + MEMORY_PAGE_SIZE - 1) / + MEMORY_PAGE_SIZE; + uint64_t prev_memory_size; + error = wasmtime_memory_grow(context, &self->memory, pages_to_grow, &prev_memory_size); + if (error) { + format(error_message, "invalid memory size %u", dylink_info->memory_size); + goto error; + } + } + + // Construct the language function name as string. + format(&language_function_name, "tree_sitter_%s", language_name); + + const uint64_t store_id = self->function_table.store_id; + + // Build the imports list for the module. + wasm_importtype_vec_t import_types = WASM_EMPTY_VEC; + wasmtime_module_imports(module, &import_types); + imports = ts_calloc(import_types.size, sizeof(wasmtime_extern_t)); + + for (unsigned i = 0; i < import_types.size; i++) { + const wasm_importtype_t *import_type = import_types.data[i]; + const wasm_name_t *import_name = wasm_importtype_name(import_type); + if (import_name->size == 0) { + format(error_message, "empty import name"); + goto error; + } + + if (ts_wasm_store__provide_builtin_import(self, import_name, &imports[i])) { + continue; + } + + bool defined_in_stdlib = false; + for (unsigned j = 0; j < array_len(STDLIB_SYMBOLS); j++) { + if (name_eq(import_name, STDLIB_SYMBOLS[j])) { + uint16_t address = self->stdlib_fn_indices[j]; + imports[i] = (wasmtime_extern_t) {.kind = WASMTIME_EXTERN_FUNC, .of.func = {store_id, address}}; + defined_in_stdlib = true; + break; + } + } + + if (!defined_in_stdlib) { + format( + error_message, + "invalid import '%.*s'\n", + (int)import_name->size, import_name->data + ); + goto error; + } + } + + wasmtime_instance_t instance; + error = wasmtime_instance_new(context, module, imports, import_types.size, &instance, &trap); + wasm_importtype_vec_delete(&import_types); + ts_free(imports); + imports = NULL; + if (error) { + wasmtime_error_message(error, &message); + format( + error_message, + "error instantiating wasm module: %.*s\n", + (int)message.size, message.data + ); + goto error; + } + if (trap) { + wasm_trap_message(trap, &message); + format( + error_message, + "trap when instantiating wasm module: %.*s\n", + (int)message.size, message.data + ); + goto error; + } + + self->current_memory_offset += dylink_info->memory_size; + self->current_function_table_offset += dylink_info->table_size; + + // Process the module's exports. + bool found_language = false; + wasmtime_extern_t language_extern; + wasm_exporttype_vec_t export_types = WASM_EMPTY_VEC; + wasmtime_module_exports(module, &export_types); + for (unsigned i = 0; i < export_types.size; i++) { + wasm_exporttype_t *export_type = export_types.data[i]; + const wasm_name_t *name = wasm_exporttype_name(export_type); + + size_t name_len; + char *export_name; + wasmtime_extern_t export = {.kind = WASM_EXTERN_GLOBAL}; + bool exists = wasmtime_instance_export_nth(context, &instance, i, &export_name, &name_len, &export); + ts_assert(exists); + + // If the module exports an initialization or data-relocation function, call it. + if (ts_wasm_store__call_module_initializer(self, name, &export, &trap)) { + if (trap) { + wasm_trap_message(trap, &message); + format( + error_message, + "trap when calling data relocation function: %.*s\n", + (int)message.size, message.data + ); + goto error; + } + } + + // Find the main language function for the module. + else if (name_eq(name, language_function_name)) { + language_extern = export; + found_language = true; + } + } + wasm_exporttype_vec_delete(&export_types); + + if (!found_language) { + format( + error_message, + "module did not contain language function: %s", + language_function_name + ); + goto error; + } + + // Invoke the language function to get the static address of the language object. + wasmtime_func_t language_func = language_extern.of.func; + wasmtime_val_t language_address_val; + error = wasmtime_func_call(context, &language_func, NULL, 0, &language_address_val, 1, &trap); + ts_assert(!error); + if (trap) { + wasm_trap_message(trap, &message); + format( + error_message, + "trapped when calling language function: %s: %.*s\n", + language_function_name, (int)message.size, message.data + ); + goto error; + } + + if (language_address_val.kind != WASMTIME_I32) { + format( + error_message, + "language function did not return an integer: %s\n", + language_function_name + ); + goto error; + } + + ts_free(language_function_name); + *result = instance; + *language_address = language_address_val.of.i32; + return true; + +error: + if (language_function_name) ts_free(language_function_name); + if (message.size) wasm_byte_vec_delete(&message); + if (error) wasmtime_error_delete(error); + if (trap) wasm_trap_delete(trap); + if (imports) ts_free(imports); + return false; +} + +static bool ts_wasm_store__sentinel_lex_fn(TSLexer *_lexer, TSStateId state) { + return false; +} + +const TSLanguage *ts_wasm_store_load_language( + TSWasmStore *self, + const char *language_name, + const char *wasm, + uint32_t wasm_len, + TSWasmError *wasm_error +) { + WasmDylinkInfo dylink_info; + wasmtime_module_t *module = NULL; + wasmtime_error_t *error = NULL; + wasm_error->kind = TSWasmErrorKindNone; + + if (!wasm_dylink_info__parse((const unsigned char *)wasm, wasm_len, &dylink_info)) { + wasm_error->kind = TSWasmErrorKindParse; + format(&wasm_error->message, "failed to parse dylink section of wasm module"); + goto error; + } + + // Compile the wasm code. + error = wasmtime_module_new(self->engine, (const uint8_t *)wasm, wasm_len, &module); + if (error) { + wasm_message_t message; + wasmtime_error_message(error, &message); + wasm_error->kind = TSWasmErrorKindCompile; + format(&wasm_error->message, "error compiling wasm module: %.*s", (int)message.size, message.data); + wasm_byte_vec_delete(&message); + goto error; + } + + // Instantiate the module in this store. + wasmtime_instance_t instance; + int32_t language_address; + if (!ts_wasm_store__instantiate( + self, + module, + language_name, + &dylink_info, + &instance, + &language_address, + &wasm_error->message + )) { + wasm_error->kind = TSWasmErrorKindInstantiate; + goto error; + } + + // Copy all of the static data out of the language object in wasm memory, + // constructing a native language object. + LanguageInWasmMemory wasm_language; + wasmtime_context_t *context = wasmtime_store_context(self->store); + const uint8_t *memory = wasmtime_memory_data(context, &self->memory); + memcpy(&wasm_language, &memory[language_address], sizeof(LanguageInWasmMemory)); + + bool has_supertypes = + wasm_language.abi_version > LANGUAGE_VERSION_WITH_RESERVED_WORDS && + wasm_language.supertype_count > 0; + + int32_t addresses[] = { + wasm_language.parse_table, + wasm_language.small_parse_table, + wasm_language.small_parse_table_map, + wasm_language.parse_actions, + wasm_language.symbol_names, + wasm_language.field_names, + wasm_language.field_map_slices, + wasm_language.field_map_entries, + wasm_language.symbol_metadata, + wasm_language.public_symbol_map, + wasm_language.alias_map, + wasm_language.alias_sequences, + wasm_language.lex_modes, + wasm_language.lex_fn, + wasm_language.keyword_lex_fn, + wasm_language.primary_state_ids, + wasm_language.name, + wasm_language.reserved_words, + has_supertypes ? wasm_language.supertype_symbols : 0, + has_supertypes ? wasm_language.supertype_map_entries : 0, + has_supertypes ? wasm_language.supertype_map_slices : 0, + wasm_language.external_token_count > 0 ? wasm_language.external_scanner.states : 0, + wasm_language.external_token_count > 0 ? wasm_language.external_scanner.symbol_map : 0, + wasm_language.external_token_count > 0 ? wasm_language.external_scanner.create : 0, + wasm_language.external_token_count > 0 ? wasm_language.external_scanner.destroy : 0, + wasm_language.external_token_count > 0 ? wasm_language.external_scanner.scan : 0, + wasm_language.external_token_count > 0 ? wasm_language.external_scanner.serialize : 0, + wasm_language.external_token_count > 0 ? wasm_language.external_scanner.deserialize : 0, + language_address, + self->current_memory_offset, + }; + uint32_t address_count = array_len(addresses); + + TSLanguage *language = ts_calloc(1, sizeof(TSLanguage)); + StringData symbol_name_buffer = array_new(); + StringData field_name_buffer = array_new(); + + *language = (TSLanguage) { + .abi_version = wasm_language.abi_version, + .symbol_count = wasm_language.symbol_count, + .alias_count = wasm_language.alias_count, + .token_count = wasm_language.token_count, + .external_token_count = wasm_language.external_token_count, + .state_count = wasm_language.state_count, + .large_state_count = wasm_language.large_state_count, + .production_id_count = wasm_language.production_id_count, + .field_count = wasm_language.field_count, + .supertype_count = wasm_language.supertype_count, + .max_alias_sequence_length = wasm_language.max_alias_sequence_length, + .keyword_capture_token = wasm_language.keyword_capture_token, + .metadata = wasm_language.metadata, + .parse_table = copy( + &memory[wasm_language.parse_table], + wasm_language.large_state_count * wasm_language.symbol_count * sizeof(uint16_t) + ), + .parse_actions = copy_unsized_static_array( + memory, + wasm_language.parse_actions, + addresses, + address_count + ), + .symbol_names = copy_strings( + memory, + wasm_language.symbol_names, + wasm_language.symbol_count + wasm_language.alias_count, + &symbol_name_buffer + ), + .symbol_metadata = copy( + &memory[wasm_language.symbol_metadata], + (wasm_language.symbol_count + wasm_language.alias_count) * sizeof(TSSymbolMetadata) + ), + .public_symbol_map = copy( + &memory[wasm_language.public_symbol_map], + (wasm_language.symbol_count + wasm_language.alias_count) * sizeof(TSSymbol) + ), + .lex_modes = copy( + &memory[wasm_language.lex_modes], + wasm_language.state_count * sizeof(TSLexerMode) + ), + }; + + if (language->field_count > 0 && language->production_id_count > 0) { + language->field_map_slices = copy( + &memory[wasm_language.field_map_slices], + wasm_language.production_id_count * sizeof(TSMapSlice) + ); + + // Determine the number of field map entries by finding the greatest index + // in any of the slices. + uint32_t field_map_entry_count = 0; + for (uint32_t i = 0; i < wasm_language.production_id_count; i++) { + TSMapSlice slice = language->field_map_slices[i]; + uint32_t slice_end = slice.index + slice.length; + if (slice_end > field_map_entry_count) { + field_map_entry_count = slice_end; + } + } + + language->field_map_entries = copy( + &memory[wasm_language.field_map_entries], + field_map_entry_count * sizeof(TSFieldMapEntry) + ); + language->field_names = copy_strings( + memory, + wasm_language.field_names, + wasm_language.field_count + 1, + &field_name_buffer + ); + } + + if (has_supertypes) { + language->supertype_symbols = copy( + &memory[wasm_language.supertype_symbols], + wasm_language.supertype_count * sizeof(TSSymbol) + ); + + // Determine the number of supertype map slices by finding the greatest + // supertype ID. + int largest_supertype = 0; + for (unsigned i = 0; i < language->supertype_count; i++) { + TSSymbol supertype = language->supertype_symbols[i]; + if (supertype > largest_supertype) { + largest_supertype = supertype; + } + } + + language->supertype_map_slices = copy( + &memory[wasm_language.supertype_map_slices], + (largest_supertype + 1) * sizeof(TSMapSlice) + ); + + TSSymbol last_supertype = language->supertype_symbols[language->supertype_count - 1]; + TSMapSlice last_slice = language->supertype_map_slices[last_supertype]; + uint32_t supertype_map_entry_count = last_slice.index + last_slice.length; + + language->supertype_map_entries = copy( + &memory[wasm_language.supertype_map_entries], + supertype_map_entry_count * sizeof(char *) + ); + } + + if (language->max_alias_sequence_length > 0 && language->production_id_count > 0) { + // The alias map contains symbols, alias counts, and aliases, terminated by a null symbol. + int32_t alias_map_size = 0; + for (;;) { + TSSymbol symbol; + memcpy(&symbol, &memory[wasm_language.alias_map + alias_map_size], sizeof(symbol)); + alias_map_size += sizeof(TSSymbol); + if (symbol == 0) break; + uint16_t value_count; + memcpy(&value_count, &memory[wasm_language.alias_map + alias_map_size], sizeof(value_count)); + alias_map_size += sizeof(uint16_t); + alias_map_size += value_count * sizeof(TSSymbol); + } + language->alias_map = copy( + &memory[wasm_language.alias_map], + alias_map_size + ); + language->alias_sequences = copy( + &memory[wasm_language.alias_sequences], + wasm_language.production_id_count * wasm_language.max_alias_sequence_length * sizeof(TSSymbol) + ); + } + + if (language->state_count > language->large_state_count) { + uint32_t small_state_count = wasm_language.state_count - wasm_language.large_state_count; + language->small_parse_table_map = copy( + &memory[wasm_language.small_parse_table_map], + small_state_count * sizeof(uint32_t) + ); + language->small_parse_table = copy_unsized_static_array( + memory, + wasm_language.small_parse_table, + addresses, + address_count + ); + } + + if (language->abi_version >= LANGUAGE_VERSION_WITH_PRIMARY_STATES) { + language->primary_state_ids = copy( + &memory[wasm_language.primary_state_ids], + wasm_language.state_count * sizeof(TSStateId) + ); + } + + if (language->abi_version >= LANGUAGE_VERSION_WITH_RESERVED_WORDS) { + language->name = copy_string(memory, wasm_language.name); + language->reserved_words = copy( + &memory[wasm_language.reserved_words], + wasm_language.max_reserved_word_set_size * sizeof(TSSymbol) + ); + language->max_reserved_word_set_size = wasm_language.max_reserved_word_set_size; + } + + if (language->external_token_count > 0) { + language->external_scanner.symbol_map = copy( + &memory[wasm_language.external_scanner.symbol_map], + wasm_language.external_token_count * sizeof(TSSymbol) + ); + language->external_scanner.states = (void *)(uintptr_t)wasm_language.external_scanner.states; + } + + unsigned name_len = strlen(language_name); + char *name = ts_malloc(name_len + 1); + memcpy(name, language_name, name_len); + name[name_len] = '\0'; + + LanguageWasmModule *language_module = ts_malloc(sizeof(LanguageWasmModule)); + *language_module = (LanguageWasmModule) { + .language_id = language_id_new(), + .module = module, + .name = name, + .symbol_name_buffer = symbol_name_buffer.contents, + .field_name_buffer = field_name_buffer.contents, + .dylink_info = dylink_info, + .ref_count = 1, + }; + + // The lex functions are not used for wasm languages. Use those two fields + // to mark this language as WASM-based and to store the language's + // WASM-specific data. + language->lex_fn = ts_wasm_store__sentinel_lex_fn; + language->keyword_lex_fn = (bool (*)(TSLexer *, TSStateId))language_module; + + // Clear out any instances of languages that have been deleted. + for (unsigned i = 0; i < self->language_instances.size; i++) { + WasmLanguageId *id = array_get(&self->language_instances, i)->language_id; + if (id->is_language_deleted) { + language_id_delete(id); + array_erase(&self->language_instances, i); + i--; + } + } + + // Store this store's instance of this language module. + array_push(&self->language_instances, ((LanguageWasmInstance) { + .language_id = language_id_clone(language_module->language_id), + .instance = instance, + .external_states_address = wasm_language.external_scanner.states, + .lex_main_fn_index = wasm_language.lex_fn, + .lex_keyword_fn_index = wasm_language.keyword_lex_fn, + .scanner_create_fn_index = wasm_language.external_scanner.create, + .scanner_destroy_fn_index = wasm_language.external_scanner.destroy, + .scanner_serialize_fn_index = wasm_language.external_scanner.serialize, + .scanner_deserialize_fn_index = wasm_language.external_scanner.deserialize, + .scanner_scan_fn_index = wasm_language.external_scanner.scan, + })); + + return language; + +error: + if (module) wasmtime_module_delete(module); + return NULL; +} + +bool ts_wasm_store_add_language( + TSWasmStore *self, + const TSLanguage *language, + uint32_t *index +) { + wasmtime_context_t *context = wasmtime_store_context(self->store); + const LanguageWasmModule *language_module = (void *)language->keyword_lex_fn; + + // Search for this store's instance of the language module. Also clear out any + // instances of languages that have been deleted. + bool exists = false; + for (unsigned i = 0; i < self->language_instances.size; i++) { + WasmLanguageId *id = array_get(&self->language_instances, i)->language_id; + if (id->is_language_deleted) { + language_id_delete(id); + array_erase(&self->language_instances, i); + i--; + } else if (id == language_module->language_id) { + exists = true; + *index = i; + } + } + + // If the language module has not been instantiated in this store, then add + // it to this store. + if (!exists) { + *index = self->language_instances.size; + char *message; + wasmtime_instance_t instance; + int32_t language_address; + if (!ts_wasm_store__instantiate( + self, + language_module->module, + language_module->name, + &language_module->dylink_info, + &instance, + &language_address, + &message + )) { + ts_free(message); + return false; + } + + LanguageInWasmMemory wasm_language; + const uint8_t *memory = wasmtime_memory_data(context, &self->memory); + memcpy(&wasm_language, &memory[language_address], sizeof(LanguageInWasmMemory)); + array_push(&self->language_instances, ((LanguageWasmInstance) { + .language_id = language_id_clone(language_module->language_id), + .instance = instance, + .external_states_address = wasm_language.external_scanner.states, + .lex_main_fn_index = wasm_language.lex_fn, + .lex_keyword_fn_index = wasm_language.keyword_lex_fn, + .scanner_create_fn_index = wasm_language.external_scanner.create, + .scanner_destroy_fn_index = wasm_language.external_scanner.destroy, + .scanner_serialize_fn_index = wasm_language.external_scanner.serialize, + .scanner_deserialize_fn_index = wasm_language.external_scanner.deserialize, + .scanner_scan_fn_index = wasm_language.external_scanner.scan, + })); + } + + return true; +} + +void ts_wasm_store_reset_heap(TSWasmStore *self) { + wasmtime_context_t *context = wasmtime_store_context(self->store); + wasmtime_func_t func = { + self->function_table.store_id, + self->builtin_fn_indices.reset_heap + }; + wasm_trap_t *trap = NULL; + wasmtime_val_t args[1] = { + {.of.i32 = ts_wasm_store__heap_address(self), .kind = WASMTIME_I32}, + }; + + wasmtime_error_t *error = wasmtime_func_call(context, &func, args, 1, NULL, 0, &trap); + ts_assert(!error); + ts_assert(!trap); +} + +bool ts_wasm_store_start(TSWasmStore *self, TSLexer *lexer, const TSLanguage *language) { + uint32_t instance_index; + if (!ts_wasm_store_add_language(self, language, &instance_index)) return false; + self->current_lexer = lexer; + self->current_instance = array_get(&self->language_instances, instance_index); + self->has_error = false; + ts_wasm_store_reset_heap(self); + return true; +} + +void ts_wasm_store_reset(TSWasmStore *self) { + self->current_lexer = NULL; + self->current_instance = NULL; + self->has_error = false; + ts_wasm_store_reset_heap(self); +} + +static void ts_wasm_store__call( + TSWasmStore *self, + int32_t function_index, + wasmtime_val_raw_t *args_and_results, + size_t args_and_results_len +) { + wasmtime_context_t *context = wasmtime_store_context(self->store); + wasmtime_val_t value; + bool succeeded = wasmtime_table_get(context, &self->function_table, function_index, &value); + ts_assert(succeeded); + ts_assert(value.kind == WASMTIME_FUNCREF); + wasmtime_func_t func = value.of.funcref; + + wasm_trap_t *trap = NULL; + wasmtime_error_t *error = wasmtime_func_call_unchecked(context, &func, args_and_results, args_and_results_len, &trap); + if (error) { + // wasm_message_t message; + // wasmtime_error_message(error, &message); + // fprintf( + // stderr, + // "error in wasm module: %.*s\n", + // (int)message.size, message.data + // ); + wasmtime_error_delete(error); + self->has_error = true; + } else if (trap) { + // wasm_message_t message; + // wasm_trap_message(trap, &message); + // fprintf( + // stderr, + // "trap in wasm module: %.*s\n", + // (int)message.size, message.data + // ); + wasm_trap_delete(trap); + self->has_error = true; + } +} + +// The data fields of TSLexer, without the function pointers. +// +// This portion of the struct needs to be copied in and out +// of wasm memory before and after calling a scan function. +typedef struct { + int32_t lookahead; + TSSymbol result_symbol; +} TSLexerDataPrefix; + +static bool ts_wasm_store__call_lex_function(TSWasmStore *self, unsigned function_index, TSStateId state) { + wasmtime_context_t *context = wasmtime_store_context(self->store); + uint8_t *memory_data = wasmtime_memory_data(context, &self->memory); + memcpy( + &memory_data[self->lexer_address], + self->current_lexer, + sizeof(TSLexerDataPrefix) + ); + + wasmtime_val_raw_t args[2] = { + {.i32 = self->lexer_address}, + {.i32 = state}, + }; + ts_wasm_store__call(self, function_index, args, 2); + if (self->has_error) return false; + bool result = args[0].i32; + + memcpy( + self->current_lexer, + &memory_data[self->lexer_address], + sizeof(TSLexerDataPrefix) + ); + return result; +} + +bool ts_wasm_store_call_lex_main(TSWasmStore *self, TSStateId state) { + return ts_wasm_store__call_lex_function( + self, + self->current_instance->lex_main_fn_index, + state + ); +} + +bool ts_wasm_store_call_lex_keyword(TSWasmStore *self, TSStateId state) { + return ts_wasm_store__call_lex_function( + self, + self->current_instance->lex_keyword_fn_index, + state + ); +} + +uint32_t ts_wasm_store_call_scanner_create(TSWasmStore *self) { + wasmtime_val_raw_t args[1] = {{.i32 = 0}}; + ts_wasm_store__call(self, self->current_instance->scanner_create_fn_index, args, 1); + if (self->has_error) return 0; + return args[0].i32; +} + +void ts_wasm_store_call_scanner_destroy(TSWasmStore *self, uint32_t scanner_address) { + if (self->current_instance) { + wasmtime_val_raw_t args[1] = {{.i32 = scanner_address}}; + ts_wasm_store__call(self, self->current_instance->scanner_destroy_fn_index, args, 1); + } +} + +bool ts_wasm_store_call_scanner_scan( + TSWasmStore *self, + uint32_t scanner_address, + uint32_t valid_tokens_ix +) { + wasmtime_context_t *context = wasmtime_store_context(self->store); + uint8_t *memory_data = wasmtime_memory_data(context, &self->memory); + + memcpy( + &memory_data[self->lexer_address], + self->current_lexer, + sizeof(TSLexerDataPrefix) + ); + + uint32_t valid_tokens_address = + self->current_instance->external_states_address + + (valid_tokens_ix * sizeof(bool)); + wasmtime_val_raw_t args[3] = { + {.i32 = scanner_address}, + {.i32 = self->lexer_address}, + {.i32 = valid_tokens_address} + }; + ts_wasm_store__call(self, self->current_instance->scanner_scan_fn_index, args, 3); + if (self->has_error) return false; + + memcpy( + self->current_lexer, + &memory_data[self->lexer_address], + sizeof(TSLexerDataPrefix) + ); + return args[0].i32; +} + +uint32_t ts_wasm_store_call_scanner_serialize( + TSWasmStore *self, + uint32_t scanner_address, + char *buffer +) { + wasmtime_context_t *context = wasmtime_store_context(self->store); + uint8_t *memory_data = wasmtime_memory_data(context, &self->memory); + uint32_t serialization_buffer_address = ts_wasm_store__serialization_buffer_address(self); + + wasmtime_val_raw_t args[2] = { + {.i32 = scanner_address}, + {.i32 = serialization_buffer_address}, + }; + ts_wasm_store__call(self, self->current_instance->scanner_serialize_fn_index, args, 2); + if (self->has_error) return 0; + + uint32_t length = args[0].i32; + if (length > TREE_SITTER_SERIALIZATION_BUFFER_SIZE) { + self->has_error = true; + return 0; + } + + if (length > 0) { + memcpy( + ((Lexer *)self->current_lexer)->debug_buffer, + &memory_data[serialization_buffer_address], + length + ); + } + return length; +} + +void ts_wasm_store_call_scanner_deserialize( + TSWasmStore *self, + uint32_t scanner_address, + const char *buffer, + unsigned length +) { + wasmtime_context_t *context = wasmtime_store_context(self->store); + uint8_t *memory_data = wasmtime_memory_data(context, &self->memory); + uint32_t serialization_buffer_address = ts_wasm_store__serialization_buffer_address(self); + + if (length > 0) { + memcpy( + &memory_data[serialization_buffer_address], + buffer, + length + ); + } + + wasmtime_val_raw_t args[3] = { + {.i32 = scanner_address}, + {.i32 = serialization_buffer_address}, + {.i32 = length}, + }; + ts_wasm_store__call(self, self->current_instance->scanner_deserialize_fn_index, args, 3); +} + +bool ts_wasm_store_has_error(const TSWasmStore *self) { + return self->has_error; +} + +bool ts_language_is_wasm(const TSLanguage *self) { + return self->lex_fn == ts_wasm_store__sentinel_lex_fn; +} + +static inline LanguageWasmModule *ts_language__wasm_module(const TSLanguage *self) { + return (LanguageWasmModule *)self->keyword_lex_fn; +} + +void ts_wasm_language_retain(const TSLanguage *self) { + LanguageWasmModule *module = ts_language__wasm_module(self); + ts_assert(module->ref_count > 0); + atomic_inc(&module->ref_count); +} + +void ts_wasm_language_release(const TSLanguage *self) { + LanguageWasmModule *module = ts_language__wasm_module(self); + ts_assert(module->ref_count > 0); + if (atomic_dec(&module->ref_count) == 0) { + // Update the language id to reflect that the language is deleted. This allows any wasm stores + // that hold wasm instances for this language to delete those instances. + atomic_inc(&module->language_id->is_language_deleted); + language_id_delete(module->language_id); + + ts_free((void *)module->field_name_buffer); + ts_free((void *)module->symbol_name_buffer); + ts_free((void *)module->name); + wasmtime_module_delete(module->module); + ts_free(module); + + ts_free((void *)self->alias_map); + ts_free((void *)self->alias_sequences); + ts_free((void *)self->external_scanner.symbol_map); + ts_free((void *)self->field_map_entries); + ts_free((void *)self->field_map_slices); + ts_free((void *)self->supertype_symbols); + ts_free((void *)self->supertype_map_entries); + ts_free((void *)self->supertype_map_slices); + ts_free((void *)self->field_names); + ts_free((void *)self->lex_modes); + ts_free((void *)self->name); + ts_free((void *)self->reserved_words); + ts_free((void *)self->parse_actions); + ts_free((void *)self->parse_table); + ts_free((void *)self->primary_state_ids); + ts_free((void *)self->public_symbol_map); + ts_free((void *)self->small_parse_table); + ts_free((void *)self->small_parse_table_map); + ts_free((void *)self->symbol_metadata); + ts_free((void *)self->symbol_names); + ts_free((void *)self); + } +} + +#ifdef _MSC_VER +#pragma warning(pop) +#elif defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif + +#else + +// If the WASM feature is not enabled, define dummy versions of all of the +// wasm-related functions. + +void ts_wasm_store_delete(TSWasmStore *self) { + (void)self; +} + +bool ts_wasm_store_start( + TSWasmStore *self, + TSLexer *lexer, + const TSLanguage *language +) { + (void)self; + (void)lexer; + (void)language; + return false; +} + +void ts_wasm_store_reset(TSWasmStore *self) { + (void)self; +} + +bool ts_wasm_store_call_lex_main(TSWasmStore *self, TSStateId state) { + (void)self; + (void)state; + return false; +} + +bool ts_wasm_store_call_lex_keyword(TSWasmStore *self, TSStateId state) { + (void)self; + (void)state; + return false; +} + +uint32_t ts_wasm_store_call_scanner_create(TSWasmStore *self) { + (void)self; + return 0; +} + +void ts_wasm_store_call_scanner_destroy( + TSWasmStore *self, + uint32_t scanner_address +) { + (void)self; + (void)scanner_address; +} + +bool ts_wasm_store_call_scanner_scan( + TSWasmStore *self, + uint32_t scanner_address, + uint32_t valid_tokens_ix +) { + (void)self; + (void)scanner_address; + (void)valid_tokens_ix; + return false; +} + +uint32_t ts_wasm_store_call_scanner_serialize( + TSWasmStore *self, + uint32_t scanner_address, + char *buffer +) { + (void)self; + (void)scanner_address; + (void)buffer; + return 0; +} + +void ts_wasm_store_call_scanner_deserialize( + TSWasmStore *self, + uint32_t scanner_address, + const char *buffer, + unsigned length +) { + (void)self; + (void)scanner_address; + (void)buffer; + (void)length; +} + +bool ts_wasm_store_has_error(const TSWasmStore *self) { + (void)self; + return false; +} + +bool ts_language_is_wasm(const TSLanguage *self) { + (void)self; + return false; +} + +void ts_wasm_language_retain(const TSLanguage *self) { + (void)self; +} + +void ts_wasm_language_release(const TSLanguage *self) { + (void)self; +} + +#endif diff --git a/Externals/crystaledit/editlib/tree-sitter/lib/src/wasm_store.h b/Externals/crystaledit/editlib/tree-sitter/lib/src/wasm_store.h new file mode 100644 index 00000000000..0fd17e0d173 --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/lib/src/wasm_store.h @@ -0,0 +1,31 @@ +#ifndef TREE_SITTER_WASM_H_ +#define TREE_SITTER_WASM_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tree_sitter/api.h" +#include "./parser.h" + +bool ts_wasm_store_start(TSWasmStore *self, TSLexer *lexer, const TSLanguage *language); +void ts_wasm_store_reset(TSWasmStore *self); +bool ts_wasm_store_has_error(const TSWasmStore *self); + +bool ts_wasm_store_call_lex_main(TSWasmStore *self, TSStateId state); +bool ts_wasm_store_call_lex_keyword(TSWasmStore *self, TSStateId state); + +uint32_t ts_wasm_store_call_scanner_create(TSWasmStore *self); +void ts_wasm_store_call_scanner_destroy(TSWasmStore *self, uint32_t scanner_address); +bool ts_wasm_store_call_scanner_scan(TSWasmStore *self, uint32_t scanner_address, uint32_t valid_tokens_ix); +uint32_t ts_wasm_store_call_scanner_serialize(TSWasmStore *self, uint32_t scanner_address, char *buffer); +void ts_wasm_store_call_scanner_deserialize(TSWasmStore *self, uint32_t scanner, const char *buffer, unsigned length); + +void ts_wasm_language_retain(const TSLanguage *self); +void ts_wasm_language_release(const TSLanguage *self); + +#ifdef __cplusplus +} +#endif + +#endif // TREE_SITTER_WASM_H_ diff --git a/Externals/crystaledit/editlib/tree-sitter/tree-sitter.vcxitems b/Externals/crystaledit/editlib/tree-sitter/tree-sitter.vcxitems new file mode 100644 index 00000000000..69e7c612f8a --- /dev/null +++ b/Externals/crystaledit/editlib/tree-sitter/tree-sitter.vcxitems @@ -0,0 +1,24 @@ + + + + $(MSBuildAllProjects);$(MSBuildThisFileFullPath) + true + {a3e7e04f-5d2b-4f3a-b725-8f5c4b1e9d3c} + + + + %(AdditionalIncludeDirectories);$(MSBuildThisFileDirectory)lib\include;$(MSBuildThisFileDirectory)lib\src + + + + + + + + NotUsing + + + + + + diff --git a/Externals/crystaledit/test/batchTests.cpp b/Externals/crystaledit/test/batchTests.cpp index 88c19f07e23..513764435cf 100644 --- a/Externals/crystaledit/test/batchTests.cpp +++ b/Externals/crystaledit/test/batchTests.cpp @@ -51,7 +51,7 @@ namespace test std::wstring msg = L"index: " + std::to_wstring(i); Assert::AreEqual( static_cast(expected[i].dwCookie), - CrystalLineParser::ParseLineBatch(data[i].dwCookie, data[i].pszChars, static_cast(tc::tcslen(data[i].pszChars)), pblocks.get(), nActualItems), msg.c_str()); + CrystalLineParser::ParseLineBatch(data[i].dwCookie, i, data[i].pszChars, static_cast(tc::tcslen(data[i].pszChars)), pblocks.get(), nActualItems, nullptr), msg.c_str()); Assert::AreEqual(static_cast(expected[i].nblocks), nActualItems, msg.c_str()); for (int j = 0; j < nActualItems; ++j) { diff --git a/Externals/crystaledit/test/htmlTests.cpp b/Externals/crystaledit/test/htmlTests.cpp index eb0587d6ac6..b35c1e19656 100644 --- a/Externals/crystaledit/test/htmlTests.cpp +++ b/Externals/crystaledit/test/htmlTests.cpp @@ -94,7 +94,7 @@ namespace test std::wstring msg = L"index: " + std::to_wstring(i) + L" dwCookie"; Assert::AreEqual( static_cast(expected[i].dwCookie), - CrystalLineParser::ParseLineHtml(data[i].dwCookie, data[i].pszChars, static_cast(tc::tcslen(data[i].pszChars)), pblocks.get(), nActualItems), msg.c_str()); + CrystalLineParser::ParseLineHtml(data[i].dwCookie, i, data[i].pszChars, static_cast(tc::tcslen(data[i].pszChars)), pblocks.get(), nActualItems, nullptr), msg.c_str()); msg = L"index: " + std::to_wstring(i) + L" nblocks"; Assert::AreEqual(static_cast(expected[i].nblocks), nActualItems, msg.c_str()); for (int j = 0; j < nActualItems; ++j) diff --git a/Externals/crystaledit/test/luaTests.cpp b/Externals/crystaledit/test/luaTests.cpp index e48b0731b6d..42ec84ed7ad 100644 --- a/Externals/crystaledit/test/luaTests.cpp +++ b/Externals/crystaledit/test/luaTests.cpp @@ -83,7 +83,7 @@ namespace test std::wstring msg = L"index: " + std::to_wstring(i); Assert::AreEqual( static_cast(expected[i].dwCookie), - CrystalLineParser::ParseLineLua(data[i].dwCookie, data[i].pszChars, static_cast(tc::tcslen(data[i].pszChars)), pblocks.get(), nActualItems), msg.c_str()); + CrystalLineParser::ParseLineLua(data[i].dwCookie, i, data[i].pszChars, static_cast(tc::tcslen(data[i].pszChars)), pblocks.get(), nActualItems, nullptr), msg.c_str()); Assert::AreEqual(static_cast(expected[i].nblocks), nActualItems, msg.c_str()); for (int j = 0; j < nActualItems; ++j) { diff --git a/Externals/versions.txt b/Externals/versions.txt index bb0a67ccf51..2bacfc47561 100644 --- a/Externals/versions.txt +++ b/Externals/versions.txt @@ -23,3 +23,4 @@ This file lists versions of the external components we are using. - md4c: 0.5.2 - darkmodelib: 0.21.2.0 - StackWalker: 7af4024 on Aug 4, 2025 (https://github.com/JochenKalmbach/StackWalker) +- tree-sitter: 0.25.10 (https://github.com/tree-sitter/tree-sitter) diff --git a/Installer/InnoSetup/WinMergeARM64.is6.iss b/Installer/InnoSetup/WinMergeARM64.is6.iss index e9f2a91ef9e..ce45e2908ad 100644 --- a/Installer/InnoSetup/WinMergeARM64.is6.iss +++ b/Installer/InnoSetup/WinMergeARM64.is6.iss @@ -434,6 +434,8 @@ Source: ..\..\Build\{#ARCH}\Release\WinMergeU.VisualElementsManifest.xml; DestDi Source: ..\..\Build\{#ARCH}\Release\LogoImages\*.png; DestDir: {app}\LogoImages; Flags: promptifolder; Components: Core ; Resources Source: ..\..\Build\{#ARCH}\Release\Resources\*.png; DestDir: {app}\Resources; Flags: promptifolder; Components: Core +Source: ..\..\Build\{#ARCH}\Release\TreeSitterGrammars\*.dll; DestDir: {app}\TreeSitterGrammars; Flags: ignoreversion recursesubdirs; Components: Core +Source: ..\..\Build\{#ARCH}\Release\TreeSitterGrammars\*.scm; DestDir: {app}\TreeSitterGrammars; Flags: ignoreversion recursesubdirs; Components: Core ; 32Bit Plugin Proxy Source: ..\..\Plugins\WinMerge32BitPluginProxy\Release\WinMerge32BitPluginProxy.exe; DestDir: {app}; Flags: promptifolder; Components: Core diff --git a/Installer/InnoSetup/WinMergeX64NonAdmin.iss b/Installer/InnoSetup/WinMergeX64NonAdmin.iss index e3411d1b2b8..d5fcab2a6a8 100644 --- a/Installer/InnoSetup/WinMergeX64NonAdmin.iss +++ b/Installer/InnoSetup/WinMergeX64NonAdmin.iss @@ -423,6 +423,8 @@ Source: ..\..\Build\{#ARCH}\Release\WinMergeU.VisualElementsManifest.xml; DestDi Source: ..\..\Build\{#ARCH}\Release\LogoImages\*.png; DestDir: {app}\LogoImages; Flags: promptifolder; Components: Core ; Resources Source: ..\..\Build\{#ARCH}\Release\Resources\*.png; DestDir: {app}\Resources; Flags: promptifolder; Components: Core +Source: ..\..\Build\{#ARCH}\Release\TreeSitterGrammars\*.dll; DestDir: {app}\TreeSitterGrammars; Flags: ignoreversion recursesubdirs; Components: Core +Source: ..\..\Build\{#ARCH}\Release\TreeSitterGrammars\*.scm; DestDir: {app}\TreeSitterGrammars; Flags: ignoreversion recursesubdirs; Components: Core ; 32Bit Plugin Proxy Source: ..\..\Plugins\WinMerge32BitPluginProxy\Release\WinMerge32BitPluginProxy.exe; DestDir: {app}; Flags: promptifolder; Components: Core diff --git a/Installer/WiX/Config.wxi b/Installer/WiX/Config.wxi index 7e1be4c2ac9..dc9b67c7b38 100644 --- a/Installer/WiX/Config.wxi +++ b/Installer/WiX/Config.wxi @@ -13,26 +13,28 @@ - - - - - - - - + + + + + + + + + - - - - - - - - + + + + + + + + + diff --git a/Installer/WiX/Fragments/Features.wxs b/Installer/WiX/Fragments/Features.wxs index 953efb3d39a..67cc15cb708 100644 --- a/Installer/WiX/Fragments/Features.wxs +++ b/Installer/WiX/Fragments/Features.wxs @@ -13,12 +13,15 @@ Display="expand" Level="1" Title="WinMerge"> - - - - + + + + + + diff --git a/Installer/WiX/WinMerge.wixproj b/Installer/WiX/WinMerge.wixproj index b72defb9f46..67e25ba4470 100644 --- a/Installer/WiX/WinMerge.wixproj +++ b/Installer/WiX/WinMerge.wixproj @@ -13,9 +13,15 @@ $(MSBuildExtensionsPath)\Microsoft\WiX\v3.x\Wix.targets ..\..\BuildTmp\WiX\$(Configuration)\$(Platform)\ - - Debug; - + + Debug; + + + $(MSBuildProjectDirectory)\..\..\Build\x64\$(Configuration)\TreeSitterGrammars + $(MSBuildProjectDirectory)\..\..\Build\x86\$(Configuration)\TreeSitterGrammars + $(MSBuildProjectDirectory)\..\..\Build\ARM64\$(Configuration)\TreeSitterGrammars + $(DefineConstants)TreeSitterGrammarsFiles=1; + C:\Program Files (x86)\WiX Toolset v3.10\bin\WixUIExtension.dll @@ -30,8 +36,18 @@ - - - - - \ No newline at end of file + + + + + + TreeSitterGrammarsFiles + TREESITTERGRAMMARSDIR + true + true + true + true + + + + diff --git a/Installer/WiX/WinMerge.wxs b/Installer/WiX/WinMerge.wxs index 0fc4d749c0e..3371a2eaa89 100644 --- a/Installer/WiX/WinMerge.wxs +++ b/Installer/WiX/WinMerge.wxs @@ -43,14 +43,15 @@ EmbedCab="yes" /> - - - - - - - - + + + + + + + + + diff --git a/Src/CompareEngines/FullQuickCompare.cpp b/Src/CompareEngines/FullQuickCompare.cpp index c97d919b1e1..40ff2998ced 100644 --- a/Src/CompareEngines/FullQuickCompare.cpp +++ b/Src/CompareEngines/FullQuickCompare.cpp @@ -17,6 +17,8 @@ #include "codepage_detect.h" #include "Logger.h" #include "TFile.h" +#include "coretools.h" +#include "../../../Externals/crystaledit/editlib/TreeSitterWrapper.h" namespace CompareEngines { @@ -199,6 +201,30 @@ void FullQuickCompare::CompareFiles(DIFFITEM& di) const dw.SetFilterList(m_pCtxt->m_pFilterList); dw.SetSubstitutionList(m_pCtxt->m_pSubstitutionList); dw.SetFilterCommentsSourceDef(Ext); + + std::unique_ptr treeSitterParsers[3] = { + { nullptr, DestroyTreeSitterParseContextForDiff }, + { nullptr, DestroyTreeSitterParseContextForDiff }, + { nullptr, DestroyTreeSitterParseContextForDiff } + }; + if (dw.GetOptions().m_filterCommentsLines) + { + DiffFileData* fileDiffs[3] = { &diffdata10, &diffdata12, &diffdata02 }; + for (int i = 0; i < 3; ++i) + { + const int targetIndex = (i == 0) ? 1 : (i == 1 ? 0 : 1); + std::vector lines; + for (int line = 0; line < fileDiffs[i]->m_inf[targetIndex].valid_lines; ++line) + { + const char* start = fileDiffs[i]->m_inf[targetIndex].linbuf[fileDiffs[i]->m_inf[targetIndex].linbuf_base + line]; + const char* end = fileDiffs[i]->m_inf[targetIndex].linbuf[fileDiffs[i]->m_inf[targetIndex].linbuf_base + line + 1]; + const size_t fullLen = static_cast(end - start); + lines.push_back(ucr::toTString(std::string(start, linelen(start, fullLen)))); + } + treeSitterParsers[i].reset(CreateTreeSitterParseContextForDiff(ucr::toTString(fileDiffs[i]->m_inf[targetIndex].name), lines)); + dw.SetFilterCommentsParseContext(treeSitterParsers[i].get(), i); + } + } dw.SetCreateDiffList(&diffList); dw.LoadWinMergeDiffsFromDiffUtilsScript3( script10, script12, script02, diff --git a/Src/CompareEngines/Wrap_DiffUtils.cpp b/Src/CompareEngines/Wrap_DiffUtils.cpp index 57a91e74d25..53e7187b16b 100644 --- a/Src/CompareEngines/Wrap_DiffUtils.cpp +++ b/Src/CompareEngines/Wrap_DiffUtils.cpp @@ -22,6 +22,7 @@ #include "xdiff_gnudiff_compat.h" #include "unicoder.h" #include "DiffFileData.h" +#include "../../../Externals/crystaledit/editlib/TreeSitterWrapper.h" namespace CompareEngines { @@ -127,6 +128,28 @@ int DiffUtils::CompareFiles(DiffFileData* diffData) m_pDiffWrapper->SetFilterCommentsSourceDef(Ext); + std::unique_ptr treeSitterParsers[2] = { + { nullptr, DestroyTreeSitterParseContextForDiff }, + { nullptr, DestroyTreeSitterParseContextForDiff } + }; + if (m_pDiffWrapper->GetOptions().m_filterCommentsLines) + { + for (int i = 0; i < 2; ++i) + { + std::vector lines; + for (int line = 0; line < diffData->m_inf[i].valid_lines; ++line) + { + const char* start = diffData->m_inf[i].linbuf[diffData->m_inf[i].linbuf_base + line]; + const char* end = diffData->m_inf[i].linbuf[diffData->m_inf[i].linbuf_base + line + 1]; + const size_t fullLen = static_cast(end - start); + String text = ucr::toTString(std::string(start, linelen(start, fullLen))); + lines.push_back(std::move(text)); + } + treeSitterParsers[i].reset(CreateTreeSitterParseContextForDiff(ucr::toTString(diffData->m_inf[i].name), lines)); + m_pDiffWrapper->SetFilterCommentsParseContext(treeSitterParsers[i].get(), i); + } + } + struct change *next = script; while (next != nullptr) diff --git a/Src/DiffWrapper.cpp b/Src/DiffWrapper.cpp index e5c02e161e5..a3b8546018d 100644 --- a/Src/DiffWrapper.cpp +++ b/Src/DiffWrapper.cpp @@ -39,6 +39,7 @@ #include "SubstitutionList.h" #include "codepage_detect.h" #include "cio.h" +#include "../Externals/crystaledit/editlib/TreeSitterWrapper.h" using Poco::Exception; @@ -230,7 +231,7 @@ static String convertToTString(const char* start, const char* end) } } -static unsigned GetLastLineCookie(unsigned dwCookie, int startLine, int endLine, const char **linbuf, CrystalLineParser::TextDefinition* enuType) +static unsigned GetLastLineCookie(unsigned dwCookie, int startLine, int endLine, const char **linbuf, CrystalLineParser::TextDefinition* enuType, void* parseContext) { if (!enuType) return dwCookie; @@ -239,12 +240,12 @@ static unsigned GetLastLineCookie(unsigned dwCookie, int startLine, int endLine, String text = convertToTString(linbuf[i], linbuf[i + 1]); int nActualItems = 0; std::vector blocks(text.length()); - dwCookie = enuType->ParseLineX(dwCookie, text.c_str(), static_cast(text.length()), blocks.data(), nActualItems); + dwCookie = enuType->ParseLineX(dwCookie, i, text.c_str(), static_cast(text.length()), blocks.data(), nActualItems, parseContext); } return dwCookie; } -static std::tuple> GetCommentsFilteredText(unsigned dwCookie, int startLine, int endLine, const char **linbuf, CrystalLineParser::TextDefinition* enuType) +static std::tuple> GetCommentsFilteredText(unsigned dwCookie, int startLine, int endLine, const char **linbuf, CrystalLineParser::TextDefinition* enuType, void* parseContext) { String filteredT; std::vector allTextIsComment(endLine - startLine + 1); @@ -260,7 +261,7 @@ static std::tuple> GetCommentsFilteredT { int nActualItems = 0; std::vector blocks(textlen); - dwCookie = enuType->ParseLineX(dwCookie, text.c_str(), textlen, blocks.data(), nActualItems); + dwCookie = enuType->ParseLineX(dwCookie, i, text.c_str(), textlen, blocks.data(), nActualItems, parseContext); if (nActualItems == 0) { @@ -298,6 +299,32 @@ static std::tuple> GetCommentsFilteredT return { ucr::toUTF8(filteredT), dwCookie, allTextIsComment }; } +static std::tuple> GetTreeSitterCommentsFilteredText(int startLine, int endLine, const char **linbuf, void* parseContext) +{ + String filteredT; + std::vector allTextIsComment(endLine - startLine + 1, true); + for (int i = startLine; i <= endLine; ++i) + { + String text = convertToTString(linbuf[i], linbuf[i + 1]); + for (int j = 0; j < static_cast(text.size()); ++j) + { + const tchar_t ch = text[j]; + const bool isComment = IsTreeSitterCommentPositionForDiff(parseContext, i, j); + if (!isComment) + { + filteredT += ch; + if (ch != '\r' && ch != '\n') + allTextIsComment[i - startLine] = false; + } + } + + if (text.empty()) + allTextIsComment[i - startLine] = false; + } + + return { ucr::toUTF8(filteredT), allTextIsComment }; +} + /** * @brief Replace a string inside a string with another string. * This function searches for a string inside another string an if found, @@ -381,25 +408,44 @@ int CDiffWrapper::PostFilter(PostFilterContext& ctxt, change* thisob, const file if (m_options.m_filterCommentsLines) { - ctxt.dwCookieLeft = GetLastLineCookie(ctxt.dwCookieLeft, - ctxt.nParsedLineEndLeft + 1, lineNumberLeft - 1, file_data_ary[0].linbuf + file_data_ary[0].linbuf_base, m_pFilterCommentsDef); - ctxt.dwCookieRight = GetLastLineCookie(ctxt.dwCookieRight, - ctxt.nParsedLineEndRight + 1, lineNumberRight - 1, file_data_ary[1].linbuf + file_data_ary[1].linbuf_base, m_pFilterCommentsDef); + const bool bUseTreeSitter = m_pParseContext[0] != nullptr && m_pParseContext[1] != nullptr; ctxt.nParsedLineEndLeft = lineNumberLeft + qtyLinesLeft - 1; ctxt.nParsedLineEndRight = lineNumberRight + qtyLinesRight - 1;; - auto resultLeft = GetCommentsFilteredText(ctxt.dwCookieLeft, - lineNumberLeft, ctxt.nParsedLineEndLeft, file_data_ary[0].linbuf + file_data_ary[0].linbuf_base, m_pFilterCommentsDef); - lineDataLeft = std::move(std::get<0>(resultLeft)); - ctxt.dwCookieLeft = std::get<1>(resultLeft); - allTextIsCommentLeft = std::move(std::get<2>(resultLeft)); - - auto resultRight = GetCommentsFilteredText(ctxt.dwCookieRight, - lineNumberRight, ctxt.nParsedLineEndRight, file_data_ary[1].linbuf + file_data_ary[1].linbuf_base, m_pFilterCommentsDef); - lineDataRight = std::move(std::get<0>(resultRight)); - ctxt.dwCookieRight = std::get<1>(resultRight); - allTextIsCommentRight = std::move(std::get<2>(resultRight)); + if (bUseTreeSitter) + { + auto resultLeft = GetTreeSitterCommentsFilteredText( + lineNumberLeft, ctxt.nParsedLineEndLeft, + file_data_ary[0].linbuf + file_data_ary[0].linbuf_base, m_pParseContext[0]); + lineDataLeft = std::move(std::get<0>(resultLeft)); + allTextIsCommentLeft = std::move(std::get<1>(resultLeft)); + + auto resultRight = GetTreeSitterCommentsFilteredText( + lineNumberRight, ctxt.nParsedLineEndRight, + file_data_ary[1].linbuf + file_data_ary[1].linbuf_base, m_pParseContext[1]); + lineDataRight = std::move(std::get<0>(resultRight)); + allTextIsCommentRight = std::move(std::get<1>(resultRight)); + } + else + { + ctxt.dwCookieLeft = GetLastLineCookie(ctxt.dwCookieLeft, + ctxt.nParsedLineEndLeft + 1, lineNumberLeft - 1, file_data_ary[0].linbuf + file_data_ary[0].linbuf_base, m_pFilterCommentsDef, m_pParseContext[0]); + ctxt.dwCookieRight = GetLastLineCookie(ctxt.dwCookieRight, + ctxt.nParsedLineEndRight + 1, lineNumberRight - 1, file_data_ary[1].linbuf + file_data_ary[1].linbuf_base, m_pFilterCommentsDef, m_pParseContext[1]); + + auto resultLeft = GetCommentsFilteredText(ctxt.dwCookieLeft, + lineNumberLeft, ctxt.nParsedLineEndLeft, file_data_ary[0].linbuf + file_data_ary[0].linbuf_base, m_pFilterCommentsDef, m_pParseContext[0]); + lineDataLeft = std::move(std::get<0>(resultLeft)); + ctxt.dwCookieLeft = std::get<1>(resultLeft); + allTextIsCommentLeft = std::move(std::get<2>(resultLeft)); + + auto resultRight = GetCommentsFilteredText(ctxt.dwCookieRight, + lineNumberRight, ctxt.nParsedLineEndRight, file_data_ary[1].linbuf + file_data_ary[1].linbuf_base, m_pFilterCommentsDef, m_pParseContext[1]); + lineDataRight = std::move(std::get<0>(resultRight)); + ctxt.dwCookieRight = std::get<1>(resultRight); + allTextIsCommentRight = std::move(std::get<2>(resultRight)); + } } else { diff --git a/Src/DiffWrapper.h b/Src/DiffWrapper.h index 9c38a03c2e1..78b66ee0e59 100644 --- a/Src/DiffWrapper.h +++ b/Src/DiffWrapper.h @@ -9,6 +9,7 @@ #pragma once #include +#include #include "diff.h" #include "FileLocation.h" #include "PathContext.h" @@ -196,6 +197,7 @@ class CDiffWrapper void SetSubstitutionList(std::shared_ptr pSubstitutionFiltersList); void SetFilterCommentsSourceDef(CrystalLineParser::TextDefinition *def) { m_pFilterCommentsDef = def; }; void SetFilterCommentsSourceDef(const String& ext); + void SetFilterCommentsParseContext(void* parseContext, int index) { m_pParseContext[index] = parseContext; } void SetCodepage(int codepage) { m_codepage = codepage; } void EnablePlugins(bool enable); int PostFilter(PostFilterContext& ctxt, change* thisob, const file_data* file_data_ary) const; @@ -240,6 +242,7 @@ class CDiffWrapper DiffList *m_pDiffList; /**< Pointer to external DiffList */ std::unique_ptr m_pMovedLines[3]; CrystalLineParser::TextDefinition *m_pFilterCommentsDef; /**< Text definition for Comments filter */ + void* m_pParseContext[3]; /**< Context for incremental parsing, owned by the parser */ bool m_bPluginsEnabled; /**< Are plugins enabled? */ int m_codepage; /**< Codepage used in line filter */ }; @@ -289,4 +292,3 @@ inline void CDiffWrapper::SetAlternativePaths(const PathContext &altPaths) { m_alternativePaths = altPaths; } - diff --git a/Src/Merge.rc b/Src/Merge.rc index 667d3d73a3e..d715023fa62 100644 --- a/Src/Merge.rc +++ b/Src/Merge.rc @@ -121,11 +121,12 @@ BEGIN POPUP "&Scripts" BEGIN MENUITEM "< Empty >", ID_NO_EDIT_SCRIPTS - END - MENUITEM SEPARATOR - MENUITEM "&Go to...\tCtrl+G", ID_EDIT_WMGOTO - MENUITEM "Go to Moved Line Between Left and Middle\tCtrl+Shift+G", ID_GOTO_MOVED_LINE_LM - MENUITEM "Go to Moved Line Between Middle and Right\tCtrl+Alt+G", ID_GOTO_MOVED_LINE_MR + END + MENUITEM SEPARATOR + MENUITEM "&Go to...\tCtrl+G", ID_EDIT_WMGOTO + MENUITEM "Go to &Definition\tF12", ID_EDIT_GOTO_DEFINITION + MENUITEM "Go to Moved Line Between Left and Middle\tCtrl+Shift+G", ID_GOTO_MOVED_LINE_LM + MENUITEM "Go to Moved Line Between Middle and Right\tCtrl+Alt+G", ID_GOTO_MOVED_LINE_MR MENUITEM SEPARATOR POPUP "Op&en" BEGIN @@ -694,10 +695,11 @@ BEGIN MENUITEM "&Next Bookmark\tF2", ID_EDIT_GOTO_NEXT_BOOKMARK MENUITEM "&Previous Bookmark\tShift+F2", ID_EDIT_GOTO_PREV_BOOKMARK MENUITEM "&Clear All Bookmarks", ID_EDIT_CLEAR_ALL_BOOKMARKS - END - MENUITEM "&Go to...\tCtrl+G", ID_EDIT_WMGOTO - MENUITEM SEPARATOR - MENUITEM "&Options...\tCtrl+,", ID_OPTIONS + END + MENUITEM "&Go to...\tCtrl+G", ID_EDIT_WMGOTO + MENUITEM "Go to &Definition\tF12", ID_EDIT_GOTO_DEFINITION + MENUITEM SEPARATOR + MENUITEM "&Options...\tCtrl+,", ID_OPTIONS END POPUP "&View" BEGIN @@ -1085,11 +1087,12 @@ END IDR_POPUP_LOCATIONBAR MENU BEGIN POPUP "_POPUP_" - BEGIN - MENUITEM "G&o to Diff", ID_LOCBAR_GOTODIFF - MENUITEM "&Go to...\tCtrl+G", ID_EDIT_WMGOTO - MENUITEM SEPARATOR - MENUITEM "Move Cursor on &Click", ID_LOCBAR_MOVECURSOR_ONCLICK + BEGIN + MENUITEM "G&o to Diff", ID_LOCBAR_GOTODIFF + MENUITEM "&Go to...\tCtrl+G", ID_EDIT_WMGOTO + MENUITEM "Go to &Definition\tF12", ID_EDIT_GOTO_DEFINITION + MENUITEM SEPARATOR + MENUITEM "Move Cursor on &Click", ID_LOCBAR_MOVECURSOR_ONCLICK MENUITEM SEPARATOR MENUITEM "&No Moved Blocks", ID_DISPLAY_MOVED_NONE MENUITEM "&All Moved Blocks", ID_DISPLAY_MOVED_ALL @@ -1896,11 +1899,12 @@ BEGIN VK_F7, ID_PREVFILE, VIRTKEY, CONTROL, NOINVERT VK_END, ID_LASTDIFF, VIRTKEY, ALT, NOINVERT VK_F2, ID_DIR_ITEM_RENAME, VIRTKEY, NOINVERT - VK_F3, ID_EDIT_REPEAT, VIRTKEY, SHIFT, NOINVERT - VK_F3, ID_EDIT_REPEAT, VIRTKEY, NOINVERT - VK_F3, ID_EDIT_REPEAT, VIRTKEY, SHIFT, CONTROL, NOINVERT - VK_F3, ID_EDIT_REPEAT, VIRTKEY, CONTROL, NOINVERT - VK_F4, ID_SELECTLINEDIFF, VIRTKEY, NOINVERT + VK_F3, ID_EDIT_REPEAT, VIRTKEY, SHIFT, NOINVERT + VK_F3, ID_EDIT_REPEAT, VIRTKEY, NOINVERT + VK_F3, ID_EDIT_REPEAT, VIRTKEY, SHIFT, CONTROL, NOINVERT + VK_F3, ID_EDIT_REPEAT, VIRTKEY, CONTROL, NOINVERT + VK_F12, ID_EDIT_GOTO_DEFINITION, VIRTKEY, NOINVERT + VK_F4, ID_SELECTLINEDIFF, VIRTKEY, NOINVERT VK_F4, ID_SELECTPREVLINEDIFF, VIRTKEY, SHIFT, NOINVERT VK_F5, ID_REFRESH, VIRTKEY, NOINVERT VK_F5, ID_RESCAN, VIRTKEY, CONTROL, NOINVERT @@ -2367,18 +2371,19 @@ IDD_PROPPAGE_EDITOR DIALOGEX 0, 0, 325, 254 STYLE DS_SETFONT | DS_FIXEDSYS | WS_POPUP | WS_CAPTION CAPTION "Editor" FONT 8, FONTNAME, 0, 0, 0x1 -BEGIN - CONTROL "&Highlight syntax",IDC_HILITE_CHECK,"Button",BS_AUTOCHECKBOX | WS_TABSTOP,7,6,309,10 - CONTROL "&Preserve original EOL",IDC_MIXED_EOL,"Button",BS_AUTOCHECKBOX | WS_TABSTOP,7,18,309,10 - GROUPBOX "Tabs",IDC_STATIC,7,30,311,72,WS_GROUP - LTEXT "&Tab size:",IDC_STATIC,14,42,256,10 - EDITTEXT IDC_TAB_EDIT,24,54,18,12,ES_RIGHT | ES_AUTOHSCROLL | ES_NUMBER,WS_EX_RIGHT - CONTROL "&Insert tabs",IDC_PROP_INSERT_TABS,"Button",BS_AUTORADIOBUTTON | WS_GROUP,14,72,256,10 - CONTROL "Insert &spaces",IDC_PROP_INSERT_SPACES,"Button",BS_AUTORADIOBUTTON,14,84,256,10 - LTEXT "&Rendering mode:",IDC_STATIC,7,106,310,10 - COMBOBOX IDC_RENDERING_MODE,7,118,310,34,CBS_DROPDOWNLIST | WS_VSCROLL | WS_GROUP | WS_TABSTOP - PUSHBUTTON "Defaults",IDC_COMPARE_DEFAULTS,230,280,88,14 -END +BEGIN + CONTROL "&Highlight syntax",IDC_HILITE_CHECK,"Button",BS_AUTOCHECKBOX | WS_TABSTOP,7,6,309,10 + CONTROL "Enable &Tree-sitter",IDC_TREE_SITTER_CHECK,"Button",BS_AUTOCHECKBOX | WS_TABSTOP,7,18,309,10 + CONTROL "&Preserve original EOL",IDC_MIXED_EOL,"Button",BS_AUTOCHECKBOX | WS_TABSTOP,7,30,309,10 + GROUPBOX "Tabs",IDC_STATIC,7,42,311,72,WS_GROUP + LTEXT "&Tab size:",IDC_STATIC,14,54,256,10 + EDITTEXT IDC_TAB_EDIT,24,66,18,12,ES_RIGHT | ES_AUTOHSCROLL | ES_NUMBER,WS_EX_RIGHT + CONTROL "&Insert tabs",IDC_PROP_INSERT_TABS,"Button",BS_AUTORADIOBUTTON | WS_GROUP,14,84,256,10 + CONTROL "Insert &spaces",IDC_PROP_INSERT_SPACES,"Button",BS_AUTORADIOBUTTON,14,96,256,10 + LTEXT "&Rendering mode:",IDC_STATIC,7,118,310,10 + COMBOBOX IDC_RENDERING_MODE,7,130,310,34,CBS_DROPDOWNLIST | WS_VSCROLL | WS_GROUP | WS_TABSTOP + PUSHBUTTON "Defaults",IDC_COMPARE_DEFAULTS,230,280,88,14 +END IDD_PROPPAGE_EDITOR_COMPAREMERGE DIALOGEX 0, 0, 325, 254 STYLE DS_SETFONT | DS_FIXEDSYS | WS_POPUP | WS_CAPTION diff --git a/Src/Merge.vcxproj b/Src/Merge.vcxproj index 20b2e24540b..0cd30166a3e 100644 --- a/Src/Merge.vcxproj +++ b/Src/Merge.vcxproj @@ -197,6 +197,7 @@ + @@ -1913,6 +1914,25 @@ + + + + + + + + + + + + + + + @@ -1921,4 +1941,4 @@ - + diff --git a/Src/MergeDoc.cpp b/Src/MergeDoc.cpp index 9760194e55e..c4f4cc1aa7a 100644 --- a/Src/MergeDoc.cpp +++ b/Src/MergeDoc.cpp @@ -52,6 +52,8 @@ #include "markdown.h" #include "stringdiffs.h" #include "Logger.h" +#include "../Externals/crystaledit/editlib/TreeSitterParser.h" +#include "../Externals/crystaledit/editlib/TreeSitterWrapper.h" #ifdef _DEBUG #define new DEBUG_NEW @@ -63,6 +65,53 @@ int CMergeDoc::m_nBuffersTemp = 2; static int SaveBuffForDiff(CDiffTextBuffer & buf, const String& filepath, int nStartLine = 0, int nLines = -1); +bool CMergeDoc::IsTreeSitterEnabled() const +{ + return GetOptionsMgr()->GetBool(OPT_TREE_SITTER); +} + +void CMergeDoc::UpdateTreeSitterSupport() +{ + m_diffWrapper.SetFilterCommentsSourceDef(GetFileExt(m_ptBuf[0]->m_strTempFileName.c_str(), m_strDesc[0].c_str())); + m_diffWrapper.SetFilterCommentsParseContext(nullptr, 0); + m_diffWrapper.SetFilterCommentsParseContext(nullptr, 1); + m_diffWrapper.SetFilterCommentsParseContext(nullptr, 2); + + for (int nBuffer = 0; nBuffer < m_nBuffers; ++nBuffer) + { + m_pTreeSitterTextDefs[nBuffer].reset(); + m_pTreeSitterParsers[nBuffer].reset(); + m_ptBuf[nBuffer]->SetParseContext(nullptr); + } + + if (!IsTreeSitterEnabled()) + return; + + TreeSitterRegistry& registry = TreeSitterRegistry::Instance(); + if (!registry.IsInitialized()) + registry.Initialize(); + + for (int nBuffer = 0; nBuffer < m_nBuffers; ++nBuffer) + { + String sExt = GetFileExt(m_ptBuf[nBuffer]->GetTempFileName().c_str(), m_strDesc[nBuffer].c_str()); + const CTreeSitterLanguage* pLang = registry.GetLanguageForExt(sExt.c_str()); + if (pLang == nullptr || pLang->GetLanguage() == nullptr) + continue; + + m_pTreeSitterParsers[nBuffer] = std::make_unique(); + m_pTreeSitterParsers[nBuffer]->SetLanguage(pLang); + m_pTreeSitterParsers[nBuffer]->SetBuffer(m_ptBuf[nBuffer].get()); + m_pTreeSitterParsers[nBuffer]->ParseFromBuffer(m_ptBuf[nBuffer].get()); + + m_pTreeSitterTextDefs[nBuffer].reset(CreateTreeSitterTextDefinition(sExt.c_str(), sExt.c_str(), nBuffer)); + m_ptBuf[nBuffer]->SetParseContext(m_pTreeSitterParsers[nBuffer].get()); + m_diffWrapper.SetFilterCommentsParseContext(m_pTreeSitterParsers[nBuffer].get(), nBuffer); + } + + if (m_pTreeSitterTextDefs[0]) + m_diffWrapper.SetFilterCommentsSourceDef(m_pTreeSitterTextDefs[0].get()); +} + ///////////////////////////////////////////////////////////////////////////// // CMergeDoc @@ -582,6 +631,16 @@ int CMergeDoc::Rescan(bool &bBinary, IDENTLEVEL &identical, [&](std::unique_ptr& buf) { return buf->getEncoding() != m_ptBuf[0]->getEncoding(); })) identical = IDENTLEVEL::NONE; + for (nBuffer = 0; nBuffer < m_nBuffers; nBuffer++) + { + // Parse the document + if (m_pTreeSitterParsers[nBuffer]) + { + m_pTreeSitterParsers[nBuffer]->Invalidate(); + m_pTreeSitterParsers[nBuffer]->ParseFromBuffer(m_ptBuf[nBuffer].get()); + } + } + GetParentFrame()->SetLastCompareResult(identical != IDENTLEVEL::ALL ? 1 : 0); return nResult; @@ -2441,13 +2500,26 @@ bool CMergeDoc::OpenDocs(int nFiles, const FileLocation ifileloc[], UpdateHeaderPath(nBuffer); ForEachView(nBuffer, [](auto& pView) { pView->DocumentsLoaded(); }); - + if ((m_nBufferType[nBuffer] == BUFFERTYPE::NORMAL) || - (m_nBufferType[nBuffer] == BUFFERTYPE::NORMAL_NAMED)) + (m_nBufferType[nBuffer] == BUFFERTYPE::NORMAL_NAMED)) { nNormalBuffer++; } - + + } + + UpdateTreeSitterSupport(); + + // Set TreeSitter TextDefinition as syntax highlighting for each view + for (nBuffer = 0; nBuffer < m_nBuffers; nBuffer++) + { + if (m_pTreeSitterTextDefs[nBuffer]) + { + ForEachView(nBuffer, [&](auto& pView) { + pView->SetTextType(m_pTreeSitterTextDefs[nBuffer].get()); + }); + } } CMergeFrameCommon::LogComparisonCompleted(*this); @@ -2571,6 +2643,7 @@ void CMergeDoc::RefreshOptions() Options::DiffOptions::Load(GetOptionsMgr(), options); m_diffWrapper.SetOptions(&options); + UpdateTreeSitterSupport(); // Refresh view options ForEachView([](auto& pView) { pView->RefreshOptions(); }); @@ -3565,4 +3638,3 @@ std::vector > CMergeDoc::GetSyncPointList() } return list; } - diff --git a/Src/MergeDoc.h b/Src/MergeDoc.h index 0668b3b3091..bbc05b89236 100644 --- a/Src/MergeDoc.h +++ b/Src/MergeDoc.h @@ -24,6 +24,9 @@ #include #include +class CTreeSitterParser; +namespace CrystalLineParser { struct TextDefinition; } + /** * @brief Additional action codes for WinMerge. * @note Reserve first 100 for CrystalEditor @@ -194,6 +197,10 @@ class CMergeDoc : public CDocument, public IMergeDoc, public IMDITab void GetPrediffer(PrediffingInfo* infoPrediffer) const; const PrediffingInfo* GetPrediffer() const override; const EditorScriptInfo* GetEditorScript() const override { return &m_editorScriptInfo; }; + CTreeSitterParser* GetTreeSitterParser(int nBuffer) { return m_pTreeSitterParsers[nBuffer].get(); } + CrystalLineParser::TextDefinition* GetTreeSitterTextDefinition(int nBuffer) { return m_pTreeSitterTextDefs[nBuffer].get(); } + bool IsTreeSitterEnabled() const; + void UpdateTreeSitterSupport(); void AddMergeViews(CMergeEditSplitterView* pMergeEditSplitterView, CMergeEditView* pView[3]); void RemoveMergeViews(CMergeEditSplitterView* pMergeEditSplitterView); void SetLocationView(CLocationView* pLocationView) { m_pLocationView = pLocationView; } @@ -389,6 +396,8 @@ class CMergeDoc : public CDocument, public IMergeDoc, public IMDITab bool m_bAutoMerged; std::optional m_bEnableTableEditing; std::unique_ptr m_pTablePropsPrepared; + std::unique_ptr m_pTreeSitterParsers[3]; /**< TreeSitter parsers for each pane */ + std::unique_ptr m_pTreeSitterTextDefs[3]; /**< TreeSitter TextDefinitions for each pane */ /** * Are automatic rescans enabled? * If automatic rescans are enabled then we rescan files after edit @@ -483,4 +492,3 @@ inline bool CMergeDoc::HasSyncPoints() { return m_bHasSyncPoints; } - diff --git a/Src/MergeEditStatus.h b/Src/MergeEditStatus.h index 60e5fa3e3f4..f70c8fc35a1 100644 --- a/Src/MergeEditStatus.h +++ b/Src/MergeEditStatus.h @@ -14,4 +14,5 @@ class IMergeEditStatus public: virtual void SetLineInfo(const tchar_t* szLine, int nChar, int nChars, int nColumn, int nColumns, int nSelectedLines, int nSelectedChars, const tchar_t* szEol, int nCodepage, bool bHasBom) = 0; + virtual void SetSyntaxParser(const tchar_t* szParser) = 0; }; diff --git a/Src/MergeEditView.cpp b/Src/MergeEditView.cpp index 58d1e562065..2e09a656da3 100644 --- a/Src/MergeEditView.cpp +++ b/Src/MergeEditView.cpp @@ -106,6 +106,8 @@ BEGIN_MESSAGE_MAP(CMergeEditView, CCrystalEditViewEx) ON_UPDATE_COMMAND_UI(ID_EDIT_REDO, OnUpdateEditRedo) ON_UPDATE_COMMAND_UI(ID_EDIT_REPLACE, OnUpdateEditReplace) ON_COMMAND(ID_EDIT_WMGOTO, OnWMGoto) + ON_COMMAND(ID_EDIT_GOTO_DEFINITION, OnGotoDefinition) + ON_UPDATE_COMMAND_UI(ID_EDIT_GOTO_DEFINITION, OnUpdateGotoDefinition) ON_COMMAND(ID_EDIT_COPY_LINENUMBERS, OnEditCopyLineNumbers) ON_UPDATE_COMMAND_UI(ID_EDIT_COPY_LINENUMBERS, OnUpdateEditCopyLinenumbers) // [View] menu @@ -2419,6 +2421,16 @@ void CMergeEditView::OnEditOperation(int nAction, const tchar_t* pszText, size_t // perform original function CCrystalEditViewEx::OnEditOperation(nAction, pszText, cchText); + // Notify tree-sitter parser of the edit for incremental reparsing. + // This calls ts_tree_edit() on the existing tree so tree-sitter can + // reuse unchanged subtrees during the next reparse (deferred to paint). + // We do NOT reparse here -- doing a full reparse on every keystroke + // causes catastrophic memory/CPU usage. Instead, ParseLine() will + // call EnsureParsed() lazily when the view is next painted. + CTreeSitterParser* pParser = pDoc->GetTreeSitterParser(m_nThisPane); + if (pParser && pParser->HasLanguage()) + pParser->NotifyEdit(m_pTextBuffer); + // augment with additional operations // Change header to inform about changed doc @@ -2991,6 +3003,28 @@ void CMergeEditView::OnContextMenu(CWnd* pWnd, CPoint point) point = rect.TopLeft(); point.Offset(5, 5); } + else + { + CPoint clientPoint = point; + ScreenToClient(&clientPoint); + if (clientPoint.y >= GetTopMarginHeight()) + { + CPoint adjustedPoint = clientPoint; + AdjustTextPoint(adjustedPoint); + CEPoint textPoint = ClientToText(adjustedPoint); + if (IsValidTextPosY(textPoint)) + { + if (!IsValidTextPosX(textPoint)) + textPoint.x = GetLineLength(textPoint.y); + SetCursorPos(textPoint); + if (!IsSelection() || !IsInsideSelBlock(textPoint)) + { + SetAnchor(textPoint); + SetSelection(textPoint, textPoint); + } + } + } + } pSub->TrackPopupMenu(TPM_LEFTALIGN | TPM_RIGHTBUTTON, point.x, point.y, AfxGetMainWnd()); @@ -3272,6 +3306,48 @@ void CMergeEditView::OnWMGoto() } } +void CMergeEditView::GotoTreeSitterDefinition() +{ + CMergeDoc* pDoc = GetDocument(); + if (!pDoc->IsTreeSitterEnabled()) + return; + + CTreeSitterParser* pParser = pDoc->GetTreeSitterParser(m_nThisPane); + if (!pParser || !pParser->HasLanguage()) + return; + + pParser->EnsureParsed(pDoc->m_ptBuf[m_nThisPane].get()); + + const CEPoint pos = GetCursorPos(); + int nDefLine = 0; + int nDefChar = 0; + if (pParser->FindDefinition(pos.y, pos.x, nDefLine, nDefChar)) + GotoLine(nDefLine, false, m_nThisPane, true, nDefChar); +} + +void CMergeEditView::OnGotoDefinition() +{ + GotoTreeSitterDefinition(); +} + +void CMergeEditView::OnUpdateGotoDefinition(CCmdUI* pCmdUI) +{ + CMergeDoc* pDoc = GetDocument(); + CTreeSitterParser* pParser = pDoc->GetTreeSitterParser(m_nThisPane); + if (!pDoc->IsTreeSitterEnabled() || !pParser || !pParser->HasLanguage()) + { + pCmdUI->Enable(FALSE); + return; + } + + pParser->EnsureParsed(pDoc->m_ptBuf[m_nThisPane].get()); + + const CEPoint pos = GetCursorPos(); + int nDefLine = 0; + int nDefChar = 0; + pCmdUI->Enable(pParser->FindDefinition(pos.y, pos.x, nDefLine, nDefChar)); +} + /** * @brief Called when "Go to Moved Line Between Left and Middle" item is selected. * Go to moved line between the left and right panes when in 2-way file comparison. @@ -3436,6 +3512,8 @@ void CMergeEditView::RefreshOptions() if (!GetOptionsMgr()->GetBool(OPT_SYNTAX_HIGHLIGHT)) SetTextType(CrystalLineParser::SRC_PLAIN); + else if (!GetDocument()->GetChangedSchemeManually() && GetDocument()->IsTreeSitterEnabled() && GetDocument()->GetTreeSitterTextDefinition(m_nThisPane) != nullptr) + SetTextType(GetDocument()->GetTreeSitterTextDefinition(m_nThisPane)); else if (!GetDocument()->GetChangedSchemeManually()) { // The syntax highlighting scheme should only be applied if it has not been manually changed. @@ -4086,6 +4164,8 @@ void CMergeEditView::DocumentsLoaded() bool bInsertTabs = (GetOptionsMgr()->GetInt(OPT_TAB_TYPE) == 0); SetInsertTabs(bInsertTabs); + // Note: TreeSitter initialization is now handled by CMergeDoc::OpenDocs + // Sometimes WinMerge doesn't update scrollbars correctly (they remain // disabled) after docs are open in screen. So lets make sure they are // really updated, even though this is unnecessary in most cases. diff --git a/Src/MergeEditView.h b/Src/MergeEditView.h index b62f74dae15..ae4352937c7 100644 --- a/Src/MergeEditView.h +++ b/Src/MergeEditView.h @@ -30,6 +30,7 @@ constexpr unsigned FLAG_RESCAN_WAITS_FOR_IDLE = 1; #include "edtlib.h" #include "GhostTextView.h" #include "OptionsDiffColors.h" +#include "TreeSitterParser.h" #include #include @@ -126,6 +127,7 @@ class CMergeEditView : public CGhostTextView virtual void GetLineColors2 (int nLineIndex, DWORD ignoreFlags , CEColor & crBkgnd, CEColor & crText, bool & bDrawWhitespace); void WMGoto() { OnWMGoto(); }; + void GotoTreeSitterDefinition(); void GotoLine(UINT nLine, bool bRealLine, int pane, bool bMoveAnchor = true, int nChar = -1); int GetTopLine() const { return m_nTopLine; } using CCrystalTextView::GetScreenLines; @@ -201,6 +203,8 @@ class CMergeEditView : public CGhostTextView afx_msg void OnEditPaste(); afx_msg void OnUpdateEditPaste(CCmdUI* pCmdUI); afx_msg void OnEditUndo(); + afx_msg void OnGotoDefinition(); + afx_msg void OnUpdateGotoDefinition(CCmdUI* pCmdUI); afx_msg void OnFirstdiff(); afx_msg void OnUpdateFirstdiff(CCmdUI* pCmdUI); afx_msg void OnLastdiff(); @@ -362,4 +366,3 @@ inline bool CMergeEditView::IsCursorInDiff() const { return m_bCurrentLineIsDiff; } - diff --git a/Src/MergeStatusBar.cpp b/Src/MergeStatusBar.cpp index 5ac81c9e23c..dbf8bdd9524 100644 --- a/Src/MergeStatusBar.cpp +++ b/Src/MergeStatusBar.cpp @@ -287,6 +287,14 @@ void CMergeStatusBar::MergeStatus::Update() if (m_nCodepage > 0) strEncoding.Format(_("%s").c_str(), m_sCodepageName.c_str()); + if (!m_sSyntaxParser.empty()) + { + if (!strEncoding.IsEmpty()) + strEncoding += _T(" "); + strEncoding += _T("[TS:"); + strEncoding += m_sSyntaxParser.c_str(); + strEncoding += _T("]"); + } m_pWndStatusBar->SetPaneText(m_base, strInfo); m_pWndStatusBar->SetPaneText(m_base + 1, strEncoding); } @@ -300,6 +308,20 @@ void CMergeStatusBar::MergeStatus::UpdateResources() Update(); } +/** + * @brief Set the syntax parser indicator text. + * @param szParser Language name (e.g. "fsharp"), or empty/null to clear. + */ +void CMergeStatusBar::MergeStatus::SetSyntaxParser(const tchar_t* szParser) +{ + String sParser = szParser ? szParser : _T(""); + if (m_sSyntaxParser != sParser) + { + m_sSyntaxParser = sParser; + Update(); + } +} + /// Visible representation of eol static String EolString(const String & sEol) { diff --git a/Src/MergeStatusBar.h b/Src/MergeStatusBar.h index 45830346e74..bdbf1958ae0 100644 --- a/Src/MergeStatusBar.h +++ b/Src/MergeStatusBar.h @@ -48,6 +48,7 @@ public : // Implement MergeEditStatus void SetLineInfo(const tchar_t* szLine, int nColumn, int nColumns, int nChar, int nChars, int nSelectedLines, int nSelectedChars, const tchar_t* szEol, int nCodepage, bool bHasBom) override; + void SetSyntaxParser(const tchar_t* szParser) override; void UpdateResources(); protected: void Update(); @@ -67,6 +68,7 @@ public : String m_sEol; String m_sEolDisplay; String m_sCodepageName; + String m_sSyntaxParser; /**< Tree-sitter language name, empty if not active */ }; friend class MergeStatus; // MergeStatus accesses status bar MergeStatus m_status[3]; diff --git a/Src/OptionsDef.h b/Src/OptionsDef.h index 7bd02e4e5cb..91632bbaae9 100644 --- a/Src/OptionsDef.h +++ b/Src/OptionsDef.h @@ -41,6 +41,7 @@ inline const String OPT_TOOLBAR_SIZE {_T("Settings/ToolbarSize"s)}; inline const String OPT_RESIZE_PANES {_T("Settings/AutoResizePanes"s)}; inline const String OPT_SYNTAX_HIGHLIGHT {_T("Settings/HiliteSyntax"s)}; +inline const String OPT_TREE_SITTER {_T("Settings/TreeSitter"s)}; inline const String OPT_VIEW_WHITESPACE {_T("Settings/ViewWhitespace"s)}; inline const String OPT_VIEW_EOL {_T("Settings/ViewEOL"s)}; inline const String OPT_SCROLL_TO_FIRST {_T("Settings/ScrollToFirst"s)}; diff --git a/Src/OptionsInit.cpp b/Src/OptionsInit.cpp index ebe509f34a1..1d65429ed22 100644 --- a/Src/OptionsInit.cpp +++ b/Src/OptionsInit.cpp @@ -69,6 +69,7 @@ void Init(COptionsMgr *pOptions) pOptions->InitOption(OPT_RESIZE_PANES, false); pOptions->InitOption(OPT_SYNTAX_HIGHLIGHT, true); + pOptions->InitOption(OPT_TREE_SITTER, true); pOptions->InitOption(OPT_WORDWRAP, false); pOptions->InitOption(OPT_WORDWRAP_TABLE, false); pOptions->InitOption(OPT_VIEW_LINENUMBERS, false); diff --git a/Src/PropEditor.cpp b/Src/PropEditor.cpp index c30adddf7b4..b770f24d6f5 100644 --- a/Src/PropEditor.cpp +++ b/Src/PropEditor.cpp @@ -24,6 +24,7 @@ static const unsigned MAX_TABSIZE = 64u; PropEditor::PropEditor(COptionsMgr *optionsMgr) : OptionsPanel(optionsMgr, PropEditor::IDD) , m_bHiliteSyntax(false) +, m_bTreeSitter(false) , m_nTabType(-1) , m_nTabSize(0) , m_bAllowMixedEol(false) @@ -33,6 +34,7 @@ PropEditor::PropEditor(COptionsMgr *optionsMgr) +[](unsigned v) { return v; }, +[](unsigned v) { return std::clamp(v, 1u, MAX_TABSIZE); }); BindOption(OPT_TAB_TYPE, m_nTabType, IDC_PROP_INSERT_TABS, DDX_Radio); BindOption(OPT_SYNTAX_HIGHLIGHT, m_bHiliteSyntax, IDC_HILITE_CHECK, DDX_Check); + BindOption(OPT_TREE_SITTER, m_bTreeSitter, IDC_TREE_SITTER_CHECK, DDX_Check); BindOption(OPT_ALLOW_MIXED_EOL, m_bAllowMixedEol, IDC_MIXED_EOL, DDX_Check); BindOptionCustom(OPT_RENDERING_MODE, m_nRenderingMode, IDC_RENDERING_MODE, DDX_CBIndex, +[](int v) { return v + 1; }, +[](int v) { return (v - 1); }); diff --git a/Src/PropEditor.h b/Src/PropEditor.h index 8dd3eee027e..c0b4361a009 100644 --- a/Src/PropEditor.h +++ b/Src/PropEditor.h @@ -27,6 +27,7 @@ class PropEditor : public OptionsPanel //{{AFX_DATA(PropEditor) enum { IDD = IDD_PROPPAGE_EDITOR }; bool m_bHiliteSyntax; + bool m_bTreeSitter; int m_nTabType; unsigned m_nTabSize; bool m_bAllowMixedEol; diff --git a/Src/resource.h b/Src/resource.h index 2d15e271960..11a43fc38e1 100644 --- a/Src/resource.h +++ b/Src/resource.h @@ -240,8 +240,9 @@ #define IDC_WWW 1019 #define IDC_IGNCASE_CHECK 1020 #define IDC_COMPANY 1021 -#define IDC_GNU_ASCII 1022 -#define IDC_HILITE_CHECK 1025 +#define IDC_GNU_ASCII 1022 +#define IDC_TREE_SITTER_CHECK 1023 +#define IDC_HILITE_CHECK 1025 #define IDC_IGNOREREGEXP 1026 #define IDC_ALL_WHITE 1027 #define IDC_WHITE_CHANGE 1028 @@ -993,11 +994,12 @@ #define ID_PROJECT_DIFF_OPTIONS_COMPMETHOD_SIZE 33140 #define ID_PROJECT_DIFF_OPTIONS_COMPMETHOD_EXISTENCE 33141 #define ID_FILE_OPENCLIPBOARD 33142 -#define ID_EDIT_TOGGLE_BOOKMARK 33145 -#define ID_EDIT_GOTO_NEXT_BOOKMARK 33146 -#define ID_EDIT_GOTO_PREV_BOOKMARK 33147 -#define ID_EDIT_CLEAR_ALL_BOOKMARKS 33148 -#define ID_VIEW_CHANGESCHEME 33150 +#define ID_EDIT_TOGGLE_BOOKMARK 33145 +#define ID_EDIT_GOTO_NEXT_BOOKMARK 33146 +#define ID_EDIT_GOTO_PREV_BOOKMARK 33147 +#define ID_EDIT_CLEAR_ALL_BOOKMARKS 33148 +#define ID_EDIT_GOTO_DEFINITION 33149 +#define ID_VIEW_CHANGESCHEME 33150 #define ID_COLORSCHEME_FIRST 33151 #define IDS_COLORSCHEME_PLAIN 33151 #define IDS_COLORSCHEME_ABAP 33152 diff --git a/Testing/GoogleTest/UnitTests/UnitTests.vcxproj b/Testing/GoogleTest/UnitTests/UnitTests.vcxproj index c8823e73293..d6bdc413606 100644 --- a/Testing/GoogleTest/UnitTests/UnitTests.vcxproj +++ b/Testing/GoogleTest/UnitTests/UnitTests.vcxproj @@ -930,4 +930,4 @@ - + \ No newline at end of file diff --git a/Testing/GoogleTest/UnitTests/UnitTests.vcxproj.filters b/Testing/GoogleTest/UnitTests/UnitTests.vcxproj.filters index 17b6dc94f57..ef04712b869 100644 --- a/Testing/GoogleTest/UnitTests/UnitTests.vcxproj.filters +++ b/Testing/GoogleTest/UnitTests/UnitTests.vcxproj.filters @@ -559,4 +559,4 @@ TestData\DirWatcher\Subdir1 - + \ No newline at end of file diff --git a/Tools/TreeSitterGrammars/build-grammars.ps1 b/Tools/TreeSitterGrammars/build-grammars.ps1 new file mode 100644 index 00000000000..1c22ace3487 --- /dev/null +++ b/Tools/TreeSitterGrammars/build-grammars.ps1 @@ -0,0 +1,396 @@ +<# +.SYNOPSIS + Downloads and compiles tree-sitter grammar DLLs for WinMerge. +.DESCRIPTION + Reads grammars.json, downloads release tarballs from GitHub, reads each + repo's tree-sitter.json for source layout, compiles to DLL via MSVC cl.exe. +.PARAMETER OutDir + Output directory for DLLs and .scm files. +.PARAMETER Platform + Target platform: x64, x86, or ARM64. Default: x64. +.PARAMETER Configuration + Build configuration: Release or Debug. Default: Release. +.PARAMETER GrammarFilter + Optional regex to build only matching grammar names. +.EXAMPLE + .\build-grammars.ps1 + .\build-grammars.ps1 -GrammarFilter "fsharp" +#> +param( + [string]$OutDir, + [ValidateSet("x64","x86","ARM64")] + [string]$Platform = "x64", + [ValidateSet("Release","Debug")] + [string]$Configuration = "Release", + [string]$GrammarFilter +) + +$ErrorActionPreference = "Continue" + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$RepoRoot = (Resolve-Path (Join-Path $ScriptDir "..\..")).Path +if (-not $OutDir) { + $OutDir = Join-Path $RepoRoot "Build\$Platform\$Configuration\TreeSitterGrammars" +} +$TempBase = Join-Path $RepoRoot "BuildTmp\grammar-sources" +$ConfigFile = Join-Path $ScriptDir "grammars.json" + +# ---- Locate and import MSVC environment ---- + +function Find-VcVarsAll { + $vswhere = Join-Path ${env:ProgramFiles(x86)} "Microsoft Visual Studio\Installer\vswhere.exe" + if (-not (Test-Path $vswhere)) { + $vswhere = Join-Path $env:ProgramFiles "Microsoft Visual Studio\Installer\vswhere.exe" + } + if (-not (Test-Path $vswhere)) { throw "vswhere.exe not found" } + $ip = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath 2>$null + if (-not $ip) { throw "No VS with C++ tools found" } + $vcvars = Join-Path $ip "VC\Auxiliary\Build\vcvarsall.bat" + if (-not (Test-Path $vcvars)) { throw "vcvarsall.bat not found at $vcvars" } + return $vcvars +} + +function Import-VcEnvironment { + param([string]$Arch) + $vcvars = Find-VcVarsAll + Write-Host "Importing MSVC environment ($Arch) ..." + $batContent = "@call `"$vcvars`" $Arch >nul 2>&1`r`n@set > `"$env:TEMP\vcvars_env.txt`"`r`n" + [System.IO.File]::WriteAllText("$env:TEMP\vcvars_capture.bat", $batContent) + cmd.exe /c "$env:TEMP\vcvars_capture.bat" + if (Test-Path "$env:TEMP\vcvars_env.txt") { + foreach ($line in (Get-Content "$env:TEMP\vcvars_env.txt")) { + if ($line -match '^([^=]+)=(.*)$') { + [Environment]::SetEnvironmentVariable($matches[1], $matches[2], "Process") + } + } + Remove-Item "$env:TEMP\vcvars_capture.bat","$env:TEMP\vcvars_env.txt" -Force -EA SilentlyContinue + } + $cl = Get-Command cl.exe -EA SilentlyContinue + if (-not $cl) { throw "cl.exe not found after importing vcvars" } + Write-Host " cl.exe: $($cl.Source)" +} + +function Get-VcArch { + switch ($Platform) { + "x64" { return "amd64" } + "x86" { return "x86" } + "ARM64" { return "amd64_arm64" } + default { return "amd64" } + } +} + +# ---- Download grammar source ---- + +function Get-GrammarSource { + param([string]$Repo, [string]$Tag) + $name = ($Repo -split '/')[-1] + $extractDir = Join-Path $TempBase $name + if (Test-Path (Join-Path $extractDir "tree-sitter.json")) { + Write-Host " Cached: $extractDir" + return $extractDir + } + New-Item -ItemType Directory -Path $TempBase -Force | Out-Null + New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + # Try .tar.gz first, then .tar.xz (older releases use xz) + $downloaded = $false + foreach ($ext in @("tar.gz", "tar.xz")) { + $tarUrl = "https://github.com/$Repo/releases/download/$Tag/$name.$ext" + $tarFile = Join-Path $TempBase "$name.$ext" + try { + Write-Host " Downloading $name.$ext ..." + Invoke-WebRequest -Uri $tarUrl -OutFile $tarFile -UseBasicParsing -ErrorAction Stop + $downloaded = $true + break + } catch { + # Try next format + } + } + if (-not $downloaded) { + throw "No release tarball found for $Repo $Tag" + } + + Write-Host " Extracting ..." + # Use git-bash tar with Unix-style paths (Windows tar.exe fails on ./prefixed entries) + $gitBash = Join-Path $env:ProgramFiles "Git\bin\bash.exe" + $unixTarFile = $tarFile -replace '\\','/' -replace '^([A-Za-z]):','/$1' + $unixExtractDir = $extractDir -replace '\\','/' -replace '^([A-Za-z]):','/$1' + & $gitBash -c "tar -xf '$unixTarFile' -C '$unixExtractDir'" + Remove-Item $tarFile -Force + return $extractDir +} + +function Resolve-NodeModulesPath { + param( + [string]$Candidate, + [string]$CacheBaseDir + ) + + $separator = [string][IO.Path]::DirectorySeparatorChar + $escapedSeparator = [regex]::Escape($separator) + $normalized = $Candidate -replace '/', $separator + if (-not $normalized.StartsWith("node_modules$separator")) { + return $null + } + + $parts = $normalized -split $escapedSeparator + $isScopedPackage = ($parts.Count -ge 2 -and $parts[1].StartsWith('@')) + $minPartCount = if ($isScopedPackage) { 4 } else { 3 } + if ($parts.Count -lt $minPartCount) { + return $null + } + + $packageName = $parts[1] + $restIndex = 2 + $cacheDirs = New-Object System.Collections.Generic.List[string] + if ($isScopedPackage) { + $scopedPackageName = $packageName + $separator + $parts[2] + $restIndex = 3 + $cacheDirs.Add((Join-Path $CacheBaseDir $scopedPackageName)) + $packageNameWithinScope = $parts[2] + } else { + $packageNameWithinScope = $packageName + } + $cacheDirs.Add((Join-Path $CacheBaseDir $packageNameWithinScope)) + + if ($restIndex -ge $parts.Count) { + return $null + } + $relativePath = $parts[$restIndex..($parts.Count - 1)] -join $separator + foreach ($cachedSourceDir in $cacheDirs) { + if (-not (Test-Path $cachedSourceDir)) { + continue + } + $resolvedPath = Join-Path $cachedSourceDir $relativePath + if (Test-Path $resolvedPath) { + return $resolvedPath + } + } + + return $null +} + +function Resolve-QueryFiles { + param( + [object]$QuerySpec, + [string]$SourceDir, + [string]$RepoDir, + [string]$CacheBaseDir, + [string]$FallbackRelativePath + ) + + $resolved = New-Object System.Collections.Generic.List[string] + + if ($QuerySpec) { + $queryPaths = if ($QuerySpec -is [array]) { $QuerySpec } else { @($QuerySpec) } + foreach ($candidate in $queryPaths) { + $tryPaths = @( + (Join-Path $RepoDir $candidate), + (Join-Path $SourceDir $candidate), + (Resolve-NodeModulesPath -Candidate $candidate -CacheBaseDir $CacheBaseDir) + ) | Where-Object { $_ } + + foreach ($tryPath in $tryPaths) { + if ((Test-Path $tryPath) -and (-not $resolved.Contains($tryPath))) { + $resolved.Add($tryPath) + break + } + } + } + } + + if ($resolved.Count -eq 0 -and $FallbackRelativePath) { + $fallbacks = @( + (Join-Path $SourceDir $FallbackRelativePath), + (Join-Path $RepoDir $FallbackRelativePath) + ) + foreach ($fallback in $fallbacks) { + if ((Test-Path $fallback) -and (-not $resolved.Contains($fallback))) { + $resolved.Add($fallback) + break + } + } + } + + return $resolved.ToArray() +} + +function Write-QueryBundle { + param( + [string]$DestinationPath, + [string[]]$SourcePaths + ) + + if (-not $SourcePaths -or $SourcePaths.Count -eq 0) { + return $false + } + + $contents = foreach ($sourcePath in $SourcePaths) { + try { + [System.IO.File]::ReadAllText($sourcePath) + } catch { + Write-Warning " Failed to read query source '$sourcePath': $_" + return $false + } + } + if ($contents -contains $false) { + return $false + } + $newline = [System.Environment]::NewLine + $doubleNewline = $newline + $newline + $bundle = ($contents -join $doubleNewline).TrimEnd() + [System.IO.File]::WriteAllText($DestinationPath, $bundle + $newline, [System.Text.UTF8Encoding]::new($false)) + return $true +} + +# ---- Compile a grammar to DLL ---- + +function Build-GrammarDll { + param( + [string]$GrammarName, + [string]$SourceDir, + [string]$RepoDir, + [string[]]$HighlightsScm, + [string[]]$LocalsScm, + [string[]]$TagsScm, + [string[]]$InjectionsScm, + [string]$DllName + ) + $srcDir = Join-Path $SourceDir "src" + $parserC = Join-Path $srcDir "parser.c" + $scannerC = Join-Path $srcDir "scanner.c" + if (-not (Test-Path $parserC)) { + Write-Warning " parser.c not found at $parserC - skipping $GrammarName" + return $false + } + $sources = @($parserC) + if (Test-Path $scannerC) { $sources += $scannerC } + + $buildDir = Join-Path $RepoRoot "BuildTmp\grammar-build\$DllName\$Platform\$Configuration" + New-Item -ItemType Directory -Path $buildDir -Force | Out-Null + New-Item -ItemType Directory -Path $OutDir -Force | Out-Null + $dllPath = Join-Path $OutDir "$DllName.dll" + + $cflags = @("/nologo","/c","/TC","/W3","/D_USRDLL","/D_WINDOWS","/wd4996","/wd4267","/wd4244","/wd4101") + if ($Configuration -eq "Release") { + $cflags += @("/O2","/MD","/DNDEBUG","/GL") + } else { + $cflags += @("/Od","/MDd","/D_DEBUG","/Zi") + } + + $objFiles = @() + foreach ($src in $sources) { + $objName = [IO.Path]::GetFileNameWithoutExtension($src) + ".obj" + $objPath = Join-Path $buildDir $objName + $objFiles += $objPath + $fileName = [IO.Path]::GetFileName($src) + Write-Host " Compiling $fileName ..." + $allArgs = $cflags + @("/I`"$srcDir`"", "/Fo`"$objPath`"", "`"$src`"") + $p = Start-Process cl.exe -ArgumentList $allArgs -NoNewWindow -Wait -PassThru + if ($p.ExitCode -ne 0) { + Write-Error " cl.exe failed for $fileName (exit $($p.ExitCode))" + return $false + } + } + + Write-Host " Linking $DllName.dll ..." + $linkArgs = @("/nologo","/DLL","/OUT:`"$dllPath`"") + if ($Configuration -eq "Release") { + $linkArgs += @("/LTCG","/OPT:REF","/OPT:ICF") + } else { + $linkArgs += @("/DEBUG") + } + $linkArgs += $objFiles + $p = Start-Process link.exe -ArgumentList $linkArgs -NoNewWindow -Wait -PassThru + if ($p.ExitCode -ne 0) { + Write-Error " link.exe failed for $DllName (exit $($p.ExitCode))" + return $false + } + + if (Write-QueryBundle -DestinationPath (Join-Path $OutDir "$GrammarName-highlights.scm") -SourcePaths $HighlightsScm) { + Write-Host " Bundled highlights -> $GrammarName-highlights.scm" + } else { + Write-Warning " No highlights.scm for $GrammarName" + } + if (Write-QueryBundle -DestinationPath (Join-Path $OutDir "$GrammarName-locals.scm") -SourcePaths $LocalsScm) { + Write-Host " Bundled locals -> $GrammarName-locals.scm" + } + if (Write-QueryBundle -DestinationPath (Join-Path $OutDir "$GrammarName-tags.scm") -SourcePaths $TagsScm) { + Write-Host " Bundled tags -> $GrammarName-tags.scm" + } + if (Write-QueryBundle -DestinationPath (Join-Path $OutDir "$GrammarName-injections.scm") -SourcePaths $InjectionsScm) { + Write-Host " Bundled injections -> $GrammarName-injections.scm" + } + Write-Host " OK: $dllPath" + return $true +} + +# ---- Main ---- + +$succeeded = [int]0 +$failed = [int]0 +$skipped = [int]0 + +Write-Host "=== WinMerge Tree-Sitter Grammar Builder ===" +Write-Host "Platform: $Platform" +Write-Host "Configuration: $Configuration" +Write-Host "Output: $OutDir" +Write-Host "" + +Import-VcEnvironment -Arch (Get-VcArch) +Write-Host "" + +$config = Get-Content $ConfigFile -Raw | ConvertFrom-Json + +foreach ($entry in $config.grammars) { + $repo = $entry.repo + $tag = $entry.tag + $repoName = ($repo -split '/')[-1] + Write-Host "--- $repoName ($tag) ---" + + try { + $repoDir = Get-GrammarSource -Repo $repo -Tag $tag + } catch { + Write-Warning " Download failed: $_" + $failed++ + continue + } + + $tsJsonPath = Join-Path $repoDir "tree-sitter.json" + if (-not (Test-Path $tsJsonPath)) { + Write-Warning " No tree-sitter.json - skipping" + $skipped++ + continue + } + $tsJson = Get-Content $tsJsonPath -Raw | ConvertFrom-Json + + foreach ($g in $tsJson.grammars) { + $gName = $g.name + $gPath = $g.path + if (-not $gPath) { $gPath = "." } + + if ($GrammarFilter -and ($gName -notmatch $GrammarFilter)) { + Write-Host " Skipping $gName (filtered)" + $skipped++ + continue + } + + $sourceDir = if ($gPath -eq ".") { $repoDir } else { Join-Path $repoDir $gPath } + $dllName = "tree-sitter-$gName" + + # Resolve .scm query files. When tree-sitter.json uses an array, all + # matching files are bundled in order so inherited queries are kept. + $hlScm = Resolve-QueryFiles -QuerySpec $g.highlights -SourceDir $sourceDir -RepoDir $repoDir -CacheBaseDir $TempBase -FallbackRelativePath "queries\highlights.scm" + $lcScm = Resolve-QueryFiles -QuerySpec $g.locals -SourceDir $sourceDir -RepoDir $repoDir -CacheBaseDir $TempBase -FallbackRelativePath "queries\locals.scm" + $tgScm = Resolve-QueryFiles -QuerySpec $g.tags -SourceDir $sourceDir -RepoDir $repoDir -CacheBaseDir $TempBase -FallbackRelativePath "queries\tags.scm" + $ijScm = Resolve-QueryFiles -QuerySpec $g.injections -SourceDir $sourceDir -RepoDir $repoDir -CacheBaseDir $TempBase -FallbackRelativePath "queries\injections.scm" + + Write-Host " Grammar: $gName (path: $gPath)" + $ok = Build-GrammarDll -GrammarName $gName -SourceDir $sourceDir -RepoDir $repoDir -HighlightsScm $hlScm -LocalsScm $lcScm -TagsScm $tgScm -InjectionsScm $ijScm -DllName $dllName + if ($ok) { $succeeded++ } else { $failed++ } + } + Write-Host "" +} + +Write-Host "=== Done: $succeeded succeeded, $failed failed, $skipped skipped ===" +if ($failed -gt 0) { exit 1 } diff --git a/Tools/TreeSitterGrammars/fsharp-highlights.scm b/Tools/TreeSitterGrammars/fsharp-highlights.scm new file mode 100644 index 00000000000..bb61a073695 --- /dev/null +++ b/Tools/TreeSitterGrammars/fsharp-highlights.scm @@ -0,0 +1,403 @@ +;; ---------------------------------------------------------------------------- +;; Literals and comments + +[ + (line_comment) + (xml_doc) + (block_comment) +] @comment @spell + +(xml_doc) @comment.documentation @spell + +(const + [ + (_) @constant + (unit) @constant.builtin + ]) + +(primary_constr_args (_) @variable.parameter) + +(class_as_reference + (_) @variable.parameter.builtin) + + +((argument_patterns (long_identifier (identifier) @character.special)) + (#match? @character.special "^\_.*")) + +;; ---------------------------------------------------------------------------- +;; Punctuation + +(type_name type_name: (_) @type.definition) +(exception_definition exception_name: (_) @type.definition) + +[ + (_type) + (atomic_type) +] @type + +(member_signature + . + (identifier) @function.member + (curried_spec + (arguments_spec + "*"* @operator + (argument_spec + (argument_name_spec + "?"? @character.special + name: (_) @variable.parameter))))) + +(union_type_case (identifier) @constant) + +(rules + (rule + pattern: (_) @constant + block: (_))) + +(wildcard_pattern) @character.special + +(identifier_pattern + . + (_) @constant + . + (_) @variable) + +(optional_pattern + "?" @character.special) + +(fsi_directive_decl . (string) @module) + +(import_decl . (_) @module) +(named_module + name: (_) @module) +(namespace + name: (_) @module) +(module_defn + . + (_) @module) + +(ce_expression + . + (_) @constant.macro) + +(field_initializer + field: (_) @property) + +(record_fields + (record_field + . + (identifier) @property)) + +(value_declaration_left . (_) @variable) + +(function_declaration_left + . (_) @function) + +(argument_patterns) @variable.parameter +(typed_pattern + (_pattern) @variable.parameter + (_type) @type) + +(member_defn + (method_or_prop_defn + [ + (property_or_ident) @function + (property_or_ident + instance: (identifier) @variable.parameter.builtin + method: (identifier) @function.method) + ] + args: (_)* @variable.parameter)) + + +(dot_expression + . + (_) @variable.member + . + (_)) + +(application_expression + . + [ + (long_identifier_or_op) + (dot_expression) + ] @function.call + . + (_) @variable) + +(application_expression + . + (typed_expression + (long_identifier_or_op) @function.call + "<" @operator + (_) + ">" @operator) + . + (_) @variable) + +((infix_expression + . + (_) + . + (infix_op) @operator + . + (_) @function.call + ) + (#eq? @operator "|>") + ) + +((infix_expression + . + (_) @function.call + . + (infix_op) @operator + . + (_) + ) + (#eq? @operator "<|") + ) + +[ + (xint) + (int) + (int16) + (uint16) + (int32) + (uint32) + (int64) + (uint64) + (nativeint) + (unativeint) +] @number + +[ + (ieee32) + (ieee64) + (float) + (decimal) +] @number.float + +(bool) @boolean + +([ + (string) + (triple_quoted_string) + (verbatim_string) + (char) + (format_string) + (format_triple_quoted_string) +] @spell @string) + +(compiler_directive_decl) @keyword.directive + +(preproc_line + "#line" @keyword.directive) + +(attribute + target: (identifier)? @keyword + (_type) @attribute) + +[ + "(" + ")" + "{" + "}" + "[" + "]" + "[|" + "|]" + "{|" + "|}" +] @punctuation.bracket + +[ + "[<" + ">]" +] @punctuation.special + +(format_string_eval + [ + "{" + "}" + ] @punctuation.special) + +[ + "," + ";" + ":" + "." +] @punctuation.delimiter + +[ + "|" + "=" + ">" + "<" + "-" + "~" + "->" + "<-" + "&" + "&&" + "|" + "||" + ":>" + ":?>" + ".." + (infix_op) + (prefix_op) + (op_identifier) +] @operator + +(generic_type + [ + "<" + ">" + ] @punctuation.bracket) + +[ + "if" + "then" + "else" + "elif" + "when" + "match" + "match!" +] @keyword.conditional + +[ + "and" + "or" + "not" + "upcast" + "downcast" +] @keyword.operator + +[ + "return" + "return!" + "yield" + "yield!" +] @keyword.return + +[ + "for" + "while" + "downto" + "to" +] @keyword.repeat + + +[ + "open" + "#r" + "#load" +] @keyword.import + +[ + "abstract" + "delegate" + "extern" + "static" + "inline" + "mutable" + "override" + "rec" + "global" + (access_modifier) +] @keyword.modifier + +[ + "let" + "let!" + "use" + "use!" + "and!" + "member" +] @keyword.function + +[ + "enum" + "type" + "exception" + "inherit" + "interface" + "and" + "class" + "struct" +] @keyword.type + +((identifier) @keyword.exception + (#any-of? @keyword.exception "failwith" "failwithf" "raise" "reraise")) + +[ + "as" + "assert" + "begin" + "end" + "done" + "default" + "in" + "do" + "do!" + "fun" + "function" + "get" + "set" + "lazy" + "new" + "of" + "struct" + "val" + "module" + "namespace" + "with" +] @keyword + +[ + "null" +] @constant.builtin + +(match_expression "with" @keyword.conditional) + +(try_expression + [ + "try" + "with" + "finally" + ] @keyword.exception) + +((_type + (simple_type + (long_identifier + (identifier) @type.builtin))) + (#any-of? @type.builtin "bool" "byte" "sbyte" "int16" "uint16" "int" "uint" "int64" "uint64" "nativeint" "unativeint" "decimal" "float" "double" "float32" "single" "char" "string" "unit")) + +(preproc_if + [ + "#if" @keyword.directive + "#endif" @keyword.directive + ] + condition: (_)? @keyword.directive) + +(preproc_else + "#else" @keyword.directive) + +((long_identifier + (identifier)+ @variable.member + . + (identifier))) + +((identifier) @module.builtin + (#any-of? @module.builtin "Array" "Async" "Directory" "File" "List" "Option" "Path" "Map" "Set" "Lazy" "Seq" "Task" "String" "Result" )) + +((value_declaration + (attributes + (attribute + (_type + (simple_type + (long_identifier + (identifier) @attribute))))) + (function_or_value_defn + (value_declaration_left + . + (_) @constant))) + (#eq? @attribute "Literal")) diff --git a/Tools/TreeSitterGrammars/fsharp-injections.scm b/Tools/TreeSitterGrammars/fsharp-injections.scm new file mode 100644 index 00000000000..71457af2948 --- /dev/null +++ b/Tools/TreeSitterGrammars/fsharp-injections.scm @@ -0,0 +1,10 @@ +([ + (line_comment) + (block_comment_content) +] @injection.content + (#set! injection.language "comment")) + +((xml_doc) @injection.content + (#offset! @injection.content 0 3 0 0) + (#set! injection.language "xml") + (#set! injection.combined)) diff --git a/Tools/TreeSitterGrammars/fsharp-locals.scm b/Tools/TreeSitterGrammars/fsharp-locals.scm new file mode 100644 index 00000000000..9e473a94e0d --- /dev/null +++ b/Tools/TreeSitterGrammars/fsharp-locals.scm @@ -0,0 +1,33 @@ +(identifier) @local.reference + +[ + (namespace) + (named_module) + (function_or_value_defn) +] @local.scope + +(value_declaration_left + . + [ + (_ (identifier) @local.definition) + (_ (_ (identifier) @local.definition)) + (_ (_ (_ (identifier) @local.definition))) + (_ (_ (_ (_ (identifier) @local.definition)))) + (_ (_ (_ (_ (_ (identifier) @local.definition))))) + (_ (_ (_ (_ (_ (_ (identifier) @local.definition)))))) + ]) + +(function_declaration_left + . + ((_) @local.definition + (#set! "definition.function.scope" "parent")) + ((argument_patterns + [ + (_ (identifier) @local.definition) + (_ (_ (identifier) @local.definition)) + (_ (_ (_ (identifier) @local.definition))) + (_ (_ (_ (_ (identifier) @local.definition)))) + (_ (_ (_ (_ (_ (identifier) @local.definition))))) + (_ (_ (_ (_ (_ (_ (identifier) @local.definition)))))) + ]) + )) diff --git a/Tools/TreeSitterGrammars/grammars.json b/Tools/TreeSitterGrammars/grammars.json new file mode 100644 index 00000000000..e65bb5c44c1 --- /dev/null +++ b/Tools/TreeSitterGrammars/grammars.json @@ -0,0 +1,21 @@ +{ + "grammars": [ + { "repo": "ionide/tree-sitter-fsharp", "tag": "0.3.0" }, + { "repo": "tree-sitter/tree-sitter-c", "tag": "v0.24.1" }, + { "repo": "tree-sitter/tree-sitter-cpp", "tag": "v0.23.4" }, + { "repo": "tree-sitter/tree-sitter-c-sharp", "tag": "v0.23.1" }, + { "repo": "tree-sitter/tree-sitter-python", "tag": "v0.25.0" }, + { "repo": "tree-sitter/tree-sitter-javascript", "tag": "v0.25.0" }, + { "repo": "tree-sitter/tree-sitter-typescript", "tag": "v0.23.2" }, + { "repo": "tree-sitter/tree-sitter-java", "tag": "v0.23.5" }, + { "repo": "tree-sitter/tree-sitter-go", "tag": "v0.25.0" }, + { "repo": "tree-sitter/tree-sitter-rust", "tag": "v0.24.1" }, + { "repo": "tree-sitter/tree-sitter-json", "tag": "v0.24.8" }, + { "repo": "tree-sitter/tree-sitter-html", "tag": "v0.23.2" }, + { "repo": "tree-sitter/tree-sitter-css", "tag": "v0.25.0" }, + { "repo": "tree-sitter/tree-sitter-bash", "tag": "v0.25.1" }, + { "repo": "tree-sitter/tree-sitter-ruby", "tag": "v0.23.1" }, + { "repo": "tree-sitter/tree-sitter-php", "tag": "v0.24.2" }, + { "repo": "tree-sitter-grammars/tree-sitter-xml", "tag": "v0.7.0" } + ] +} diff --git a/Translations/Docs/Manual/Italian.po b/Translations/Docs/Manual/Italian.po index 84def1c5491..d269e646251 100644 --- a/Translations/Docs/Manual/Italian.po +++ b/Translations/Docs/Manual/Italian.po @@ -15181,8 +15181,8 @@ msgstr "" "tramite i prefissi e:, e!:, " "fe:, fe!:, de: o de!: in il campo Filtro). Non devono essere " -"usati nelle regole di inclusione/esclusione del filtro file (.flt), poiché richiedono che il confronto venga prima completato e " +"uszati nelle regole di inclusione/esclusione del filtro file (." +"flt), poiché richiedono che il confronto venga prima completato e " "potrebbero causare comportamenti imprevisti." #. type: Content of:
diff --git a/WinMerge.sln b/WinMerge.sln index f902ef20ca5..193a40d837c 100644 --- a/WinMerge.sln +++ b/WinMerge.sln @@ -1,211 +1,215 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 18 -VisualStudioVersion = 18.0.11205.157 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Merge", "Src\Merge.vcxproj", "{9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "MergeLang", "Src\MergeLang.vcxproj", "{4B011DDA-2279-437D-903C-8028913AF31B}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "UnitTests", "Testing\GoogleTest\UnitTests\UnitTests.vcxproj", "{733E7C0B-AC3D-47AC-A8DA-E13644D6294D}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Poco", "Poco", "{220B870C-D051-463E-997B-8C392081EE15}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Batch Files", "Batch Files", "{DC3B258E-444F-460D-8FD9-09A8165212FA}" - ProjectSection(SolutionItems) = preProject - BuildAll.cmd = BuildAll.cmd - BuildAll.vs2017.cmd = BuildAll.vs2017.cmd - BuildAll.vs2017_32bit_latest_64bit.cmd = BuildAll.vs2017_32bit_latest_64bit.cmd - BuildAll.vs2022.cmd = BuildAll.vs2022.cmd - BuildArc.cmd = BuildArc.cmd - BuildBin.cmd = BuildBin.cmd - BuildBin.vs2017.cmd = BuildBin.vs2017.cmd - BuildBin.vs2022.cmd = BuildBin.vs2022.cmd - BuildInstaller.cmd = BuildInstaller.cmd - BuildManual.cmd = BuildManual.cmd - CreateRelease.cmd = CreateRelease.cmd - DownloadDeps.cmd = DownloadDeps.cmd - SetVersion.cmd = SetVersion.cmd - UploadToVirusTotal.cmd = UploadToVirusTotal.cmd - EndProjectSection -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Scripts", "Scripts", "{8B625EC8-5063-4336-84F9-AA7FD5348525}" - ProjectSection(SolutionItems) = preProject - ExpandEnvironmenStrings.js = ExpandEnvironmenStrings.js - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GUITests", "Testing\GoogleTest\GUITests\GUITests.vcxproj", "{0A3727B1-51E7-4702-AD0C-8AEE317EA510}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{2A835469-516E-4B8A-93F1-33EF70B80981}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "editlib", "Externals\crystaledit\editlib\editlib.vcxitems", "{7515AC3C-389A-44CD-B940-A59DDE5B8AE3}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "xdiff", "Externals\xdiff\xdiff.vcxitems", "{68F1D3A1-9DCA-4B3D-B245-F4ACA5F16563}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "diffutils", "Src\diffutils\diffutils.vcxitems", "{FC3B9DF3-2854-4264-AB4B-EE8C43982513}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CompareEngines", "Src\CompareEngines\CompareEngines.vcxitems", "{0F686AFA-D587-43C0-BADA-2BEDDC3FA758}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "googletest", "Externals\googletest\googletest\googletest.vcxitems", "{9EE35458-B145-444F-92B7-27FF72112C42}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "editlibparsers", "Externals\crystaledit\editlib\editlibparsers.vcxitems", "{4170552A-09E2-4FAC-B71D-0E2F5EB3C869}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "FilterEngine", "Src\FilterEngine\FilterEngine.vcxitems", "{9C37E5D8-1DC0-4EAC-AADB-5FC8BE4FB1BC}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Foundation", "Externals\poco\Foundation\Foundation.vcxitems", "{B30837C4-F99F-46F9-ADFB-5FAEFB1D2FBB}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "XML", "Externals\poco\XML\XML.vcxitems", "{DAFD8D9E-6509-4482-A533-44C843C3BCC7}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CrashLogger", "Src\CrashLogger\CrashLogger.vcxitems", "{CDEDDAC4-79B9-4A34-A564-8019F60D241F}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|ARM = Debug|ARM - Debug|ARM64 = Debug|ARM64 - Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 - Release|ARM = Release|ARM - Release|ARM64 = Release|ARM64 - Release|x64 = Release|x64 - Release|x86 = Release|x86 - Test|ARM = Test|ARM - Test|ARM64 = Test|ARM64 - Test|x64 = Test|x64 - Test|x86 = Test|x86 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Debug|ARM.ActiveCfg = Debug|ARM - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Debug|ARM.Build.0 = Debug|ARM - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Debug|ARM64.ActiveCfg = Debug|ARM64 - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Debug|ARM64.Build.0 = Debug|ARM64 - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Debug|x64.ActiveCfg = Debug|x64 - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Debug|x64.Build.0 = Debug|x64 - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Debug|x86.ActiveCfg = Debug|Win32 - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Debug|x86.Build.0 = Debug|Win32 - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Release|ARM.ActiveCfg = Release|ARM - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Release|ARM.Build.0 = Release|ARM - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Release|ARM64.ActiveCfg = Release|ARM64 - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Release|ARM64.Build.0 = Release|ARM64 - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Release|x64.ActiveCfg = Release|x64 - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Release|x64.Build.0 = Release|x64 - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Release|x86.ActiveCfg = Release|Win32 - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Release|x86.Build.0 = Release|Win32 - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Test|ARM.ActiveCfg = Test|ARM - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Test|ARM.Build.0 = Test|ARM - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Test|ARM64.ActiveCfg = Test|ARM64 - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Test|ARM64.Build.0 = Test|ARM64 - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Test|x64.ActiveCfg = Test|x64 - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Test|x64.Build.0 = Test|x64 - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Test|x86.ActiveCfg = Test|Win32 - {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Test|x86.Build.0 = Test|Win32 - {4B011DDA-2279-437D-903C-8028913AF31B}.Debug|ARM.ActiveCfg = Debug|ARM - {4B011DDA-2279-437D-903C-8028913AF31B}.Debug|ARM.Build.0 = Debug|ARM - {4B011DDA-2279-437D-903C-8028913AF31B}.Debug|ARM64.ActiveCfg = Debug|ARM64 - {4B011DDA-2279-437D-903C-8028913AF31B}.Debug|ARM64.Build.0 = Debug|ARM64 - {4B011DDA-2279-437D-903C-8028913AF31B}.Debug|x64.ActiveCfg = Debug|x64 - {4B011DDA-2279-437D-903C-8028913AF31B}.Debug|x64.Build.0 = Debug|x64 - {4B011DDA-2279-437D-903C-8028913AF31B}.Debug|x86.ActiveCfg = Debug|Win32 - {4B011DDA-2279-437D-903C-8028913AF31B}.Debug|x86.Build.0 = Debug|Win32 - {4B011DDA-2279-437D-903C-8028913AF31B}.Release|ARM.ActiveCfg = Release|ARM - {4B011DDA-2279-437D-903C-8028913AF31B}.Release|ARM.Build.0 = Release|ARM - {4B011DDA-2279-437D-903C-8028913AF31B}.Release|ARM64.ActiveCfg = Release|ARM64 - {4B011DDA-2279-437D-903C-8028913AF31B}.Release|ARM64.Build.0 = Release|ARM64 - {4B011DDA-2279-437D-903C-8028913AF31B}.Release|x64.ActiveCfg = Release|x64 - {4B011DDA-2279-437D-903C-8028913AF31B}.Release|x64.Build.0 = Release|x64 - {4B011DDA-2279-437D-903C-8028913AF31B}.Release|x86.ActiveCfg = Release|Win32 - {4B011DDA-2279-437D-903C-8028913AF31B}.Release|x86.Build.0 = Release|Win32 - {4B011DDA-2279-437D-903C-8028913AF31B}.Test|ARM.ActiveCfg = Debug|ARM - {4B011DDA-2279-437D-903C-8028913AF31B}.Test|ARM.Build.0 = Debug|ARM - {4B011DDA-2279-437D-903C-8028913AF31B}.Test|ARM64.ActiveCfg = Debug|ARM64 - {4B011DDA-2279-437D-903C-8028913AF31B}.Test|ARM64.Build.0 = Debug|ARM64 - {4B011DDA-2279-437D-903C-8028913AF31B}.Test|x64.ActiveCfg = Debug|x64 - {4B011DDA-2279-437D-903C-8028913AF31B}.Test|x64.Build.0 = Debug|x64 - {4B011DDA-2279-437D-903C-8028913AF31B}.Test|x86.ActiveCfg = Debug|Win32 - {4B011DDA-2279-437D-903C-8028913AF31B}.Test|x86.Build.0 = Debug|Win32 - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Debug|ARM.ActiveCfg = Debug|ARM - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Debug|ARM.Build.0 = Debug|ARM - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Debug|ARM64.ActiveCfg = Debug|ARM64 - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Debug|ARM64.Build.0 = Debug|ARM64 - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Debug|x64.ActiveCfg = Debug|x64 - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Debug|x64.Build.0 = Debug|x64 - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Debug|x86.ActiveCfg = Debug|Win32 - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Debug|x86.Build.0 = Debug|Win32 - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Release|ARM.ActiveCfg = Release|ARM - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Release|ARM.Build.0 = Release|ARM - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Release|ARM64.ActiveCfg = Release|ARM64 - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Release|ARM64.Build.0 = Release|ARM64 - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Release|x64.ActiveCfg = Release|x64 - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Release|x64.Build.0 = Release|x64 - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Release|x86.ActiveCfg = Release|Win32 - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Release|x86.Build.0 = Release|Win32 - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Test|ARM.ActiveCfg = Debug|ARM - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Test|ARM.Build.0 = Debug|ARM - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Test|ARM64.ActiveCfg = Debug|ARM64 - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Test|x64.ActiveCfg = Debug|x64 - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Test|x86.ActiveCfg = Debug|Win32 - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Debug|ARM.ActiveCfg = Debug|ARM - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Debug|ARM.Build.0 = Debug|ARM - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Debug|ARM64.ActiveCfg = Debug|ARM64 - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Debug|ARM64.Build.0 = Debug|ARM64 - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Debug|x64.ActiveCfg = Debug|x64 - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Debug|x64.Build.0 = Debug|x64 - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Debug|x86.ActiveCfg = Debug|Win32 - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Debug|x86.Build.0 = Debug|Win32 - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Release|ARM.ActiveCfg = Release|ARM - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Release|ARM.Build.0 = Release|ARM - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Release|ARM64.ActiveCfg = Release|ARM64 - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Release|ARM64.Build.0 = Release|ARM64 - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Release|x64.ActiveCfg = Release|x64 - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Release|x64.Build.0 = Release|x64 - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Release|x86.ActiveCfg = Release|Win32 - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Release|x86.Build.0 = Release|Win32 - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Test|ARM.ActiveCfg = Debug|ARM - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Test|ARM.Build.0 = Debug|ARM - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Test|ARM64.ActiveCfg = Debug|ARM64 - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Test|x64.ActiveCfg = Debug|x64 - {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Test|x86.ActiveCfg = Debug|Win32 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(NestedProjects) = preSolution - {733E7C0B-AC3D-47AC-A8DA-E13644D6294D} = {2A835469-516E-4B8A-93F1-33EF70B80981} - {8B625EC8-5063-4336-84F9-AA7FD5348525} = {DC3B258E-444F-460D-8FD9-09A8165212FA} - {0A3727B1-51E7-4702-AD0C-8AEE317EA510} = {2A835469-516E-4B8A-93F1-33EF70B80981} - {9EE35458-B145-444F-92B7-27FF72112C42} = {2A835469-516E-4B8A-93F1-33EF70B80981} - {B30837C4-F99F-46F9-ADFB-5FAEFB1D2FBB} = {220B870C-D051-463E-997B-8C392081EE15} - {DAFD8D9E-6509-4482-A533-44C843C3BCC7} = {220B870C-D051-463E-997B-8C392081EE15} - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {BA780561-B5A9-4ED1-B35C-B342695F3F39} - EndGlobalSection - GlobalSection(SharedMSBuildProjectFiles) = preSolution - Externals\googletest\googletest\googletest.vcxitems*{0a3727b1-51e7-4702-ad0c-8aee317ea510}*SharedItemsImports = 4 - Src\CompareEngines\CompareEngines.vcxitems*{0f686afa-d587-43c0-bada-2beddc3fa758}*SharedItemsImports = 9 - Externals\crystaledit\editlib\editlibparsers.vcxitems*{4170552a-09e2-4fac-b71d-0e2f5eb3c869}*SharedItemsImports = 9 - Externals\xdiff\xdiff.vcxitems*{68f1d3a1-9dca-4b3d-b245-f4aca5f16563}*SharedItemsImports = 9 - Externals\crystaledit\editlib\editlibparsers.vcxitems*{733e7c0b-ac3d-47ac-a8da-e13644d6294d}*SharedItemsImports = 4 - Externals\googletest\googletest\googletest.vcxitems*{733e7c0b-ac3d-47ac-a8da-e13644d6294d}*SharedItemsImports = 4 - Externals\poco\Foundation\Foundation.vcxitems*{733e7c0b-ac3d-47ac-a8da-e13644d6294d}*SharedItemsImports = 4 - Externals\poco\XML\XML.vcxitems*{733e7c0b-ac3d-47ac-a8da-e13644d6294d}*SharedItemsImports = 4 - Externals\xdiff\xdiff.vcxitems*{733e7c0b-ac3d-47ac-a8da-e13644d6294d}*SharedItemsImports = 4 - Src\CompareEngines\CompareEngines.vcxitems*{733e7c0b-ac3d-47ac-a8da-e13644d6294d}*SharedItemsImports = 4 - Src\diffutils\diffutils.vcxitems*{733e7c0b-ac3d-47ac-a8da-e13644d6294d}*SharedItemsImports = 4 - Src\FilterEngine\FilterEngine.vcxitems*{733e7c0b-ac3d-47ac-a8da-e13644d6294d}*SharedItemsImports = 4 - Externals\crystaledit\editlib\editlib.vcxitems*{7515ac3c-389a-44cd-b940-a59dde5b8ae3}*SharedItemsImports = 9 - Src\FilterEngine\FilterEngine.vcxitems*{9c37e5d8-1dc0-4eac-aadb-5fc8be4fb1bc}*SharedItemsImports = 9 - Externals\googletest\googletest\googletest.vcxitems*{9ee35458-b145-444f-92b7-27ff72112c42}*SharedItemsImports = 9 - Externals\crystaledit\editlib\editlib.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 - Externals\crystaledit\editlib\editlibparsers.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 - Externals\poco\Foundation\Foundation.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 - Externals\poco\XML\XML.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 - Externals\xdiff\xdiff.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 - Src\CompareEngines\CompareEngines.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 - Src\CrashLogger\CrashLogger.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 - Src\diffutils\diffutils.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 - Src\FilterEngine\FilterEngine.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 - Externals\poco\Foundation\Foundation.vcxitems*{b30837c4-f99f-46f9-adfb-5faefb1d2fbb}*SharedItemsImports = 9 - Src\CrashLogger\CrashLogger.vcxitems*{cdeddac4-79b9-4a34-a564-8019f60d241f}*SharedItemsImports = 9 - Externals\poco\XML\XML.vcxitems*{dafd8d9e-6509-4482-a533-44c843c3bcc7}*SharedItemsImports = 9 - Src\diffutils\diffutils.vcxitems*{fc3b9df3-2854-4264-ab4b-ee8c43982513}*SharedItemsImports = 9 - EndGlobalSection -EndGlobal +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 18 +VisualStudioVersion = 18.0.11205.157 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Merge", "Src\Merge.vcxproj", "{9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "MergeLang", "Src\MergeLang.vcxproj", "{4B011DDA-2279-437D-903C-8028913AF31B}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "UnitTests", "Testing\GoogleTest\UnitTests\UnitTests.vcxproj", "{733E7C0B-AC3D-47AC-A8DA-E13644D6294D}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Poco", "Poco", "{220B870C-D051-463E-997B-8C392081EE15}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Batch Files", "Batch Files", "{DC3B258E-444F-460D-8FD9-09A8165212FA}" + ProjectSection(SolutionItems) = preProject + BuildAll.cmd = BuildAll.cmd + BuildAll.vs2017.cmd = BuildAll.vs2017.cmd + BuildAll.vs2017_32bit_latest_64bit.cmd = BuildAll.vs2017_32bit_latest_64bit.cmd + BuildAll.vs2022.cmd = BuildAll.vs2022.cmd + BuildArc.cmd = BuildArc.cmd + BuildBin.cmd = BuildBin.cmd + BuildBin.vs2017.cmd = BuildBin.vs2017.cmd + BuildBin.vs2022.cmd = BuildBin.vs2022.cmd + BuildInstaller.cmd = BuildInstaller.cmd + BuildManual.cmd = BuildManual.cmd + CreateRelease.cmd = CreateRelease.cmd + DownloadDeps.cmd = DownloadDeps.cmd + SetVersion.cmd = SetVersion.cmd + UploadToVirusTotal.cmd = UploadToVirusTotal.cmd + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Scripts", "Scripts", "{8B625EC8-5063-4336-84F9-AA7FD5348525}" + ProjectSection(SolutionItems) = preProject + ExpandEnvironmenStrings.js = ExpandEnvironmenStrings.js + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GUITests", "Testing\GoogleTest\GUITests\GUITests.vcxproj", "{0A3727B1-51E7-4702-AD0C-8AEE317EA510}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{2A835469-516E-4B8A-93F1-33EF70B80981}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "editlib", "Externals\crystaledit\editlib\editlib.vcxitems", "{7515AC3C-389A-44CD-B940-A59DDE5B8AE3}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "xdiff", "Externals\xdiff\xdiff.vcxitems", "{68F1D3A1-9DCA-4B3D-B245-F4ACA5F16563}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "diffutils", "Src\diffutils\diffutils.vcxitems", "{FC3B9DF3-2854-4264-AB4B-EE8C43982513}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CompareEngines", "Src\CompareEngines\CompareEngines.vcxitems", "{0F686AFA-D587-43C0-BADA-2BEDDC3FA758}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "googletest", "Externals\googletest\googletest\googletest.vcxitems", "{9EE35458-B145-444F-92B7-27FF72112C42}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "editlibparsers", "Externals\crystaledit\editlib\editlibparsers.vcxitems", "{4170552A-09E2-4FAC-B71D-0E2F5EB3C869}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "FilterEngine", "Src\FilterEngine\FilterEngine.vcxitems", "{9C37E5D8-1DC0-4EAC-AADB-5FC8BE4FB1BC}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Foundation", "Externals\poco\Foundation\Foundation.vcxitems", "{B30837C4-F99F-46F9-ADFB-5FAEFB1D2FBB}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "XML", "Externals\poco\XML\XML.vcxitems", "{DAFD8D9E-6509-4482-A533-44C843C3BCC7}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CrashLogger", "Src\CrashLogger\CrashLogger.vcxitems", "{CDEDDAC4-79B9-4A34-A564-8019F60D241F}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tree-sitter", "Externals\crystaledit\editlib\tree-sitter\tree-sitter.vcxitems", "{A3E7E04F-5D2B-4F3A-B725-8F5C4B1E9D3C}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|ARM = Debug|ARM + Debug|ARM64 = Debug|ARM64 + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|ARM = Release|ARM + Release|ARM64 = Release|ARM64 + Release|x64 = Release|x64 + Release|x86 = Release|x86 + Test|ARM = Test|ARM + Test|ARM64 = Test|ARM64 + Test|x64 = Test|x64 + Test|x86 = Test|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Debug|ARM.ActiveCfg = Debug|ARM + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Debug|ARM.Build.0 = Debug|ARM + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Debug|ARM64.Build.0 = Debug|ARM64 + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Debug|x64.ActiveCfg = Debug|x64 + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Debug|x64.Build.0 = Debug|x64 + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Debug|x86.ActiveCfg = Debug|Win32 + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Debug|x86.Build.0 = Debug|Win32 + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Release|ARM.ActiveCfg = Release|ARM + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Release|ARM.Build.0 = Release|ARM + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Release|ARM64.ActiveCfg = Release|ARM64 + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Release|ARM64.Build.0 = Release|ARM64 + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Release|x64.ActiveCfg = Release|x64 + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Release|x64.Build.0 = Release|x64 + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Release|x86.ActiveCfg = Release|Win32 + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Release|x86.Build.0 = Release|Win32 + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Test|ARM.ActiveCfg = Test|ARM + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Test|ARM.Build.0 = Test|ARM + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Test|ARM64.ActiveCfg = Test|ARM64 + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Test|ARM64.Build.0 = Test|ARM64 + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Test|x64.ActiveCfg = Test|x64 + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Test|x64.Build.0 = Test|x64 + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Test|x86.ActiveCfg = Test|Win32 + {9FDA4AF0-CCFD-4812-BDB9-53EFEDB32BDE}.Test|x86.Build.0 = Test|Win32 + {4B011DDA-2279-437D-903C-8028913AF31B}.Debug|ARM.ActiveCfg = Debug|ARM + {4B011DDA-2279-437D-903C-8028913AF31B}.Debug|ARM.Build.0 = Debug|ARM + {4B011DDA-2279-437D-903C-8028913AF31B}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {4B011DDA-2279-437D-903C-8028913AF31B}.Debug|ARM64.Build.0 = Debug|ARM64 + {4B011DDA-2279-437D-903C-8028913AF31B}.Debug|x64.ActiveCfg = Debug|x64 + {4B011DDA-2279-437D-903C-8028913AF31B}.Debug|x64.Build.0 = Debug|x64 + {4B011DDA-2279-437D-903C-8028913AF31B}.Debug|x86.ActiveCfg = Debug|Win32 + {4B011DDA-2279-437D-903C-8028913AF31B}.Debug|x86.Build.0 = Debug|Win32 + {4B011DDA-2279-437D-903C-8028913AF31B}.Release|ARM.ActiveCfg = Release|ARM + {4B011DDA-2279-437D-903C-8028913AF31B}.Release|ARM.Build.0 = Release|ARM + {4B011DDA-2279-437D-903C-8028913AF31B}.Release|ARM64.ActiveCfg = Release|ARM64 + {4B011DDA-2279-437D-903C-8028913AF31B}.Release|ARM64.Build.0 = Release|ARM64 + {4B011DDA-2279-437D-903C-8028913AF31B}.Release|x64.ActiveCfg = Release|x64 + {4B011DDA-2279-437D-903C-8028913AF31B}.Release|x64.Build.0 = Release|x64 + {4B011DDA-2279-437D-903C-8028913AF31B}.Release|x86.ActiveCfg = Release|Win32 + {4B011DDA-2279-437D-903C-8028913AF31B}.Release|x86.Build.0 = Release|Win32 + {4B011DDA-2279-437D-903C-8028913AF31B}.Test|ARM.ActiveCfg = Debug|ARM + {4B011DDA-2279-437D-903C-8028913AF31B}.Test|ARM.Build.0 = Debug|ARM + {4B011DDA-2279-437D-903C-8028913AF31B}.Test|ARM64.ActiveCfg = Debug|ARM64 + {4B011DDA-2279-437D-903C-8028913AF31B}.Test|ARM64.Build.0 = Debug|ARM64 + {4B011DDA-2279-437D-903C-8028913AF31B}.Test|x64.ActiveCfg = Debug|x64 + {4B011DDA-2279-437D-903C-8028913AF31B}.Test|x64.Build.0 = Debug|x64 + {4B011DDA-2279-437D-903C-8028913AF31B}.Test|x86.ActiveCfg = Debug|Win32 + {4B011DDA-2279-437D-903C-8028913AF31B}.Test|x86.Build.0 = Debug|Win32 + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Debug|ARM.ActiveCfg = Debug|ARM + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Debug|ARM.Build.0 = Debug|ARM + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Debug|ARM64.Build.0 = Debug|ARM64 + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Debug|x64.ActiveCfg = Debug|x64 + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Debug|x64.Build.0 = Debug|x64 + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Debug|x86.ActiveCfg = Debug|Win32 + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Debug|x86.Build.0 = Debug|Win32 + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Release|ARM.ActiveCfg = Release|ARM + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Release|ARM.Build.0 = Release|ARM + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Release|ARM64.ActiveCfg = Release|ARM64 + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Release|ARM64.Build.0 = Release|ARM64 + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Release|x64.ActiveCfg = Release|x64 + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Release|x64.Build.0 = Release|x64 + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Release|x86.ActiveCfg = Release|Win32 + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Release|x86.Build.0 = Release|Win32 + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Test|ARM.ActiveCfg = Debug|ARM + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Test|ARM.Build.0 = Debug|ARM + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Test|ARM64.ActiveCfg = Debug|ARM64 + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Test|x64.ActiveCfg = Debug|x64 + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D}.Test|x86.ActiveCfg = Debug|Win32 + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Debug|ARM.ActiveCfg = Debug|ARM + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Debug|ARM.Build.0 = Debug|ARM + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Debug|ARM64.Build.0 = Debug|ARM64 + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Debug|x64.ActiveCfg = Debug|x64 + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Debug|x64.Build.0 = Debug|x64 + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Debug|x86.ActiveCfg = Debug|Win32 + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Debug|x86.Build.0 = Debug|Win32 + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Release|ARM.ActiveCfg = Release|ARM + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Release|ARM.Build.0 = Release|ARM + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Release|ARM64.ActiveCfg = Release|ARM64 + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Release|ARM64.Build.0 = Release|ARM64 + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Release|x64.ActiveCfg = Release|x64 + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Release|x64.Build.0 = Release|x64 + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Release|x86.ActiveCfg = Release|Win32 + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Release|x86.Build.0 = Release|Win32 + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Test|ARM.ActiveCfg = Debug|ARM + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Test|ARM.Build.0 = Debug|ARM + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Test|ARM64.ActiveCfg = Debug|ARM64 + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Test|x64.ActiveCfg = Debug|x64 + {0A3727B1-51E7-4702-AD0C-8AEE317EA510}.Test|x86.ActiveCfg = Debug|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {733E7C0B-AC3D-47AC-A8DA-E13644D6294D} = {2A835469-516E-4B8A-93F1-33EF70B80981} + {8B625EC8-5063-4336-84F9-AA7FD5348525} = {DC3B258E-444F-460D-8FD9-09A8165212FA} + {0A3727B1-51E7-4702-AD0C-8AEE317EA510} = {2A835469-516E-4B8A-93F1-33EF70B80981} + {9EE35458-B145-444F-92B7-27FF72112C42} = {2A835469-516E-4B8A-93F1-33EF70B80981} + {B30837C4-F99F-46F9-ADFB-5FAEFB1D2FBB} = {220B870C-D051-463E-997B-8C392081EE15} + {DAFD8D9E-6509-4482-A533-44C843C3BCC7} = {220B870C-D051-463E-997B-8C392081EE15} + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {BA780561-B5A9-4ED1-B35C-B342695F3F39} + EndGlobalSection + GlobalSection(SharedMSBuildProjectFiles) = preSolution + Externals\googletest\googletest\googletest.vcxitems*{0a3727b1-51e7-4702-ad0c-8aee317ea510}*SharedItemsImports = 4 + Src\CompareEngines\CompareEngines.vcxitems*{0f686afa-d587-43c0-bada-2beddc3fa758}*SharedItemsImports = 9 + Externals\crystaledit\editlib\editlibparsers.vcxitems*{4170552a-09e2-4fac-b71d-0e2f5eb3c869}*SharedItemsImports = 9 + Externals\xdiff\xdiff.vcxitems*{68f1d3a1-9dca-4b3d-b245-f4aca5f16563}*SharedItemsImports = 9 + Externals\crystaledit\editlib\editlibparsers.vcxitems*{733e7c0b-ac3d-47ac-a8da-e13644d6294d}*SharedItemsImports = 4 + Externals\googletest\googletest\googletest.vcxitems*{733e7c0b-ac3d-47ac-a8da-e13644d6294d}*SharedItemsImports = 4 + Externals\poco\Foundation\Foundation.vcxitems*{733e7c0b-ac3d-47ac-a8da-e13644d6294d}*SharedItemsImports = 4 + Externals\poco\XML\XML.vcxitems*{733e7c0b-ac3d-47ac-a8da-e13644d6294d}*SharedItemsImports = 4 + Externals\xdiff\xdiff.vcxitems*{733e7c0b-ac3d-47ac-a8da-e13644d6294d}*SharedItemsImports = 4 + Src\CompareEngines\CompareEngines.vcxitems*{733e7c0b-ac3d-47ac-a8da-e13644d6294d}*SharedItemsImports = 4 + Src\diffutils\diffutils.vcxitems*{733e7c0b-ac3d-47ac-a8da-e13644d6294d}*SharedItemsImports = 4 + Src\FilterEngine\FilterEngine.vcxitems*{733e7c0b-ac3d-47ac-a8da-e13644d6294d}*SharedItemsImports = 4 + Externals\crystaledit\editlib\editlib.vcxitems*{7515ac3c-389a-44cd-b940-a59dde5b8ae3}*SharedItemsImports = 9 + Src\FilterEngine\FilterEngine.vcxitems*{9c37e5d8-1dc0-4eac-aadb-5fc8be4fb1bc}*SharedItemsImports = 9 + Externals\googletest\googletest\googletest.vcxitems*{9ee35458-b145-444f-92b7-27ff72112c42}*SharedItemsImports = 9 + Externals\crystaledit\editlib\editlib.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 + Externals\crystaledit\editlib\editlibparsers.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 + Externals\crystaledit\editlib\tree-sitter\tree-sitter.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 + Externals\poco\Foundation\Foundation.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 + Externals\poco\XML\XML.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 + Externals\xdiff\xdiff.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 + Src\CompareEngines\CompareEngines.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 + Src\CrashLogger\CrashLogger.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 + Src\diffutils\diffutils.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 + Src\FilterEngine\FilterEngine.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 + Externals\crystaledit\editlib\tree-sitter\tree-sitter.vcxitems*{a3e7e04f-5d2b-4f3a-b725-8f5c4b1e9d3c}*SharedItemsImports = 9 + Externals\poco\Foundation\Foundation.vcxitems*{b30837c4-f99f-46f9-adfb-5faefb1d2fbb}*SharedItemsImports = 9 + Src\CrashLogger\CrashLogger.vcxitems*{cdeddac4-79b9-4a34-a564-8019f60d241f}*SharedItemsImports = 9 + Externals\poco\XML\XML.vcxitems*{dafd8d9e-6509-4482-a533-44c843c3bcc7}*SharedItemsImports = 9 + Src\diffutils\diffutils.vcxitems*{fc3b9df3-2854-4264-ab4b-ee8c43982513}*SharedItemsImports = 9 + EndGlobalSection +EndGlobal diff --git a/WinMerge.vs2017.sln b/WinMerge.vs2017.sln index 98638e43a78..5bc08cf79a1 100644 --- a/WinMerge.vs2017.sln +++ b/WinMerge.vs2017.sln @@ -57,6 +57,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "XML", "Externals\poco\XML\X EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CrashLogger", "Src\CrashLogger\CrashLogger.vcxitems", "{CDEDDAC4-79B9-4A34-A564-8019F60D241F}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tree-sitter", "Externals\crystaledit\editlib\tree-sitter\tree-sitter.vcxitems", "{A3E7E04F-5D2B-4F3A-B725-8F5C4B1E9D3C}" +EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution Externals\googletest\googletest\googletest.vcxitems*{0a3727b1-51e7-4702-ad0c-8aee317ea510}*SharedItemsImports = 4 @@ -76,6 +78,7 @@ Global Externals\googletest\googletest\googletest.vcxitems*{9ee35458-b145-444f-92b7-27ff72112c42}*SharedItemsImports = 9 Externals\crystaledit\editlib\editlib.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 Externals\crystaledit\editlib\editlibparsers.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 + Externals\crystaledit\editlib\tree-sitter\tree-sitter.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 Externals\poco\Foundation\Foundation.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 Externals\poco\XML\XML.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 Externals\xdiff\xdiff.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 @@ -83,6 +86,7 @@ Global Src\CrashLogger\CrashLogger.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 Src\diffutils\diffutils.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 Src\FilterEngine\FilterEngine.vcxitems*{9fda4af0-ccfd-4812-bdb9-53efedb32bde}*SharedItemsImports = 4 + Externals\crystaledit\editlib\tree-sitter\tree-sitter.vcxitems*{a3e7e04f-5d2b-4f3a-b725-8f5c4b1e9d3c}*SharedItemsImports = 9 Externals\poco\Foundation\Foundation.vcxitems*{b30837c4-f99f-46f9-adfb-5faefb1d2fbb}*SharedItemsImports = 9 Src\CrashLogger\CrashLogger.vcxitems*{cdeddac4-79b9-4a34-a564-8019f60d241f}*SharedItemsImports = 9 Externals\poco\XML\XML.vcxitems*{dafd8d9e-6509-4482-a533-44c843c3bcc7}*SharedItemsImports = 9