Add ReadOnlyMemory<char> support

.gitignore

@@ -372,3 +372,7 @@ src/.vscode/launch.json
 .idea/.idea.TextMateSharp.dir/.idea/indexLayout.xml
 .idea/.idea.TextMateSharp.dir/.idea/vcs.xml
 .idea/.idea.TextMateSharp/.idea/riderMarkupCache.xml
+.idea/.idea.TextMateSharp/.idea/copilot.data.migration.agent.xml
+.idea/.idea.TextMateSharp/.idea/copilot.data.migration.ask.xml
+.idea/.idea.TextMateSharp/.idea/copilot.data.migration.ask2agent.xml
+.idea/.idea.TextMateSharp/.idea/copilot.data.migration.edit.xml

TextMateSharp.sln

@@ -31,39 +31,104 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Workflows", "Workflows", "{
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TextMateSharp.Grammars.Tests", "src\TextMateSharp.Grammars.Tests\TextMateSharp.Grammars.Tests.csproj", "{B9194474-83A7-47E6-B5E6-6CE360B1189B}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TextMateSharp.Benchmarks", "src\TextMateSharp.Benchmarks\TextMateSharp.Benchmarks.csproj", "{C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Benchmarks", "Benchmarks", "{0D367332-B489-41A1-AD22-3F8D07F627C1}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{F84B0BEF-53D7-43AD-93AB-2025127B6D84}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
+		Debug|x64 = Debug|x64
+		Debug|x86 = Debug|x86
 		Release|Any CPU = Release|Any CPU
+		Release|x64 = Release|x64
+		Release|x86 = Release|x86
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{664F185F-961B-496E-9159-3CC8F05DBBE5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{664F185F-961B-496E-9159-3CC8F05DBBE5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{664F185F-961B-496E-9159-3CC8F05DBBE5}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{664F185F-961B-496E-9159-3CC8F05DBBE5}.Debug|x64.Build.0 = Debug|Any CPU
+		{664F185F-961B-496E-9159-3CC8F05DBBE5}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{664F185F-961B-496E-9159-3CC8F05DBBE5}.Debug|x86.Build.0 = Debug|Any CPU
 		{664F185F-961B-496E-9159-3CC8F05DBBE5}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{664F185F-961B-496E-9159-3CC8F05DBBE5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{664F185F-961B-496E-9159-3CC8F05DBBE5}.Release|x64.ActiveCfg = Release|Any CPU
+		{664F185F-961B-496E-9159-3CC8F05DBBE5}.Release|x64.Build.0 = Release|Any CPU
+		{664F185F-961B-496E-9159-3CC8F05DBBE5}.Release|x86.ActiveCfg = Release|Any CPU
+		{664F185F-961B-496E-9159-3CC8F05DBBE5}.Release|x86.Build.0 = Release|Any CPU
 		{DB75EFF5-4248-4679-9C59-9533998936B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{DB75EFF5-4248-4679-9C59-9533998936B3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DB75EFF5-4248-4679-9C59-9533998936B3}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{DB75EFF5-4248-4679-9C59-9533998936B3}.Debug|x64.Build.0 = Debug|Any CPU
+		{DB75EFF5-4248-4679-9C59-9533998936B3}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{DB75EFF5-4248-4679-9C59-9533998936B3}.Debug|x86.Build.0 = Debug|Any CPU
 		{DB75EFF5-4248-4679-9C59-9533998936B3}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{DB75EFF5-4248-4679-9C59-9533998936B3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DB75EFF5-4248-4679-9C59-9533998936B3}.Release|x64.ActiveCfg = Release|Any CPU
+		{DB75EFF5-4248-4679-9C59-9533998936B3}.Release|x64.Build.0 = Release|Any CPU
+		{DB75EFF5-4248-4679-9C59-9533998936B3}.Release|x86.ActiveCfg = Release|Any CPU
+		{DB75EFF5-4248-4679-9C59-9533998936B3}.Release|x86.Build.0 = Release|Any CPU
 		{B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Debug|x64.Build.0 = Debug|Any CPU
+		{B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Debug|x86.Build.0 = Debug|Any CPU
 		{B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Release|x64.ActiveCfg = Release|Any CPU
+		{B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Release|x64.Build.0 = Release|Any CPU
+		{B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Release|x86.ActiveCfg = Release|Any CPU
+		{B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Release|x86.Build.0 = Release|Any CPU
 		{DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Debug|x64.Build.0 = Debug|Any CPU
+		{DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Debug|x86.Build.0 = Debug|Any CPU
 		{DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Release|x64.ActiveCfg = Release|Any CPU
+		{DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Release|x64.Build.0 = Release|Any CPU
+		{DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Release|x86.ActiveCfg = Release|Any CPU
+		{DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Release|x86.Build.0 = Release|Any CPU
 		{B9194474-83A7-47E6-B5E6-6CE360B1189B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{B9194474-83A7-47E6-B5E6-6CE360B1189B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B9194474-83A7-47E6-B5E6-6CE360B1189B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B9194474-83A7-47E6-B5E6-6CE360B1189B}.Debug|x64.Build.0 = Debug|Any CPU
+		{B9194474-83A7-47E6-B5E6-6CE360B1189B}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B9194474-83A7-47E6-B5E6-6CE360B1189B}.Debug|x86.Build.0 = Debug|Any CPU
 		{B9194474-83A7-47E6-B5E6-6CE360B1189B}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B9194474-83A7-47E6-B5E6-6CE360B1189B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B9194474-83A7-47E6-B5E6-6CE360B1189B}.Release|x64.ActiveCfg = Release|Any CPU
+		{B9194474-83A7-47E6-B5E6-6CE360B1189B}.Release|x64.Build.0 = Release|Any CPU
+		{B9194474-83A7-47E6-B5E6-6CE360B1189B}.Release|x86.ActiveCfg = Release|Any CPU
+		{B9194474-83A7-47E6-B5E6-6CE360B1189B}.Release|x86.Build.0 = Release|Any CPU
+		{C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Debug|x64.Build.0 = Debug|Any CPU
+		{C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Debug|x86.Build.0 = Debug|Any CPU
+		{C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Release|x64.ActiveCfg = Release|Any CPU
+		{C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Release|x64.Build.0 = Release|Any CPU
+		{C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Release|x86.ActiveCfg = Release|Any CPU
+		{C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{C4A8E28E-70B0-4184-B62B-7286CB2F5756} = {FB55729C-1952-4D20-BFE7-C3202B160A0B}
 		{46BA508A-D22E-4F76-AD27-68AC62725952} = {FB55729C-1952-4D20-BFE7-C3202B160A0B}
+		{B9194474-83A7-47E6-B5E6-6CE360B1189B} = {F84B0BEF-53D7-43AD-93AB-2025127B6D84}
+		{B49D3C2E-6C4E-45B3-A645-592994B7B94D} = {F84B0BEF-53D7-43AD-93AB-2025127B6D84}
+		{C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54} = {0D367332-B489-41A1-AD22-3F8D07F627C1}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {D82FE2B4-7A75-444B-AB90-DC50F82D89A8}

build/Directory.Build.props

@@ -3,6 +3,6 @@
     <LangVersion>latest</LangVersion>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     <SystemTextJsonVersion>8.0.5</SystemTextJsonVersion>
-    <OnigwrapVersion>1.0.8</OnigwrapVersion>
+    <OnigwrapVersion>1.0.9</OnigwrapVersion>
   </PropertyGroup>
 </Project>

src/TextMateSharp.Benchmarks/BigFileTokenizationBenchmark.cs

@@ -0,0 +1,73 @@
+using System;
+using System.IO;
+
+using BenchmarkDotNet.Attributes;
+
+using TextMateSharp.Grammars;
+
+namespace TextMateSharp.Benchmarks
+{
+    [MemoryDiagnoser]
+    public class BigFileTokenizationBenchmark
+    {
+        private IGrammar _grammar = null!;
+        private string[] _lines = null!;
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            // Walk up directories to find the solution root
+            string? dir = AppDomain.CurrentDomain.BaseDirectory;
+            string bigFilePath = "";
+
+            while (dir != null)
+            {
+                string candidate = Path.Combine(dir, "src", "TextMateSharp.Demo", 
+                    "testdata", "samplefiles", "bigfile.cs");
+                if (File.Exists(candidate))
+                {
+                    bigFilePath = candidate;
+                    break;
+                }
+                dir = Path.GetDirectoryName(dir);
+            }
+
+            if (string.IsNullOrEmpty(bigFilePath) || !File.Exists(bigFilePath))
+            {
+                throw new FileNotFoundException(
+                    "Could not find bigfile.cs. Make sure you're running from the TextMateSharp solution directory.");
+            }
+
+
+            // Load the file into memory
+            _lines = File.ReadAllLines(bigFilePath);
+            Console.WriteLine($"Loaded {_lines.Length} lines from bigfile.cs");
+
+            // Load the C# grammar
+            RegistryOptions options = new RegistryOptions(ThemeName.DarkPlus);
+            Registry.Registry registry = new Registry.Registry(options);
+            _grammar = registry.LoadGrammar("source.cs");
+
+            if (_grammar == null)
+            {
+                throw new InvalidOperationException("Failed to load C# grammar");
+            }
+        }
+
+        [Benchmark]
+        public int TokenizeAllLines()
+        {
+            int totalTokens = 0;
+            IStateStack? ruleStack = null;
+
+            for (int i = 0; i < _lines.Length; i++)
+            {
+                ITokenizeLineResult result = _grammar.TokenizeLine(_lines[i], ruleStack, TimeSpan.MaxValue);
+                ruleStack = result.RuleStack;
+                totalTokens += result.Tokens.Length;
+            }
+
+            return totalTokens;
+        }
+    }
+}

src/TextMateSharp.Benchmarks/Program.cs

@@ -0,0 +1,12 @@
+using BenchmarkDotNet.Running;
+
+namespace TextMateSharp.Benchmarks
+{
+    public class Program
+    {
+        public static void Main(string[] args)
+        {
+            BenchmarkRunner.Run<BigFileTokenizationBenchmark>();
+        }
+    }
+}

src/TextMateSharp.Benchmarks/TextMateSharp.Benchmarks.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <IsPackable>False</IsPackable>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.14.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\TextMateSharp.Grammars\TextMateSharp.Grammars.csproj" />
+    <ProjectReference Include="..\TextMateSharp\TextMateSharp.csproj" />
+  </ItemGroup>
+
+</Project>

src/TextMateSharp.Demo/TextMateSharp.Demo.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <IsPackable>False</IsPackable>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/TextMateSharp.Grammars.Tests/TextMateSharp.Grammars.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <IsPackable>false</IsPackable>
     <SignAssembly>True</SignAssembly>
     <AssemblyOriginatorKeyFile>..\TextMateSharp.snk</AssemblyOriginatorKeyFile>

src/TextMateSharp.Tests/Grammar/LineTextTests.cs

@@ -0,0 +1,153 @@
+using System;
+
+using NUnit.Framework;
+
+using TextMateSharp.Grammars;
+
+namespace TextMateSharp.Tests.Grammar
+{
+    [TestFixture]
+    public class LineTextTests
+    {
+        [Test]
+        public void Constructor_WithString_ShouldStoreText()
+        {
+            LineText lineText = new LineText("hello world");
+
+            Assert.AreEqual(11, lineText.Length);
+            Assert.AreEqual("hello world", lineText.ToString());
+        }
+
+        [Test]
+        public void Constructor_WithNullString_ShouldBeEmpty()
+        {
+            LineText lineText = new LineText((string)null);
+
+            Assert.IsTrue(lineText.IsEmpty);
+            Assert.AreEqual(0, lineText.Length);
+        }
+
+        [Test]
+        public void Constructor_WithReadOnlyMemory_ShouldStoreText()
+        {
+            ReadOnlyMemory<char> memory = "hello world".AsMemory();
+            LineText lineText = new LineText(memory);
+
+            Assert.AreEqual(11, lineText.Length);
+            Assert.AreEqual("hello world", lineText.ToString());
+        }
+
+        [Test]
+        public void Constructor_WithEmptyMemory_ShouldBeEmpty()
+        {
+            LineText lineText = new LineText(ReadOnlyMemory<char>.Empty);
+
+            Assert.IsTrue(lineText.IsEmpty);
+            Assert.AreEqual(0, lineText.Length);
+        }
+
+        [Test]
+        public void ImplicitConversion_FromString_ShouldWork()
+        {
+            LineText lineText = "test string";
+
+            Assert.AreEqual("test string", lineText.ToString());
+            Assert.AreEqual(11, lineText.Length);
+        }
+
+        [Test]
+        public void ImplicitConversion_FromReadOnlyMemory_ShouldWork()
+        {
+            ReadOnlyMemory<char> memory = "test memory".AsMemory();
+            LineText lineText = memory;
+
+            Assert.AreEqual("test memory", lineText.ToString());
+            Assert.AreEqual(11, lineText.Length);
+        }
+
+        [Test]
+        public void ImplicitConversion_ToReadOnlyMemory_ShouldWork()
+        {
+            LineText lineText = "test";
+            ReadOnlyMemory<char> memory = lineText;
+
+            Assert.AreEqual(4, memory.Length);
+            Assert.AreEqual("test", memory.Span.ToString());
+        }
+
+        [Test]
+        public void Memory_Property_ShouldReturnUnderlyingMemory()
+        {
+            LineText lineText = "hello";
+
+            ReadOnlyMemory<char> memory = lineText.Memory;
+
+            Assert.AreEqual(5, memory.Length);
+            Assert.AreEqual('h', memory.Span[0]);
+            Assert.AreEqual('o', memory.Span[4]);
+        }
+
+        [Test]
+        public void IsEmpty_WithEmptyString_ShouldReturnTrue()
+        {
+            LineText lineText = "";
+
+            Assert.IsTrue(lineText.IsEmpty);
+        }
+
+        [Test]
+        public void IsEmpty_WithNonEmptyString_ShouldReturnFalse()
+        {
+            LineText lineText = "x";
+
+            Assert.IsFalse(lineText.IsEmpty);
+        }
+
+        [Test]
+        public void Default_LineText_ShouldBeEmpty()
+        {
+            LineText lineText = default;
+
+            Assert.IsTrue(lineText.IsEmpty);
+            Assert.AreEqual(0, lineText.Length);
+        }
+
+        [Test]
+        public void ToString_ShouldReturnStringRepresentation()
+        {
+            LineText lineText = "hello world";
+
+            Assert.AreEqual("hello world", lineText.ToString());
+        }
+
+        [Test]
+        public void SlicedMemory_ShouldWorkCorrectly()
+        {
+            char[] buffer = "hello world".ToCharArray();
+            ReadOnlyMemory<char> sliced = buffer.AsMemory().Slice(6, 5);
+            LineText lineText = sliced;
+
+            Assert.AreEqual("world", lineText.ToString());
+            Assert.AreEqual(5, lineText.Length);
+        }
+
+        [Test]
+        public void UnicodeText_ShouldBeHandledCorrectly()
+        {
+            LineText lineText = "안녕하세요";
+
+            Assert.AreEqual(5, lineText.Length);
+            Assert.AreEqual("안녕하세요", lineText.ToString());
+        }
+
+        [Test]
+        public void CharArrayMemory_ShouldWorkWithLineText()
+        {
+            char[] buffer = new char[] { 'a', 'b', 'c', 'd', 'e' };
+            LineText lineText = (ReadOnlyMemory<char>)buffer.AsMemory();
+
+            Assert.AreEqual(5, lineText.Length);
+            Assert.AreEqual("abcde", lineText.ToString());
+        }
+    }
+}