diff --git a/.gitignore b/.gitignore index 75f0aa7..e773ddd 100644 --- a/.gitignore +++ b/.gitignore @@ -372,3 +372,7 @@ src/.vscode/launch.json .idea/.idea.TextMateSharp.dir/.idea/indexLayout.xml .idea/.idea.TextMateSharp.dir/.idea/vcs.xml .idea/.idea.TextMateSharp/.idea/riderMarkupCache.xml +.idea/.idea.TextMateSharp/.idea/copilot.data.migration.agent.xml +.idea/.idea.TextMateSharp/.idea/copilot.data.migration.ask.xml +.idea/.idea.TextMateSharp/.idea/copilot.data.migration.ask2agent.xml +.idea/.idea.TextMateSharp/.idea/copilot.data.migration.edit.xml diff --git a/TextMateSharp.sln b/TextMateSharp.sln index d23b39d..f6b02e7 100644 --- a/TextMateSharp.sln +++ b/TextMateSharp.sln @@ -31,32 +31,94 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Workflows", "Workflows", "{ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TextMateSharp.Grammars.Tests", "src\TextMateSharp.Grammars.Tests\TextMateSharp.Grammars.Tests.csproj", "{B9194474-83A7-47E6-B5E6-6CE360B1189B}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TextMateSharp.Benchmarks", "src\TextMateSharp.Benchmarks\TextMateSharp.Benchmarks.csproj", "{C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Benchmarks", "Benchmarks", "{0D367332-B489-41A1-AD22-3F8D07F627C1}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{F84B0BEF-53D7-43AD-93AB-2025127B6D84}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {664F185F-961B-496E-9159-3CC8F05DBBE5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {664F185F-961B-496E-9159-3CC8F05DBBE5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {664F185F-961B-496E-9159-3CC8F05DBBE5}.Debug|x64.ActiveCfg = Debug|Any CPU + {664F185F-961B-496E-9159-3CC8F05DBBE5}.Debug|x64.Build.0 = Debug|Any CPU + {664F185F-961B-496E-9159-3CC8F05DBBE5}.Debug|x86.ActiveCfg = Debug|Any CPU + {664F185F-961B-496E-9159-3CC8F05DBBE5}.Debug|x86.Build.0 = Debug|Any CPU {664F185F-961B-496E-9159-3CC8F05DBBE5}.Release|Any CPU.ActiveCfg = Release|Any CPU {664F185F-961B-496E-9159-3CC8F05DBBE5}.Release|Any CPU.Build.0 = Release|Any CPU + {664F185F-961B-496E-9159-3CC8F05DBBE5}.Release|x64.ActiveCfg = Release|Any CPU + {664F185F-961B-496E-9159-3CC8F05DBBE5}.Release|x64.Build.0 = Release|Any CPU + {664F185F-961B-496E-9159-3CC8F05DBBE5}.Release|x86.ActiveCfg = Release|Any CPU + {664F185F-961B-496E-9159-3CC8F05DBBE5}.Release|x86.Build.0 = Release|Any CPU {DB75EFF5-4248-4679-9C59-9533998936B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {DB75EFF5-4248-4679-9C59-9533998936B3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DB75EFF5-4248-4679-9C59-9533998936B3}.Debug|x64.ActiveCfg = Debug|Any CPU + {DB75EFF5-4248-4679-9C59-9533998936B3}.Debug|x64.Build.0 = Debug|Any CPU + {DB75EFF5-4248-4679-9C59-9533998936B3}.Debug|x86.ActiveCfg = Debug|Any CPU + {DB75EFF5-4248-4679-9C59-9533998936B3}.Debug|x86.Build.0 = Debug|Any CPU {DB75EFF5-4248-4679-9C59-9533998936B3}.Release|Any CPU.ActiveCfg = Release|Any CPU {DB75EFF5-4248-4679-9C59-9533998936B3}.Release|Any CPU.Build.0 = Release|Any CPU + {DB75EFF5-4248-4679-9C59-9533998936B3}.Release|x64.ActiveCfg = Release|Any CPU + {DB75EFF5-4248-4679-9C59-9533998936B3}.Release|x64.Build.0 = Release|Any CPU + {DB75EFF5-4248-4679-9C59-9533998936B3}.Release|x86.ActiveCfg = Release|Any CPU + {DB75EFF5-4248-4679-9C59-9533998936B3}.Release|x86.Build.0 = Release|Any CPU {B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Debug|x64.ActiveCfg = Debug|Any CPU + {B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Debug|x64.Build.0 = Debug|Any CPU + {B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Debug|x86.ActiveCfg = Debug|Any CPU + {B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Debug|x86.Build.0 = Debug|Any CPU {B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Release|Any CPU.ActiveCfg = Release|Any CPU {B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Release|Any CPU.Build.0 = Release|Any CPU + {B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Release|x64.ActiveCfg = Release|Any CPU + {B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Release|x64.Build.0 = Release|Any CPU + {B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Release|x86.ActiveCfg = Release|Any CPU + {B49D3C2E-6C4E-45B3-A645-592994B7B94D}.Release|x86.Build.0 = Release|Any CPU {DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Debug|x64.ActiveCfg = Debug|Any CPU + {DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Debug|x64.Build.0 = Debug|Any CPU + {DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Debug|x86.ActiveCfg = Debug|Any CPU + {DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Debug|x86.Build.0 = Debug|Any CPU {DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Release|Any CPU.ActiveCfg = Release|Any CPU {DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Release|Any CPU.Build.0 = Release|Any CPU + {DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Release|x64.ActiveCfg = Release|Any CPU + {DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Release|x64.Build.0 = Release|Any CPU + {DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Release|x86.ActiveCfg = Release|Any CPU + {DDB3D93D-BFAA-4CE6-B98D-74497DDE0D62}.Release|x86.Build.0 = Release|Any CPU {B9194474-83A7-47E6-B5E6-6CE360B1189B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B9194474-83A7-47E6-B5E6-6CE360B1189B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B9194474-83A7-47E6-B5E6-6CE360B1189B}.Debug|x64.ActiveCfg = Debug|Any CPU + {B9194474-83A7-47E6-B5E6-6CE360B1189B}.Debug|x64.Build.0 = Debug|Any CPU + {B9194474-83A7-47E6-B5E6-6CE360B1189B}.Debug|x86.ActiveCfg = Debug|Any CPU + {B9194474-83A7-47E6-B5E6-6CE360B1189B}.Debug|x86.Build.0 = Debug|Any CPU {B9194474-83A7-47E6-B5E6-6CE360B1189B}.Release|Any CPU.ActiveCfg = Release|Any CPU {B9194474-83A7-47E6-B5E6-6CE360B1189B}.Release|Any CPU.Build.0 = Release|Any CPU + {B9194474-83A7-47E6-B5E6-6CE360B1189B}.Release|x64.ActiveCfg = Release|Any CPU + {B9194474-83A7-47E6-B5E6-6CE360B1189B}.Release|x64.Build.0 = Release|Any CPU + {B9194474-83A7-47E6-B5E6-6CE360B1189B}.Release|x86.ActiveCfg = Release|Any CPU + {B9194474-83A7-47E6-B5E6-6CE360B1189B}.Release|x86.Build.0 = Release|Any CPU + {C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Debug|x64.ActiveCfg = Debug|Any CPU + {C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Debug|x64.Build.0 = Debug|Any CPU + {C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Debug|x86.ActiveCfg = Debug|Any CPU + {C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Debug|x86.Build.0 = Debug|Any CPU + {C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Release|Any CPU.Build.0 = Release|Any CPU + {C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Release|x64.ActiveCfg = Release|Any CPU + {C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Release|x64.Build.0 = Release|Any CPU + {C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Release|x86.ActiveCfg = Release|Any CPU + {C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -64,6 +126,9 @@ Global GlobalSection(NestedProjects) = preSolution {C4A8E28E-70B0-4184-B62B-7286CB2F5756} = {FB55729C-1952-4D20-BFE7-C3202B160A0B} {46BA508A-D22E-4F76-AD27-68AC62725952} = {FB55729C-1952-4D20-BFE7-C3202B160A0B} + {B9194474-83A7-47E6-B5E6-6CE360B1189B} = {F84B0BEF-53D7-43AD-93AB-2025127B6D84} + {B49D3C2E-6C4E-45B3-A645-592994B7B94D} = {F84B0BEF-53D7-43AD-93AB-2025127B6D84} + {C1F336BA-0CAD-4A76-8C83-E0CA2DB9DA54} = {0D367332-B489-41A1-AD22-3F8D07F627C1} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D82FE2B4-7A75-444B-AB90-DC50F82D89A8} diff --git a/build/Directory.Build.props b/build/Directory.Build.props index d2bc446..7d9da74 100644 --- a/build/Directory.Build.props +++ b/build/Directory.Build.props @@ -3,6 +3,6 @@ latest true 8.0.5 - 1.0.8 + 1.0.9 diff --git a/src/TextMateSharp.Benchmarks/BigFileTokenizationBenchmark.cs b/src/TextMateSharp.Benchmarks/BigFileTokenizationBenchmark.cs new file mode 100644 index 0000000..c77c547 --- /dev/null +++ b/src/TextMateSharp.Benchmarks/BigFileTokenizationBenchmark.cs @@ -0,0 +1,73 @@ +using System; +using System.IO; + +using BenchmarkDotNet.Attributes; + +using TextMateSharp.Grammars; + +namespace TextMateSharp.Benchmarks +{ + [MemoryDiagnoser] + public class BigFileTokenizationBenchmark + { + private IGrammar _grammar = null!; + private string[] _lines = null!; + + [GlobalSetup] + public void Setup() + { + // Walk up directories to find the solution root + string? dir = AppDomain.CurrentDomain.BaseDirectory; + string bigFilePath = ""; + + while (dir != null) + { + string candidate = Path.Combine(dir, "src", "TextMateSharp.Demo", + "testdata", "samplefiles", "bigfile.cs"); + if (File.Exists(candidate)) + { + bigFilePath = candidate; + break; + } + dir = Path.GetDirectoryName(dir); + } + + if (string.IsNullOrEmpty(bigFilePath) || !File.Exists(bigFilePath)) + { + throw new FileNotFoundException( + "Could not find bigfile.cs. Make sure you're running from the TextMateSharp solution directory."); + } + + + // Load the file into memory + _lines = File.ReadAllLines(bigFilePath); + Console.WriteLine($"Loaded {_lines.Length} lines from bigfile.cs"); + + // Load the C# grammar + RegistryOptions options = new RegistryOptions(ThemeName.DarkPlus); + Registry.Registry registry = new Registry.Registry(options); + _grammar = registry.LoadGrammar("source.cs"); + + if (_grammar == null) + { + throw new InvalidOperationException("Failed to load C# grammar"); + } + } + + [Benchmark] + public int TokenizeAllLines() + { + int totalTokens = 0; + IStateStack? ruleStack = null; + + for (int i = 0; i < _lines.Length; i++) + { + ITokenizeLineResult result = _grammar.TokenizeLine(_lines[i], ruleStack, TimeSpan.MaxValue); + ruleStack = result.RuleStack; + totalTokens += result.Tokens.Length; + } + + return totalTokens; + } + } +} diff --git a/src/TextMateSharp.Benchmarks/Program.cs b/src/TextMateSharp.Benchmarks/Program.cs new file mode 100644 index 0000000..38c7edf --- /dev/null +++ b/src/TextMateSharp.Benchmarks/Program.cs @@ -0,0 +1,12 @@ +using BenchmarkDotNet.Running; + +namespace TextMateSharp.Benchmarks +{ + public class Program + { + public static void Main(string[] args) + { + BenchmarkRunner.Run(); + } + } +} diff --git a/src/TextMateSharp.Benchmarks/TextMateSharp.Benchmarks.csproj b/src/TextMateSharp.Benchmarks/TextMateSharp.Benchmarks.csproj new file mode 100644 index 0000000..403cf07 --- /dev/null +++ b/src/TextMateSharp.Benchmarks/TextMateSharp.Benchmarks.csproj @@ -0,0 +1,19 @@ + + + + Exe + net8.0 + False + enable + + + + + + + + + + + + diff --git a/src/TextMateSharp.Demo/TextMateSharp.Demo.csproj b/src/TextMateSharp.Demo/TextMateSharp.Demo.csproj index 1edf6a8..f648298 100644 --- a/src/TextMateSharp.Demo/TextMateSharp.Demo.csproj +++ b/src/TextMateSharp.Demo/TextMateSharp.Demo.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 False enable diff --git a/src/TextMateSharp.Grammars.Tests/TextMateSharp.Grammars.Tests.csproj b/src/TextMateSharp.Grammars.Tests/TextMateSharp.Grammars.Tests.csproj index 449f714..3db4421 100644 --- a/src/TextMateSharp.Grammars.Tests/TextMateSharp.Grammars.Tests.csproj +++ b/src/TextMateSharp.Grammars.Tests/TextMateSharp.Grammars.Tests.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false True ..\TextMateSharp.snk diff --git a/src/TextMateSharp.Tests/Grammar/LineTextTests.cs b/src/TextMateSharp.Tests/Grammar/LineTextTests.cs new file mode 100644 index 0000000..23f2f0e --- /dev/null +++ b/src/TextMateSharp.Tests/Grammar/LineTextTests.cs @@ -0,0 +1,153 @@ +using System; + +using NUnit.Framework; + +using TextMateSharp.Grammars; + +namespace TextMateSharp.Tests.Grammar +{ + [TestFixture] + public class LineTextTests + { + [Test] + public void Constructor_WithString_ShouldStoreText() + { + LineText lineText = new LineText("hello world"); + + Assert.AreEqual(11, lineText.Length); + Assert.AreEqual("hello world", lineText.ToString()); + } + + [Test] + public void Constructor_WithNullString_ShouldBeEmpty() + { + LineText lineText = new LineText((string)null); + + Assert.IsTrue(lineText.IsEmpty); + Assert.AreEqual(0, lineText.Length); + } + + [Test] + public void Constructor_WithReadOnlyMemory_ShouldStoreText() + { + ReadOnlyMemory memory = "hello world".AsMemory(); + LineText lineText = new LineText(memory); + + Assert.AreEqual(11, lineText.Length); + Assert.AreEqual("hello world", lineText.ToString()); + } + + [Test] + public void Constructor_WithEmptyMemory_ShouldBeEmpty() + { + LineText lineText = new LineText(ReadOnlyMemory.Empty); + + Assert.IsTrue(lineText.IsEmpty); + Assert.AreEqual(0, lineText.Length); + } + + [Test] + public void ImplicitConversion_FromString_ShouldWork() + { + LineText lineText = "test string"; + + Assert.AreEqual("test string", lineText.ToString()); + Assert.AreEqual(11, lineText.Length); + } + + [Test] + public void ImplicitConversion_FromReadOnlyMemory_ShouldWork() + { + ReadOnlyMemory memory = "test memory".AsMemory(); + LineText lineText = memory; + + Assert.AreEqual("test memory", lineText.ToString()); + Assert.AreEqual(11, lineText.Length); + } + + [Test] + public void ImplicitConversion_ToReadOnlyMemory_ShouldWork() + { + LineText lineText = "test"; + ReadOnlyMemory memory = lineText; + + Assert.AreEqual(4, memory.Length); + Assert.AreEqual("test", memory.Span.ToString()); + } + + [Test] + public void Memory_Property_ShouldReturnUnderlyingMemory() + { + LineText lineText = "hello"; + + ReadOnlyMemory memory = lineText.Memory; + + Assert.AreEqual(5, memory.Length); + Assert.AreEqual('h', memory.Span[0]); + Assert.AreEqual('o', memory.Span[4]); + } + + [Test] + public void IsEmpty_WithEmptyString_ShouldReturnTrue() + { + LineText lineText = ""; + + Assert.IsTrue(lineText.IsEmpty); + } + + [Test] + public void IsEmpty_WithNonEmptyString_ShouldReturnFalse() + { + LineText lineText = "x"; + + Assert.IsFalse(lineText.IsEmpty); + } + + [Test] + public void Default_LineText_ShouldBeEmpty() + { + LineText lineText = default; + + Assert.IsTrue(lineText.IsEmpty); + Assert.AreEqual(0, lineText.Length); + } + + [Test] + public void ToString_ShouldReturnStringRepresentation() + { + LineText lineText = "hello world"; + + Assert.AreEqual("hello world", lineText.ToString()); + } + + [Test] + public void SlicedMemory_ShouldWorkCorrectly() + { + char[] buffer = "hello world".ToCharArray(); + ReadOnlyMemory sliced = buffer.AsMemory().Slice(6, 5); + LineText lineText = sliced; + + Assert.AreEqual("world", lineText.ToString()); + Assert.AreEqual(5, lineText.Length); + } + + [Test] + public void UnicodeText_ShouldBeHandledCorrectly() + { + LineText lineText = "안녕하세요"; + + Assert.AreEqual(5, lineText.Length); + Assert.AreEqual("안녕하세요", lineText.ToString()); + } + + [Test] + public void CharArrayMemory_ShouldWorkWithLineText() + { + char[] buffer = new char[] { 'a', 'b', 'c', 'd', 'e' }; + LineText lineText = (ReadOnlyMemory)buffer.AsMemory(); + + Assert.AreEqual(5, lineText.Length); + Assert.AreEqual("abcde", lineText.ToString()); + } + } +} diff --git a/src/TextMateSharp.Tests/Model/TMModelTests.cs b/src/TextMateSharp.Tests/Model/TMModelTests.cs index e02e1c8..5e6f901 100644 --- a/src/TextMateSharp.Tests/Model/TMModelTests.cs +++ b/src/TextMateSharp.Tests/Model/TMModelTests.cs @@ -125,7 +125,7 @@ public override int GetLineLength(int lineIndex) { return _lines[lineIndex].Length; } - public override string GetLineText(int lineIndex) + public override LineText GetLineText(int lineIndex) { return _lines[lineIndex]; } diff --git a/src/TextMateSharp.Tests/TextMateSharp.Tests.csproj b/src/TextMateSharp.Tests/TextMateSharp.Tests.csproj index ad51fbf..61499ce 100644 --- a/src/TextMateSharp.Tests/TextMateSharp.Tests.csproj +++ b/src/TextMateSharp.Tests/TextMateSharp.Tests.csproj @@ -1,7 +1,7 @@  - net6.0 + net8.0 False True ..\TextMateSharp.snk diff --git a/src/TextMateSharp/Grammar/IGrammar.cs b/src/TextMateSharp/Grammar/IGrammar.cs index e82df51..05e0d03 100644 --- a/src/TextMateSharp/Grammar/IGrammar.cs +++ b/src/TextMateSharp/Grammar/IGrammar.cs @@ -9,9 +9,9 @@ public interface IGrammar string GetName(); string GetScopeName(); ICollection GetFileTypes(); - ITokenizeLineResult TokenizeLine(string lineText); - ITokenizeLineResult TokenizeLine(string lineText, IStateStack prevState, TimeSpan timeLimit); - ITokenizeLineResult2 TokenizeLine2(string lineText); - ITokenizeLineResult2 TokenizeLine2(string lineText, IStateStack prevState, TimeSpan timeLimit); + ITokenizeLineResult TokenizeLine(LineText lineText); + ITokenizeLineResult TokenizeLine(LineText lineText, IStateStack prevState, TimeSpan timeLimit); + ITokenizeLineResult2 TokenizeLine2(LineText lineText); + ITokenizeLineResult2 TokenizeLine2(LineText lineText, IStateStack prevState, TimeSpan timeLimit); } } \ No newline at end of file diff --git a/src/TextMateSharp/Grammar/LineText.cs b/src/TextMateSharp/Grammar/LineText.cs new file mode 100644 index 0000000..e78c1c0 --- /dev/null +++ b/src/TextMateSharp/Grammar/LineText.cs @@ -0,0 +1,33 @@ +using System; + +namespace TextMateSharp.Grammars +{ + public readonly struct LineText + { + private readonly ReadOnlyMemory _memory; + + public LineText(ReadOnlyMemory memory) + { + _memory = memory; + } + + public LineText(string text) + { + _memory = text?.AsMemory() ?? ReadOnlyMemory.Empty; + } + + public ReadOnlyMemory Memory => _memory; + + public int Length => _memory.Length; + + public bool IsEmpty => _memory.IsEmpty; + + public static implicit operator LineText(string text) => new LineText(text); + + public static implicit operator LineText(ReadOnlyMemory memory) => new LineText(memory); + + public static implicit operator ReadOnlyMemory(LineText lineText) => lineText._memory; + + public override string ToString() => _memory.Span.ToString(); + } +} diff --git a/src/TextMateSharp/Internal/Grammars/AttributedScopeStack.cs b/src/TextMateSharp/Internal/Grammars/AttributedScopeStack.cs index 3d869ab..c79ffe8 100644 --- a/src/TextMateSharp/Internal/Grammars/AttributedScopeStack.cs +++ b/src/TextMateSharp/Internal/Grammars/AttributedScopeStack.cs @@ -10,6 +10,7 @@ public class AttributedScopeStack public AttributedScopeStack Parent { get; private set; } public string ScopePath { get; private set; } public int TokenAttributes { get; private set; } + private List _cachedScopeNames; public AttributedScopeStack(AttributedScopeStack parent, string scopePath, int tokenAttributes) { @@ -157,13 +158,18 @@ private static AttributedScopeStack Push(AttributedScopeStack target, Grammar gr { foreach (string scope in scopes) { - BasicScopeAttributes rawMetadata = grammar.GetMetadataForScope(scope); - int metadata = AttributedScopeStack.MergeAttributes(target.TokenAttributes, target, rawMetadata); - target = new AttributedScopeStack(target, scope, metadata); + target = PushSingleScope(target, grammar, scope); } return target; } + private static AttributedScopeStack PushSingleScope(AttributedScopeStack target, Grammar grammar, string scope) + { + BasicScopeAttributes rawMetadata = grammar.GetMetadataForScope(scope); + int metadata = AttributedScopeStack.MergeAttributes(target.TokenAttributes, target, rawMetadata); + return new AttributedScopeStack(target, scope, metadata); + } + public AttributedScopeStack PushAtributed(string scopePath, Grammar grammar) { if (scopePath == null) @@ -175,13 +181,17 @@ public AttributedScopeStack PushAtributed(string scopePath, Grammar grammar) // there are multiple scopes to push return Push(this, grammar, new List(scopePath.Split(new[] {" "}, StringSplitOptions.None))); } - // there is a single scope to push - return Push(this, grammar, new List() { scopePath }); + // there is a single scope to push - avoid List allocation + return PushSingleScope(this, grammar, scopePath); } public List GetScopeNames() { - return AttributedScopeStack.GenerateScopes(this); + if (_cachedScopeNames == null) + { + _cachedScopeNames = GenerateScopes(this); + } + return _cachedScopeNames; } private static List GenerateScopes(AttributedScopeStack scopesList) diff --git a/src/TextMateSharp/Internal/Grammars/Grammar.cs b/src/TextMateSharp/Internal/Grammars/Grammar.cs index 8700f84..ff277ea 100644 --- a/src/TextMateSharp/Internal/Grammars/Grammar.cs +++ b/src/TextMateSharp/Internal/Grammars/Grammar.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; using System.Collections.Generic; using TextMateSharp.Grammars; @@ -200,27 +201,27 @@ private IRawGrammar Clone(IRawGrammar grammar) return (IRawGrammar)((Raw)grammar).Clone(); } - public ITokenizeLineResult TokenizeLine(string lineText) + public ITokenizeLineResult TokenizeLine(LineText lineText) { return TokenizeLine(lineText, null, TimeSpan.MaxValue); } - public ITokenizeLineResult TokenizeLine(string lineText, IStateStack prevState, TimeSpan timeLimit) + public ITokenizeLineResult TokenizeLine(LineText lineText, IStateStack prevState, TimeSpan timeLimit) { - return (ITokenizeLineResult)Tokenize(lineText, (StateStack)prevState, false, timeLimit); + return (ITokenizeLineResult)Tokenize(lineText.Memory, (StateStack)prevState, false, timeLimit); } - public ITokenizeLineResult2 TokenizeLine2(string lineText) + public ITokenizeLineResult2 TokenizeLine2(LineText lineText) { return TokenizeLine2(lineText, null, TimeSpan.MaxValue); } - public ITokenizeLineResult2 TokenizeLine2(string lineText, IStateStack prevState, TimeSpan timeLimit) + public ITokenizeLineResult2 TokenizeLine2(LineText lineText, IStateStack prevState, TimeSpan timeLimit) { - return (ITokenizeLineResult2)Tokenize(lineText, (StateStack)prevState, true, timeLimit); + return (ITokenizeLineResult2)Tokenize(lineText.Memory, (StateStack)prevState, true, timeLimit); } - private object Tokenize(string lineText, StateStack prevState, bool emitBinaryTokens, TimeSpan timeLimit) + private object Tokenize(ReadOnlyMemory lineText, StateStack prevState, bool emitBinaryTokens, TimeSpan timeLimit) { if (this._rootId == null) { @@ -237,7 +238,7 @@ private object Tokenize(string lineText, StateStack prevState, bool emitBinaryTo rawDefaultMetadata.TokenType, null, defaultTheme.fontStyle, defaultTheme.foreground, defaultTheme.background); - string rootScopeName = this.GetRule(this._rootId)?.GetName(null, null); + string rootScopeName = this.GetRule(this._rootId)?.GetName(ReadOnlyMemory.Empty, null); if (rootScopeName == null) return null; BasicScopeAttributes rawRootMetadata = this._basicScopeAttributesProvider.GetBasicScopeAttributes(rootScopeName); @@ -253,23 +254,47 @@ private object Tokenize(string lineText, StateStack prevState, bool emitBinaryTo prevState.Reset(); } - if (string.IsNullOrEmpty(lineText) || lineText[lineText.Length - 1] != '\n') - { - // Only add \n if the passed lineText didn't have it. - lineText += '\n'; - } - int lineLength = lineText.Length; - LineTokens lineTokens = new LineTokens(emitBinaryTokens, lineText, _tokenTypeMatchers, _balancedBracketSelectors); - TokenizeStringResult tokenizeResult = LineTokenizer.TokenizeString(this, lineText, isFirstLine, 0, prevState, - lineTokens, true, timeLimit); + // Check if we need to append newline + char[] rentedBuffer = null; + ReadOnlyMemory effectiveLineText; - if (emitBinaryTokens) + try { - return new TokenizeLineResult2(lineTokens.GetBinaryResult(tokenizeResult.Stack, lineLength), + if (lineText.Length == 0 || lineText.Span[lineText.Length - 1] != '\n') + { + // Only add \n if the passed lineText didn't have it. + // Use ArrayPool to avoid per-line allocation + int requiredLength = lineText.Length + 1; + rentedBuffer = ArrayPool.Shared.Rent(requiredLength); + lineText.Span.CopyTo(rentedBuffer); + rentedBuffer[lineText.Length] = '\n'; + effectiveLineText = rentedBuffer.AsMemory(0, requiredLength); + } + else + { + effectiveLineText = lineText; + } + + int lineLength = effectiveLineText.Length; + LineTokens lineTokens = new LineTokens(emitBinaryTokens, effectiveLineText, _tokenTypeMatchers, _balancedBracketSelectors); + TokenizeStringResult tokenizeResult = LineTokenizer.TokenizeString(this, effectiveLineText, isFirstLine, 0, prevState, + lineTokens, true, timeLimit); + + if (emitBinaryTokens) + { + return new TokenizeLineResult2(lineTokens.GetBinaryResult(tokenizeResult.Stack, lineLength), + tokenizeResult.Stack, tokenizeResult.StoppedEarly); + } + return new TokenizeLineResult(lineTokens.GetResult(tokenizeResult.Stack, lineLength), tokenizeResult.Stack, tokenizeResult.StoppedEarly); } - return new TokenizeLineResult(lineTokens.GetResult(tokenizeResult.Stack, lineLength), - tokenizeResult.Stack, tokenizeResult.StoppedEarly); + finally + { + if (rentedBuffer != null) + { + ArrayPool.Shared.Return(rentedBuffer); + } + } } private void GenerateRootId() diff --git a/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs b/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs index 05095e0..c6b4948 100644 --- a/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs +++ b/src/TextMateSharp/Internal/Grammars/LineTokenizer.cs @@ -13,16 +13,19 @@ namespace TextMateSharp.Internal.Grammars class LineTokenizer { private Grammar _grammar; - private string _lineText; + private ReadOnlyMemory _lineText; private bool _isFirstLine; private int _linePos; private StateStack _stack; private LineTokens _lineTokens; private int _anchorPosition = -1; private bool _stop; + private Stopwatch _stopwatch = new Stopwatch(); private int _lineLength; + private readonly List _localStackBuffer = new List(); + private readonly List _whileRulesBuffer = new List(); - public LineTokenizer(Grammar grammar, string lineText, bool isFirstLine, int linePos, StateStack stack, + public LineTokenizer(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, int linePos, StateStack stack, LineTokens lineTokens) { this._grammar = grammar; @@ -48,11 +51,10 @@ public TokenizeStringResult Scan(bool checkWhileConditions, TimeSpan timeLimit) _anchorPosition = whileCheckResult.AnchorPosition; } - var stopWatch = new Stopwatch(); - stopWatch.Start(); + _stopwatch.Restart(); while (!_stop) { - if (stopWatch.Elapsed > timeLimit) + if (_stopwatch.Elapsed > timeLimit) { return new TokenizeStringResult(_stack, true); } @@ -253,7 +255,7 @@ private void ScanNext() } } - private MatchResult MatchRule(Grammar grammar, string lineText, in bool isFirstLine, in int linePos, + private MatchResult MatchRule(Grammar grammar, ReadOnlyMemory lineText, in bool isFirstLine, in int linePos, StateStack stack, in int anchorPosition) { Rule rule = stack.GetRule(grammar); @@ -277,7 +279,7 @@ private MatchResult MatchRule(Grammar grammar, string lineText, in bool isFirstL return null; } - private MatchResult MatchRuleOrInjections(Grammar grammar, string lineText, bool isFirstLine, + private MatchResult MatchRuleOrInjections(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, in int linePos, StateStack stack, in int anchorPosition) { // Look for normal grammar rule @@ -319,7 +321,7 @@ private MatchResult MatchRuleOrInjections(Grammar grammar, string lineText, bool return matchResult; } - private MatchInjectionsResult MatchInjections(List injections, Grammar grammar, string lineText, + private MatchInjectionsResult MatchInjections(List injections, Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, in int linePos, StateStack stack, in int anchorPosition) { // The lower the better @@ -383,7 +385,7 @@ private MatchInjectionsResult MatchInjections(List injections, Gramma return null; } - private void HandleCaptures(Grammar grammar, string lineText, bool isFirstLine, StateStack stack, + private void HandleCaptures(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, StateStack stack, LineTokens lineTokens, List captures, IOnigCaptureIndex[] captureIndices) { if (captures.Count == 0) @@ -392,7 +394,8 @@ private void HandleCaptures(Grammar grammar, string lineText, bool isFirstLine, } int len = Math.Min(captures.Count, captureIndices.Length); - List localStack = new List(); + _localStackBuffer.Clear(); + var localStack = _localStackBuffer; int maxEnd = captureIndices[0].End; IOnigCaptureIndex captureIndex; @@ -457,7 +460,7 @@ private void HandleCaptures(Grammar grammar, string lineText, bool isFirstLine, contentNameScopesList); TokenizeString(grammar, - lineText.SubstringAtIndexes(0, captureIndex.End), + lineText.SliceAtIndexes(0, captureIndex.End), (isFirstLine && captureIndex.Start == 0), captureIndex.Start, stackClone, lineTokens, false, TimeSpan.MaxValue); continue; } @@ -488,11 +491,12 @@ private void HandleCaptures(Grammar grammar, string lineText, bool isFirstLine, * order. If any fails, cut off the entire stack above the failed while * condition. While conditions may also advance the linePosition. */ - private WhileCheckResult CheckWhileConditions(Grammar grammar, string lineText, bool isFirstLine, + private WhileCheckResult CheckWhileConditions(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, int linePos, StateStack stack, LineTokens lineTokens) { int anchorPosition = stack.BeginRuleCapturedEOL ? 0 : -1; - List whileRules = new List(); + _whileRulesBuffer.Clear(); + var whileRules = _whileRulesBuffer; for (StateStack node = stack; node != null; node = node.Pop()) { Rule nodeRule = node.GetRule(grammar); @@ -541,7 +545,7 @@ private WhileCheckResult CheckWhileConditions(Grammar grammar, string lineText, return new WhileCheckResult(stack, linePos, anchorPosition, isFirstLine); } - public static TokenizeStringResult TokenizeString(Grammar grammar, string lineText, bool isFirstLine, int linePos, + public static TokenizeStringResult TokenizeString(Grammar grammar, ReadOnlyMemory lineText, bool isFirstLine, int linePos, StateStack stack, LineTokens lineTokens, bool checkWhileConditions, TimeSpan timeLimit) { return new LineTokenizer(grammar, lineText, isFirstLine, linePos, stack, lineTokens).Scan(checkWhileConditions, timeLimit); diff --git a/src/TextMateSharp/Internal/Grammars/LineTokens.cs b/src/TextMateSharp/Internal/Grammars/LineTokens.cs index d535d46..dcf9f34 100644 --- a/src/TextMateSharp/Internal/Grammars/LineTokens.cs +++ b/src/TextMateSharp/Internal/Grammars/LineTokens.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using TextMateSharp.Grammars; using TextMateSharp.Themes; @@ -6,7 +7,7 @@ namespace TextMateSharp.Internal.Grammars { internal class LineTokens { - private string _lineText; + private ReadOnlyMemory _lineText; // used only if `_emitBinaryTokens` is false. private List _tokens; @@ -23,7 +24,7 @@ internal class LineTokens internal LineTokens( bool emitBinaryTokens, - string lineText, + ReadOnlyMemory lineText, List tokenTypeOverrides, BalancedBracketSelectors balancedBracketSelectors) { diff --git a/src/TextMateSharp/Internal/Rules/BeginEndRule.cs b/src/TextMateSharp/Internal/Rules/BeginEndRule.cs index a925a71..38d0259 100644 --- a/src/TextMateSharp/Internal/Rules/BeginEndRule.cs +++ b/src/TextMateSharp/Internal/Rules/BeginEndRule.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using Onigwrap; @@ -33,7 +34,7 @@ public BeginEndRule(RuleId id, string name, string contentName, string begin, Li _cachedCompiledPatterns = null; } - public string GetEndWithResolvedBackReferences(string lineText, IOnigCaptureIndex[] captureIndices) + public string GetEndWithResolvedBackReferences(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices) { return this._end.ResolveBackReferences(lineText, captureIndices); } diff --git a/src/TextMateSharp/Internal/Rules/BeginWhileRule.cs b/src/TextMateSharp/Internal/Rules/BeginWhileRule.cs index f3d52c5..99290e2 100644 --- a/src/TextMateSharp/Internal/Rules/BeginWhileRule.cs +++ b/src/TextMateSharp/Internal/Rules/BeginWhileRule.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using Onigwrap; @@ -33,7 +34,7 @@ public BeginWhileRule(RuleId id, string name, string contentName, string begin, _cachedCompiledWhilePatterns = null; } - public string getWhileWithResolvedBackReferences(string lineText, IOnigCaptureIndex[] captureIndices) + public string getWhileWithResolvedBackReferences(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices) { return this._while.ResolveBackReferences(lineText, captureIndices); } diff --git a/src/TextMateSharp/Internal/Rules/RegExpSource.cs b/src/TextMateSharp/Internal/Rules/RegExpSource.cs index faef2ca..a3dee4a 100644 --- a/src/TextMateSharp/Internal/Rules/RegExpSource.cs +++ b/src/TextMateSharp/Internal/Rules/RegExpSource.cs @@ -119,7 +119,7 @@ private void HandleAnchors(string regExpSource) } } - public string ResolveBackReferences(string lineText, IOnigCaptureIndex[] captureIndices) + public string ResolveBackReferences(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices) { List capturedValues = new List(); @@ -151,7 +151,7 @@ public string ResolveBackReferences(string lineText, IOnigCaptureIndex[] capture System.Diagnostics.Debug.WriteLine(ex.Message); } - return lineText; + return lineText.Span.ToString(); } private string EscapeRegExpCharacters(string value) diff --git a/src/TextMateSharp/Internal/Rules/Rule.cs b/src/TextMateSharp/Internal/Rules/Rule.cs index 0641e23..a238361 100644 --- a/src/TextMateSharp/Internal/Rules/Rule.cs +++ b/src/TextMateSharp/Internal/Rules/Rule.cs @@ -1,3 +1,4 @@ +using System; using Onigwrap; using TextMateSharp.Internal.Utils; @@ -24,7 +25,7 @@ public Rule(RuleId id, string name, string contentName) _contentNameIsCapturing = RegexSource.HasCaptures(this._contentName); } - public string GetName(string lineText, IOnigCaptureIndex[] captureIndices) + public string GetName(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices) { if (!this._nameIsCapturing) { @@ -34,7 +35,7 @@ public string GetName(string lineText, IOnigCaptureIndex[] captureIndices) return RegexSource.ReplaceCaptures(this._name, lineText, captureIndices); } - public string GetContentName(string lineText, IOnigCaptureIndex[] captureIndices) + public string GetContentName(ReadOnlyMemory lineText, IOnigCaptureIndex[] captureIndices) { if (!this._contentNameIsCapturing) { diff --git a/src/TextMateSharp/Internal/Utils/RegexSource.cs b/src/TextMateSharp/Internal/Utils/RegexSource.cs index b8c26c3..762948b 100644 --- a/src/TextMateSharp/Internal/Utils/RegexSource.cs +++ b/src/TextMateSharp/Internal/Utils/RegexSource.cs @@ -62,13 +62,13 @@ public static bool HasCaptures(string regexSource) return CAPTURING_REGEX_SOURCE.Match(regexSource).Success; } - public static string ReplaceCaptures(string regexSource, string captureSource, IOnigCaptureIndex[] captureIndices) + public static string ReplaceCaptures(string regexSource, ReadOnlyMemory captureSource, IOnigCaptureIndex[] captureIndices) { return CAPTURING_REGEX_SOURCE.Replace( regexSource, m => GetReplacement(m.Value, captureSource, captureIndices)); } - private static string GetReplacement(string match, string captureSource, IOnigCaptureIndex[] captureIndices) + private static string GetReplacement(string match, ReadOnlyMemory captureSource, IOnigCaptureIndex[] captureIndices) { int index = -1; string command = null; @@ -82,7 +82,7 @@ private static string GetReplacement(string match, string captureSource, IOnigCa { index = int.Parse(match.SubstringAtIndexes(1, match.Length)); } - IOnigCaptureIndex capture = captureIndices.Length > index ? captureIndices[index] : null; + IOnigCaptureIndex capture = captureIndices != null && captureIndices.Length > index ? captureIndices[index] : null; if (capture != null) { string result = captureSource.SubstringAtIndexes(capture.Start, capture.End); diff --git a/src/TextMateSharp/Internal/Utils/StringUtils.cs b/src/TextMateSharp/Internal/Utils/StringUtils.cs index 991d221..0047324 100644 --- a/src/TextMateSharp/Internal/Utils/StringUtils.cs +++ b/src/TextMateSharp/Internal/Utils/StringUtils.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Text.RegularExpressions; namespace TextMateSharp.Internal.Utils @@ -15,6 +16,21 @@ internal static string SubstringAtIndexes(this string str, int startIndex, int e return str.Substring(startIndex, endIndex - startIndex); } + internal static ReadOnlyMemory SliceAtIndexes(this ReadOnlyMemory memory, int startIndex, int endIndex) + { + return memory.Slice(startIndex, endIndex - startIndex); + } + + internal static ReadOnlySpan SliceAtIndexes(this ReadOnlySpan span, int startIndex, int endIndex) + { + return span.Slice(startIndex, endIndex - startIndex); + } + + internal static string SubstringAtIndexes(this ReadOnlyMemory memory, int startIndex, int endIndex) + { + return memory.Slice(startIndex, endIndex - startIndex).Span.ToString(); + } + internal static bool IsValidHexColor(string hex) { if (hex == null || hex.Length < 1) diff --git a/src/TextMateSharp/Model/AbstractLineList.cs b/src/TextMateSharp/Model/AbstractLineList.cs index 56d2c57..fb255da 100644 --- a/src/TextMateSharp/Model/AbstractLineList.cs +++ b/src/TextMateSharp/Model/AbstractLineList.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; +using TextMateSharp.Grammars; + namespace TextMateSharp.Model { public abstract class AbstractLineList : IModelLines @@ -94,7 +96,7 @@ public int GetSize() public abstract int GetNumberOfLines(); - public abstract string GetLineText(int lineIndex); + public abstract LineText GetLineText(int lineIndex); public abstract int GetLineLength(int lineIndex); diff --git a/src/TextMateSharp/Model/IModelLines.cs b/src/TextMateSharp/Model/IModelLines.cs index 593592e..80f7568 100644 --- a/src/TextMateSharp/Model/IModelLines.cs +++ b/src/TextMateSharp/Model/IModelLines.cs @@ -1,5 +1,7 @@ using System; +using TextMateSharp.Grammars; + namespace TextMateSharp.Model { public interface IModelLines @@ -11,7 +13,7 @@ public interface IModelLines ModelLine Get(int lineIndex); void ForEach(Action action); int GetNumberOfLines(); - string GetLineText(int lineIndex); + LineText GetLineText(int lineIndex); int GetLineLength(int lineIndex); void Dispose(); } diff --git a/src/TextMateSharp/Model/ITokenizationSupport.cs b/src/TextMateSharp/Model/ITokenizationSupport.cs index 81cb75a..813bb0f 100644 --- a/src/TextMateSharp/Model/ITokenizationSupport.cs +++ b/src/TextMateSharp/Model/ITokenizationSupport.cs @@ -1,14 +1,13 @@ using System; +using TextMateSharp.Grammars; + namespace TextMateSharp.Model { public interface ITokenizationSupport { TMState GetInitialState(); - - LineTokens Tokenize(string line, TMState state, TimeSpan timeLimit); - - LineTokens Tokenize(string line, TMState state, int offsetDelta, int maxLen, TimeSpan timeLimit); - + LineTokens Tokenize(LineText line, TMState state, TimeSpan timeLimit); + LineTokens Tokenize(LineText line, TMState state, int offsetDelta, int maxLen, TimeSpan timeLimit); } } \ No newline at end of file diff --git a/src/TextMateSharp/Model/TMModel.cs b/src/TextMateSharp/Model/TMModel.cs index 5550583..bafe0eb 100644 --- a/src/TextMateSharp/Model/TMModel.cs +++ b/src/TextMateSharp/Model/TMModel.cs @@ -112,6 +112,8 @@ void ThreadWorker(object state) } while (!IsStopped && model._thread != null); } + Stopwatch _stopwatch = new Stopwatch(); + private void RevalidateTokens(int startLine, int? toLineIndexOrNull) { if (model._tokenizer == null) @@ -130,8 +132,7 @@ private void RevalidateTokens(int startLine, int? toLineIndexOrNull) long MAX_ALLOWED_TIME = 5; long currentEstimatedTimeToTokenize = 0; long elapsedTime; - Stopwatch stopwatch = new Stopwatch(); - stopwatch.Start(); + _stopwatch.Restart(); // Tokenize at most 1000 lines. Estimate the tokenization speed per // character and stop when: // - MAX_ALLOWED_TIME is reached @@ -140,7 +141,7 @@ private void RevalidateTokens(int startLine, int? toLineIndexOrNull) int lineIndex = startLine; while (lineIndex <= toLineIndex && lineIndex < model.GetLines().GetNumberOfLines()) { - elapsedTime = stopwatch.ElapsedMilliseconds; + elapsedTime = _stopwatch.ElapsedMilliseconds; if (elapsedTime > MAX_ALLOWED_TIME) { // Stop if MAX_ALLOWED_TIME is reached @@ -190,13 +191,11 @@ public int UpdateTokensInRange(ModelTokensChangedEventBuilder eventBuilder, int int endStateIndex = lineIndex + 1; LineTokens r = null; - string text = null; + LineText text = default; ModelLine modeLine = model._lines.Get(lineIndex); try { text = model._lines.GetLineText(lineIndex); - if (text == null) - continue; // Tokenize only the first X characters r = model._tokenizer.Tokenize(text, modeLine.State, 0, MAX_LEN_TO_TOKENIZE, stopLineTokenizationAfter); } diff --git a/src/TextMateSharp/Model/Tokenizer.cs b/src/TextMateSharp/Model/Tokenizer.cs index 2859b92..d0caeda 100644 --- a/src/TextMateSharp/Model/Tokenizer.cs +++ b/src/TextMateSharp/Model/Tokenizer.cs @@ -21,22 +21,23 @@ public TMState GetInitialState() return new TMState(null, null); } - public LineTokens Tokenize(string line, TMState state, TimeSpan timeLimit) + public LineTokens Tokenize(LineText line, TMState state, TimeSpan timeLimit) { return Tokenize(line, state, 0, 0, timeLimit); } - public LineTokens Tokenize(string line, TMState state, int offsetDelta, int maxLen, TimeSpan timeLimit) + public LineTokens Tokenize(LineText line, TMState state, int offsetDelta, int maxLen, TimeSpan timeLimit) { if (_grammar == null) return null; TMState freshState = state != null ? state.Clone() : GetInitialState(); - if (line.Length > 0 && line.Length > maxLen) - line = line.Substring(0, maxLen); + ReadOnlyMemory effectiveLine = line.Memory; + if (maxLen > 0 && effectiveLine.Length > maxLen) + effectiveLine = effectiveLine.Slice(0, maxLen); - ITokenizeLineResult textMateResult = _grammar.TokenizeLine(line, freshState.GetRuleStack(), timeLimit); + ITokenizeLineResult textMateResult = _grammar.TokenizeLine(effectiveLine, freshState.GetRuleStack(), timeLimit); freshState.SetRuleStack(textMateResult.RuleStack); // Create the result early and fill in the tokens later @@ -57,7 +58,7 @@ public LineTokens Tokenize(string line, TMState state, int offsetDelta, int maxL lastTokenType = tokenType; } } - return new LineTokens(tokens, offsetDelta + line.Length, freshState); + return new LineTokens(tokens, offsetDelta + effectiveLine.Length, freshState); } private string DecodeTextMateToken(DecodeMap decodeMap, List scopes)