Add multi line text interpolation support

Package.swift

@@ -1,4 +1,4 @@
-// swift-tools-version:5.4
+// swift-tools-version:6.0
 
 /**
  *  Splash

Sources/Splash/Grammar/SwiftGrammar.swift

@@ -189,7 +189,7 @@ private extension SwiftGrammar {
                 return false
             }
 
-            return !segment.isWithinStringInterpolation
+            return !segment.isWithinMultiLineStringInterpolation
         }
     }
 
@@ -605,6 +605,95 @@ private extension Segment {
 
         return markerCounts.start != markerCounts.end
     }
+
+    var isWithinMultiLineStringInterpolation: Bool {
+        let delimiter = "\\("
+
+        if tokens.current == delimiter || tokens.previous == delimiter {
+            return true
+        }
+
+
+        /*
+         Loop back through tokens (not just same line)
+         counting closing ) and opening ( and to see if a \\(
+         can be found before the start of the string.
+
+         if the number of closed braces is < the number of opening braces + 1
+         then we are inside a multi line string interpolation.
+         */
+
+        var unbalancedClosedParenthesis = 0
+
+        /* Note the order of `(` and `)` matters.
+
+         for example \(
+            this is an interpolation
+         )(but non of this is)
+         */
+
+        for token in tokens.all.lazy.reversed() {
+            var previousChar: Character? = nil
+            // only need to search to the start of this multi line string.
+            // multi line string must have new line after """ so will always be a suffix of a token.
+            if token.hasSuffix("\"\"\"") {
+                // We are before the first interpolation.
+                return false
+            }
+
+            // The order of ( and ) is important>
+            // () does note close the interpolation
+            // )( does close the interpolation
+            for char in token.lazy.reversed() {
+                // we consume unbalancedClosedParenthesis
+                // only once we are sure we are not dealing with the start of
+                // an interpolation
+                if previousChar == "(" {
+                    if char != "\\" {
+                        if unbalancedClosedParenthesis > 0 {
+                            unbalancedClosedParenthesis -= 1
+                        }
+                        // we do not want to put unbalancedClosedParenthesis
+                        // into negative as the order of ( and ) is very important.
+                    } else {
+                        // keeping ( in the previousChar
+                        // so that if the token is \\( it still ends up consuming the open brane.
+                        continue
+                    }
+                }
+
+                previousChar = char
+
+                switch char {
+                case ")":
+                    unbalancedClosedParenthesis += 1
+                default:
+                    previousChar = char
+                }
+            }
+
+
+
+            if token.hasPrefix(delimiter) {
+                // there is a closing parenthesis that closes the scope
+                if unbalancedClosedParenthesis > 0 {
+                    return false
+                }
+                // all the closing parenthesis have matching opening parenthesis.
+                return true
+            }
+
+            // If the last char in the token was (
+            if previousChar == "(" {
+                if unbalancedClosedParenthesis > 0 {
+                    unbalancedClosedParenthesis -= 1
+                }
+            }
+        }
+
+        // not inside a multi line string
+        return false
+    }
 
     var isWithinStringInterpolation: Bool {
         let delimiter = "\\("

Sources/Splash/Theming/Theme.swift

@@ -7,7 +7,7 @@
 import Foundation
 
 #if !os(Linux)
-
+import AppKit
 /// A theme describes what fonts and colors to use when rendering
 /// certain output formats - such as `NSAttributedString`. Several
 /// default implementations are provided - see Theme+Defaults.swift.

Sources/Splash/Tokenizing/Tokenizer.swift

@@ -58,14 +58,31 @@ private extension Tokenizer {
             self.grammar = grammar
             segments = (nil, nil)
         }
-
+        
         mutating func next() -> Segment? {
+            while true {
+                switch self._next() {
+                case .next:
+                    continue
+                case .segment(let segment):
+                    return segment
+                }
+            }
+        }
+
+        private enum NextResult {
+            case segment(Segment?)
+            case next
+        }
+
+        private mutating func _next() -> NextResult {
+
             let nextIndex = makeNextIndex()
 
             guard nextIndex != code.endIndex else {
                 let segment = segments.current
                 segments.current = nil
-                return segment
+                return .segment(segment)
             }
 
             index = nextIndex
@@ -75,12 +92,14 @@ private extension Tokenizer {
             case .token, .delimiter:
                 guard var segment = segments.current else {
                     segments.current = makeSegment(with: component, at: nextIndex)
-                    return next()
+                    return .next
                 }
 
                 guard segment.trailingWhitespace == nil,
                       component.isDelimiter == segment.currentTokenIsDelimiter else {
-                    return finish(segment, with: component, at: nextIndex)
+                    return .segment(
+                        finish(segment, with: component, at: nextIndex)
+                    )
                 }
 
                 if component.isDelimiter {
@@ -89,20 +108,22 @@ private extension Tokenizer {
                                                           mergableWith: component.character)
 
                     guard shouldMerge else {
-                        return finish(segment, with: component, at: nextIndex)
+                        return .segment(
+                            finish(segment, with: component, at: nextIndex)
+                        )
                     }
                 }
 
                 segment.tokens.current.append(component.character)
                 segments.current = segment
-                return next()
+                return .next
             case .whitespace, .newline:
                 guard var segment = segments.current else {
                     var segment = makeSegment(with: component, at: nextIndex)
                     segment.trailingWhitespace = component.token
                     segment.isLastOnLine = component.isNewline
                     segments.current = segment
-                    return next()
+                    return .next
                 }
 
                 if var existingWhitespace = segment.trailingWhitespace {
@@ -117,7 +138,7 @@ private extension Tokenizer {
                 }
 
                 segments.current = segment
-                return next()
+                return .next
             }
         }
 

Tests/SplashTests/Tests/LiteralTests.swift

@@ -168,6 +168,78 @@ final class LiteralTests: SyntaxHighlighterTestCase {
             .token("\"\"\"", .string)
         ])
     }
+
+    func testMultiLineStringLiteralWithMultiLineInterpolated() {
+        let components = highlighter.highlight("""
+        let string = \"\"\"
+        Hello\\(
+            variable,
+            format: .value
+        )(not-interpolated)
+        \"\"\"
+        """)
+
+        XCTAssertEqual(components, [
+            .token("let", .keyword),
+            .whitespace(" "),
+            .plainText("string"),
+            .whitespace(" "),
+            .plainText("="),
+            .whitespace(" "),
+            .token("\"\"\"", .string),
+            .whitespace("\n"),
+            .token("Hello", .string),
+            .plainText("\\("),
+            .whitespace("\n    "),
+            .plainText("variable,"),
+            .whitespace("\n    "),
+            .plainText("format:"),
+            .whitespace(" "),
+            .plainText("."),
+            .token("value", Splash.TokenType.dotAccess),
+            .whitespace("\n"),
+            .plainText(")"),
+            .token("(not-interpolated)", .string),
+            .whitespace("\n"),
+            .token("\"\"\"", .string)
+        ])
+    }
+
+    func testMultiLineStringLiteralWithInterpolatedString() {
+        let components = highlighter.highlight("""
+        let string = \"\"\"
+        Hello \\(
+            value ? "Bob"
+        ) Welcome.
+        \"\"\"
+        """)
+
+        XCTAssertEqual(components, [
+            .token("let", .keyword),
+            .whitespace(" "),
+            .plainText("string"),
+            .whitespace(" "),
+            .plainText("="),
+            .whitespace(" "),
+            .token("\"\"\"", .string),
+            .whitespace("\n"),
+            .token("Hello", .string),
+            .whitespace(" "),
+            .plainText("\\("),
+            .whitespace("\n    "),
+            .plainText("value"),
+            .whitespace(" "),
+            .plainText("?"),
+            .whitespace(" "),
+            .token("\"Bob\"", .string),
+            .whitespace("\n"),
+            .plainText(")"),
+            .whitespace(" "),
+            .token("Welcome.", .string),
+            .whitespace("\n"),
+            .token("\"\"\"", .string)
+        ])
+    }
 
     func testSingleLineRawStringLiteral() {
         let components = highlighter.highlight("""
@@ -310,4 +382,27 @@ final class LiteralTests: SyntaxHighlighterTestCase {
             .plainText(")")
         ])
     }
+
+    /**
+
+     This test was adding since we ended up having a recursive loop that would crash.
+
+     Switching to async mode reduces your accessible stack size.
+     */
+    func testNestedMultiline() async {
+        let testString = #"""
+        struct ContentView: View {
+            @State private var text = """
+            ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~
+            """
+
+            var body: some View {
+                TextEditor(text: $text)
+                    .font(.body)
+                    .frame(width: 300, height: 300)
+            }
+        }
+        """#
+        let _ = highlighter.highlight(testString)
+    }
 }