Skip to content

Commit fa17a9a

Browse files
OceanOakOceanOak
committed
wip
1 parent 6930ff4 commit fa17a9a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

77 files changed

+9623
-8657
lines changed

‎backend/src/BuiltinExecution/Libs/Parser.fs‎

Lines changed: 88 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ module BuiltinExecution.Libs.Parser
33
open FSharp.Control.Tasks
44
open System.Threading.Tasks
55
open System.Text
6+
open System.Globalization
67

78
open Prelude
89
open LibExecution.RuntimeTypes
@@ -20,6 +21,92 @@ let pointTypeName = FQTypeName.fqPackage IDs.point
2021
let rangeTypeName = FQTypeName.fqPackage IDs.range
2122
let parsedNodeTypeName = FQTypeName.fqPackage IDs.parsedNode
2223

24+
let parse (sourceCode : string) : Dval =
25+
// This was added to handle EGCs correctly
26+
let byteIndexToCharIndex (byteIndex : int) (text : string) : int =
27+
let bytes = Encoding.UTF8.GetBytes(text)
28+
let subText = Encoding.UTF8.GetString(bytes, 0, byteIndex)
29+
StringInfo.ParseCombiningCharacters(subText).Length
30+
31+
let processLine (line : string) (startIndex : int) (endIndex : int) =
32+
let textElements = StringInfo.GetTextElementEnumerator(line)
33+
let mutable result = ""
34+
let mutable currentIndex = 0
35+
while textElements.MoveNext() do
36+
if currentIndex >= startIndex && currentIndex < endIndex then
37+
result <- result + (textElements.GetTextElement())
38+
currentIndex <- currentIndex + 1
39+
result
40+
41+
let rec mapNodeAtCursor (cursor : TreeCursor) : Dval =
42+
let mutable children = []
43+
44+
if cursor.GotoFirstChild() then
45+
children <- children @ [ mapNodeAtCursor cursor ]
46+
47+
while cursor.GotoNextSibling() do
48+
children <- children @ [ mapNodeAtCursor cursor ]
49+
50+
cursor.GotoParent() |> ignore<bool>
51+
52+
let fields =
53+
let mapPoint (point : Point) =
54+
let pointRow = point.row + 1
55+
let fields = [ "row", DInt64 pointRow; "column", DInt64 point.column ]
56+
DRecord(pointTypeName, pointTypeName, [], Map fields)
57+
58+
let startPos = cursor.Current.StartPosition
59+
let endPos = cursor.Current.EndPosition
60+
61+
let range =
62+
let fields = [ "start", mapPoint startPos; "end_", mapPoint endPos ]
63+
DRecord(rangeTypeName, rangeTypeName, [], Map fields)
64+
65+
let sourceText =
66+
let lines = String.splitOnNewline sourceCode
67+
if lines.Length = 0 then
68+
""
69+
else
70+
let startLine = lines[startPos.row]
71+
let endLine = lines[endPos.row]
72+
let startCharIndex = byteIndexToCharIndex startPos.column startLine
73+
let endCharIndex = byteIndexToCharIndex endPos.column endLine
74+
75+
match startPos.row with
76+
| row when row = endPos.row ->
77+
processLine startLine startCharIndex endCharIndex
78+
| _ ->
79+
let firstLine = processLine startLine startCharIndex startLine.Length
80+
let middleLines =
81+
if startPos.row + 1 <= endPos.row - 1 then
82+
lines[startPos.row + 1 .. endPos.row - 1]
83+
|> List.map (fun line -> processLine line 0 line.Length)
84+
else
85+
[]
86+
let lastLine = processLine endLine 0 endCharIndex
87+
String.concat "\n" (firstLine :: middleLines @ [ lastLine ])
88+
89+
let fieldName =
90+
if cursor.FieldName = null then
91+
Dval.optionNone KTString
92+
else
93+
Dval.optionSome KTString (DString cursor.FieldName)
94+
95+
[ ("fieldName", fieldName)
96+
("typ", DString cursor.Current.Kind)
97+
("text", DString sourceText)
98+
("range", range)
99+
("children", DList(VT.customType parsedNodeTypeName [], children)) ]
100+
101+
DRecord(parsedNodeTypeName, parsedNodeTypeName, [], Map fields)
102+
103+
104+
let parser = new Parser(Language = DarklangLanguage.create ())
105+
106+
let tree =
107+
parser.Parse(Encoding.UTF8.GetBytes sourceCode, InputEncoding.Utf8, None)
108+
tree.Root.Walk() |> mapNodeAtCursor
109+
23110
let fns : List<BuiltInFn> =
24111
[ { name = fn "parserParseToSimplifiedTree" 0
25112
typeParams = []
@@ -28,80 +115,7 @@ let fns : List<BuiltInFn> =
28115
description = "Parses some Darklang code"
29116
fn =
30117
(function
31-
| _, _, [ DString sourceCode ] ->
32-
// This was added to handle EGCs correctly
33-
let byteIndexToCharIndex (byteIndex : int) (text : string) : int =
34-
let bytes = Encoding.UTF8.GetBytes(text)
35-
let subText = Encoding.UTF8.GetString(bytes, 0, byteIndex)
36-
subText.Length
37-
38-
let rec mapNodeAtCursor (cursor : TreeCursor) : Dval =
39-
let mutable children = []
40-
41-
if cursor.GotoFirstChild() then
42-
children <- children @ [ mapNodeAtCursor cursor ]
43-
44-
while cursor.GotoNextSibling() do
45-
children <- children @ [ mapNodeAtCursor cursor ]
46-
47-
cursor.GotoParent() |> ignore<bool>
48-
49-
let fields =
50-
let mapPoint (point : Point) =
51-
let fields =
52-
[ "row", DInt64 point.row; "column", DInt64 point.column ]
53-
DRecord(pointTypeName, pointTypeName, [], Map fields)
54-
55-
let startPos = cursor.Current.StartPosition
56-
let endPos = cursor.Current.EndPosition
57-
58-
let range =
59-
let fields = [ "start", mapPoint startPos; "end_", mapPoint endPos ]
60-
DRecord(rangeTypeName, rangeTypeName, [], Map fields)
61-
62-
let startCharIndex = byteIndexToCharIndex startPos.column sourceCode
63-
let endCharIndex = byteIndexToCharIndex endPos.column sourceCode
64-
65-
let sourceText =
66-
let lines = String.splitOnNewline sourceCode
67-
if lines.Length = 0 then
68-
""
69-
else
70-
match startPos.row with
71-
| row when row = endPos.row ->
72-
lines[row][startCharIndex .. (endCharIndex - 1)]
73-
| _ ->
74-
let firstLine = lines[startPos.row][startCharIndex..]
75-
let middleLines =
76-
if startPos.row + 1 <= endPos.row - 1 then
77-
lines[startPos.row + 1 .. endPos.row - 1]
78-
else
79-
[]
80-
let lastLine = lines[endPos.row][.. (endCharIndex - 1)]
81-
82-
String.concat "\n" (firstLine :: middleLines @ [ lastLine ])
83-
84-
let fieldName =
85-
if cursor.FieldName = null then
86-
Dval.optionNone KTString
87-
else
88-
Dval.optionSome KTString (DString cursor.FieldName)
89-
90-
[ ("fieldName", fieldName)
91-
("typ", DString cursor.Current.Kind)
92-
("text", DString sourceText)
93-
("range", range)
94-
("children", DList(VT.customType parsedNodeTypeName [], children)) ]
95-
96-
DRecord(parsedNodeTypeName, parsedNodeTypeName, [], Map fields)
97-
98-
99-
let parser = new Parser(Language = DarklangLanguage.create ())
100-
101-
let tree =
102-
parser.Parse(Encoding.UTF8.GetBytes sourceCode, InputEncoding.Utf8, None)
103-
104-
tree.Root.Walk() |> mapNodeAtCursor |> Ply
118+
| _, _, [ DString sourceCode ] -> (parse sourceCode) |> Ply
105119
| _ -> incorrectArgs ())
106120
sqlSpec = NotQueryable
107121
previewable = Impure

‎backend/src/LibExecution/PackageIDs.fs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -383,6 +383,9 @@ module Fn =
383383
let parseSingleTestFromFile =
384384
p [] "parseSingleTestFromFile" "53f3fbc6-25fd-427a-ab0d-ba0559543c99"
385385

386+
let parseTestFile =
387+
p [] "parseTestFile" "95dc8d95-dd38-4df2-aaac-9e78187a17be"
388+
386389
// what we expose to the outside world
387390
let idForName
388391
(owner : string)

‎backend/src/LibExecution/ProgramTypesToDarkTypes.fs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -921,6 +921,9 @@ module Expr =
921921
Exception.raiseInternal "Invalid record update" [ "update", update ])
922922
PT.ERecordUpdate(uint64 id, fromDT record, updates)
923923

924+
| DEnum(_, _, [], "EConstant", [ DInt64 id; name ]) ->
925+
PT.EConstant(uint64 id, NameResolution.fromDT FQConstantName.fromDT name)
926+
924927
| e -> Exception.raiseInternal "Invalid Expr" [ "e", e ]
925928

926929

@@ -987,6 +990,7 @@ module Const =
987990
| DEnum(_, _, [], "CBool", [ DBool b ]) -> PT.Const.CBool b
988991
| DEnum(_, _, [], "CString", [ DString s ]) -> PT.Const.CString s
989992
| DEnum(_, _, [], "CChar", [ DChar c ]) -> PT.Const.CChar c
993+
| DEnum(_, _, [], "CChar", [ DString c ]) -> PT.Const.CChar c
990994
| DEnum(_, _, [], "CFloat", [ sign; DString w; DString f ]) ->
991995
PT.Const.CFloat(Sign.fromDT sign, w, f)
992996
| DEnum(_, _, [], "CUnit", []) -> PT.Const.CUnit

‎backend/src/LibExecution/ProgramTypesToRuntimeTypes.fs‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,20 @@ module MatchPattern =
165165
module Expr =
166166
let rec toRT (e : PT.Expr) : RT.Expr =
167167
match e with
168-
| PT.EChar(id, char) -> RT.EChar(id, char)
168+
| PT.EChar(id, char) ->
169+
let char =
170+
char
171+
|> fun s -> s.Replace(@"\t", "\t")
172+
|> fun s -> s.Replace(@"\n", "\n")
173+
|> fun s -> s.Replace(@"\r", "\r")
174+
|> fun s -> s.Replace(@"\b", "\b")
175+
|> fun s -> s.Replace(@"\f", "\f")
176+
|> fun s -> s.Replace(@"\v", "\v")
177+
|> fun s -> s.Replace(@"\""", "\"")
178+
|> fun s -> s.Replace(@"\'", "'")
179+
|> fun s -> s.Replace(@"\\", "\\")
180+
181+
RT.EChar(id, char)
169182
| PT.EInt64(id, num) -> RT.EInt64(id, num)
170183
| PT.EUInt64(id, num) -> RT.EUInt64(id, num)
171184
| PT.EInt8(id, num) -> RT.EInt8(id, num)
@@ -350,7 +363,28 @@ module Expr =
350363

351364
and stringSegmentToRT (segment : PT.StringSegment) : RT.StringSegment =
352365
match segment with
353-
| PT.StringText text -> RT.StringText text
366+
| PT.StringText text ->
367+
text
368+
|> fun s ->
369+
System.Text.RegularExpressions.Regex.Replace(s, @"\\x([0-9A-Fa-f]{2})",
370+
fun m ->
371+
let hexValue = System.Convert.ToByte(m.Groups[1].Value, 16)
372+
string (char hexValue))
373+
|> fun s ->
374+
System.Text.RegularExpressions.Regex.Replace(s, @"\\u([0-9A-Fa-f]{4})",
375+
fun m ->
376+
let unicodeValue = System.Convert.ToInt32(m.Groups[1].Value, 16)
377+
string (char unicodeValue))
378+
|> fun s -> s.Replace(@"\t", "\t")
379+
|> fun s -> s.Replace(@"\n", "\n")
380+
|> fun s -> s.Replace(@"\r", "\r")
381+
|> fun s -> s.Replace(@"\b", "\b")
382+
|> fun s -> s.Replace(@"\f", "\f")
383+
|> fun s -> s.Replace(@"\v", "\v")
384+
|> fun s -> s.Replace(@"\""", "\"")
385+
|> fun s -> s.Replace(@"\'", "'")
386+
|> fun s -> s.Replace(@"\\", "\\")
387+
|> RT.StringText
354388
| PT.StringInterpolation expr -> RT.StringInterpolation(toRT expr)
355389

356390

‎backend/src/Prelude/Prelude.fs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -290,7 +290,7 @@ let readFloat (f : float) : (Sign * string * string) =
290290
let makeFloat (sign : Sign) (whole : string) (fraction : string) : float =
291291
try
292292
if whole <> "" then assert_ "non-zero string" [] (whole[0] <> '-')
293-
if whole <> "0" then assertRe $"makefloat" "[1-9][0-9]*" whole
293+
if whole <> "0" then assertRe $"makefloat" "0*[0-9]+" whole
294294
let sign =
295295
match sign with
296296
| Positive -> ""

‎backend/src/Prelude/String.fs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ let lengthInEgcs (s : string) : int =
3030
System.Globalization.StringInfo(s).LengthInTextElements
3131

3232
let normalize (s : string) : string = s.Normalize()
33+
// let normalize (s : string) : string =
34+
// s.Normalize(System.Text.NormalizationForm.FormC)
3335

3436
let equalsCaseInsensitive (s1 : string) (s2 : string) : bool =
3537
System.String.Equals(s1, s2, System.StringComparison.InvariantCultureIgnoreCase)

‎backend/testfiles/execution/cloud/_events.dark‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,18 @@
44
type FruitRecord = { fruits: List<String> }
55

66
// getQueue works
7-
Builtin.testGetQueue_v0 "TestWorker" = []
7+
Builtin.testGetQueue "TestWorker" = []
88

99
// emit works
1010
(let _ = Builtin.emit "value" "TestWorker"
11-
let queue = Builtin.testGetQueue_v0 "TestWorker"
11+
let queue = Builtin.testGetQueue "TestWorker"
1212
queue) = [ "\"value\"" ]
1313

1414
// emit works with mixed values
1515
(let _ = Builtin.emit "value" "TestWorker"
1616
let _ = Builtin.emit 1 "TestWorker"
1717
let _ = Builtin.emit (FruitRecord { fruits = [ "apple"; "banana" ] }) "TestWorker"
18-
let queue = Builtin.testGetQueue_v0 "TestWorker"
19-
Stdlib.List.sort queue) = [ "\"value\""
20-
"1"
21-
"FruitRecord {\n fruits: [\n \"apple\", \"banana\"\n ]\n}" ]
18+
let queue = Builtin.testGetQueue "TestWorker"
19+
Stdlib.List.sort queue) = [ "\"value\""
20+
"1"
21+
"FruitRecord {\n fruits: [\n \"apple\", \"banana\"\n ]\n}" ]

0 commit comments

Comments
 (0)