From db559185c0847c7af7006c1fe504c54dddd9df87 Mon Sep 17 00:00:00 2001 From: MisanthropicBit Date: Wed, 8 Jul 2026 22:33:11 +0200 Subject: [PATCH 1/2] Clean up scanner and parser --- src/ast/ast.cpp | 4 +- src/ast/ast.hpp | 10 +- src/ast/ast_visitor.cpp | 4 + src/ast/ast_visitor.hpp | 2 + src/ast/expressions/expression.hpp | 1 + src/ast/expressions/parameter.cpp | 22 +- src/ast/expressions/parameter.hpp | 17 +- src/ast/expressions/type_expression.cpp | 19 + src/ast/expressions/type_expression.hpp | 24 + src/ast/parser/parser.cpp | 872 +++++++++--------------- src/ast/parser/parser.hpp | 122 ++-- src/ast/scanner/integer_format.hpp | 8 +- src/ast/scanner/keywords.cpp | 97 +-- src/ast/scanner/keywords.hpp | 43 +- src/ast/scanner/scanner.cpp | 382 +++++------ src/ast/scanner/scanner.hpp | 45 +- src/ast/scanner/token.cpp | 193 +----- src/ast/scanner/token.hpp | 77 +-- src/ast/scanner/token_type.cpp | 106 ++- src/ast/scanner/token_type.hpp | 56 +- src/ast/source_location.cpp | 10 + src/ast/source_location.hpp | 5 + src/ast/statements/function.cpp | 43 +- src/ast/statements/function.hpp | 20 +- src/ast/statements/if_statement.cpp | 4 + src/ast/statements/if_statement.hpp | 1 + src/bin/korec/main.cpp | 8 +- src/compiler/compiler.cpp | 2 +- src/utf8/utf8.cpp | 22 +- src/utf8/utf8.hpp | 6 +- tests/scanner/test_literals.cpp | 2 +- 31 files changed, 923 insertions(+), 1304 deletions(-) create mode 100644 src/ast/expressions/type_expression.cpp create mode 100644 src/ast/expressions/type_expression.hpp diff --git a/src/ast/ast.cpp b/src/ast/ast.cpp index 531b471..ba860a4 100644 --- a/src/ast/ast.cpp +++ b/src/ast/ast.cpp @@ -33,11 +33,11 @@ namespace kore { _statements.emplace_back(std::move(statement)); } - Ast::ConstIter Ast::begin() const { + Ast::const_iterator Ast::begin() const { return _statements.cbegin(); } - Ast::ConstIter Ast::end() const { + Ast::const_iterator Ast::end() const { return _statements.cend(); } diff --git a/src/ast/ast.hpp b/src/ast/ast.hpp index 0331df9..eb09ae6 100644 --- a/src/ast/ast.hpp +++ b/src/ast/ast.hpp @@ -13,14 +13,14 @@ namespace kore { class AstWriter; /// Represents an abstract syntax tree - class Ast { + class Ast final { public: - using ConstIter = std::vector::const_iterator; + using const_iterator = std::vector::const_iterator; Ast(); Ast(Ast&& ast); Ast(const fs::path& path); - virtual ~Ast(); + ~Ast(); /// The module name for the parsed file std::string module_name() const; @@ -34,8 +34,8 @@ namespace kore { /// Add a statement to this AST void add_statement(Owned statement); - ConstIter begin() const; - ConstIter end() const; + const_iterator begin() const; + const_iterator end() const; Ast& operator=(Ast&& other); diff --git a/src/ast/ast_visitor.cpp b/src/ast/ast_visitor.cpp index f57e7b7..7cfefce 100644 --- a/src/ast/ast_visitor.cpp +++ b/src/ast/ast_visitor.cpp @@ -79,6 +79,10 @@ namespace kore { UNUSED_PARAM(expr); } + void AstVisitor::visit(TypeExpression& expr) { + UNUSED_PARAM(expr); + } + void AstVisitor::visit(Branch& statement) { UNUSED_PARAM(statement); } diff --git a/src/ast/ast_visitor.hpp b/src/ast/ast_visitor.hpp index d4a094c..74d692d 100644 --- a/src/ast/ast_visitor.hpp +++ b/src/ast/ast_visitor.hpp @@ -1,6 +1,7 @@ #ifndef KORE_AST_VISITOR_HPP #define KORE_AST_VISITOR_HPP +#include "ast/expressions/type_expression.hpp" namespace kore { enum class ValueContext; @@ -55,6 +56,7 @@ namespace kore { virtual void visit(IntegerExpression& expr); virtual void visit(StringExpression& expr); virtual void visit(UnaryExpression& expr); + virtual void visit(TypeExpression& expr); // Statements virtual void visit(Branch& statement); diff --git a/src/ast/expressions/expression.hpp b/src/ast/expressions/expression.hpp index 0b0d4ee..1700b23 100644 --- a/src/ast/expressions/expression.hpp +++ b/src/ast/expressions/expression.hpp @@ -27,6 +27,7 @@ namespace kore { Literal, Parameter, Unary, + Type, }; std::ostream& operator<<(std::ostream& os, ExpressionType expr_type); diff --git a/src/ast/expressions/parameter.cpp b/src/ast/expressions/parameter.cpp index 1998f17..b0fb4c7 100644 --- a/src/ast/expressions/parameter.cpp +++ b/src/ast/expressions/parameter.cpp @@ -1,18 +1,28 @@ #include "ast/ast_visitor.hpp" #include "ast/ast_writer.hpp" #include "ast/expressions/parameter.hpp" -#include "types/unknown_type.hpp" namespace kore { - Parameter::Parameter(const Token& token) : Identifier(token, nullptr) { - set_type(Type::unknown()); - } + Parameter::Parameter(const std::string& name, const SourceLocation& location) + : Expression(ExpressionType::Identifier, location), + _name(name), + _type_expr(nullptr) {} - Parameter::Parameter(const Token& token, const Type* type) - : Identifier(token, type) { + Parameter::Parameter(const std::string& name, Owned type, const SourceLocation& location) + : Expression(ExpressionType::Identifier, location), + _name(name), + _type_expr(std::move(type)) { } Parameter::~Parameter() {} + std::string Parameter::name() const { + return _name; + } + + const TypeExpression* Parameter::type_expr() const { + return _type_expr.get(); + } + KORE_AST_VISITOR_ACCEPT_METHOD_DEFAULT_IMPL(Parameter) } diff --git a/src/ast/expressions/parameter.hpp b/src/ast/expressions/parameter.hpp index 5852b26..423d2cf 100644 --- a/src/ast/expressions/parameter.hpp +++ b/src/ast/expressions/parameter.hpp @@ -1,18 +1,25 @@ #ifndef KORE_PARAMETER_HPP #define KORE_PARAMETER_HPP -#include "ast/expressions/identifier.hpp" +#include "ast/expressions/expression.hpp" +#include "ast/expressions/type_expression.hpp" namespace kore { /// Ast node for function parameters - // TODO: Make it independent of identifiers as its type is part of the function type - class Parameter : public Identifier { + class Parameter : public Expression { public: - Parameter(const Token& token); - Parameter(const Token& token, const Type* type); + Parameter(const std::string& name, const SourceLocation& location); + Parameter(const std::string& name, Owned type, const SourceLocation& location); virtual ~Parameter(); + std::string name() const; + const TypeExpression* type_expr() const; + KORE_AST_VISITOR_ACCEPT_METHOD_DEFAULT_DEFINITION + + private: + std::string _name; + Owned _type_expr; }; } diff --git a/src/ast/expressions/type_expression.cpp b/src/ast/expressions/type_expression.cpp new file mode 100644 index 0000000..6e0d019 --- /dev/null +++ b/src/ast/expressions/type_expression.cpp @@ -0,0 +1,19 @@ +#include "ast/ast_writer.hpp" +#include "ast/expressions/type_expression.hpp" + +namespace kore { + TypeExpression::TypeExpression(const std::string& type, SourceLocation location, bool optional, int rank) + : Expression(ExpressionType::Type, location), + _type(type), + _optional(optional), + _rank(rank) + {} + + TypeExpression::~TypeExpression() {} + + std::string TypeExpression::value() const { + return _type; + } + + KORE_AST_VISITOR_ACCEPT_METHOD_DEFAULT_IMPL(TypeExpression) +} diff --git a/src/ast/expressions/type_expression.hpp b/src/ast/expressions/type_expression.hpp new file mode 100644 index 0000000..d62c5b3 --- /dev/null +++ b/src/ast/expressions/type_expression.hpp @@ -0,0 +1,24 @@ +#ifndef KORE_TYPE_EXPRESION_HPP +#define KORE_TYPE_EXPRESION_HPP + +#include "ast/expressions/expression.hpp" + +namespace kore { + /// An expression for a builtin or user-defined type + class TypeExpression : public Expression { + public: + TypeExpression(const std::string& type, SourceLocation location, bool optional, int rank); + virtual ~TypeExpression(); + + std::string value() const; + + KORE_AST_VISITOR_ACCEPT_METHOD_DEFAULT_DEFINITION + + private: + std::string _type; + bool _optional; + int _rank; + }; +} + +#endif // KORE_TYPE_EXPRESION_HPP diff --git a/src/ast/parser/parser.cpp b/src/ast/parser/parser.cpp index 77951d3..ff9efb0 100644 --- a/src/ast/parser/parser.cpp +++ b/src/ast/parser/parser.cpp @@ -12,46 +12,31 @@ #include "ast/expressions/float_expression.hpp" #include "ast/expressions/integer_expression.hpp" #include "ast/expressions/string_expression.hpp" +#include "ast/expressions/type_expression.hpp" #include "ast/parser_error_node.hpp" #include "ast/statements/expression_statement.hpp" #include "ast/statements/if_statement.hpp" #include "ast/statements/import_statement.hpp" -#include "ast/statements/module_statement.hpp" -#include "ast/statements/return_statement.hpp" #include "ast/statements/variable_assignment.hpp" -#include "logging/color_attributes.hpp" -#include "logging/color.hpp" +#include "ast/statements/return_statement.hpp" +#include "bin/korec/options.hpp" +#include "diagnostics/diagnostic2.hpp" #include "logging/logging.hpp" -#include "types/array_type.hpp" -#include "types/function_type.hpp" -#include "types/optional.hpp" -#include "types/type.hpp" #include "errors/errors.hpp" #include "operator.hpp" +#include "utils/string-utils.hpp" #include "parser.hpp" namespace kore { const std::string Parser::MUTABLE_PREFIX = "var"; - Parser::Parser() - : _failed(false), - _error_count(0), - _did_peek(false), - _ast(nullptr) { - } + Parser::Parser() : _trace(false) {} Parser::~Parser() {} - bool Parser::failed() const noexcept { - return _failed; - } - - int Parser::error_count() const noexcept { - return _error_count; - } - + // TODO: Either make a Tracer base class or make this more customisable void Parser::trace_parser(const std::string& name) { - if (_args && _args->trace == TraceOption::Parse) { + if (_trace) { const std::string group = "parse"; auto token = current_token(); @@ -73,193 +58,107 @@ namespace kore { } } - void Parser::reset() { - _module_name = ""; - _failed = false; - _error_count = 0; - _did_peek = false; - _ast = nullptr; - _args = nullptr; - } - const Token* Parser::current_token() { return &this->_current_token; } - const Token* Parser::peek_token() { - if (!_did_peek) { - _peek_token = _scanner.next_token(); - _did_peek = true; - } - - return &this->_peek_token; - } - const Token* Parser::next_token() { - if (_did_peek) { - _did_peek = false; - _current_token = _peek_token; - - return &_current_token; - } - do { _current_token = _scanner.next_token(); - } while (_current_token.category() == TokenCategory::comment); + } while (_current_token.type() == TokenType::SingleLineComment || _current_token.type() == TokenType::MultiLineComment); return &_current_token; } - bool Parser::expect_named_identifier(const std::string& name) { - const Token* const token = current_token(); - - if (token->is_identifier() && token->value() == name) { - next_token(); - - return true; - } - - return false; - } - bool Parser::expect_identifier(const std::string& error_message) { if (current_token()->is_identifier()) { return true; } if (error_message.length() > 0) { - emit_parser_error(error_message.c_str()); + emit_diagnostic(error_message.c_str()); } return false; } - bool Parser::expect_keyword(const Keyword& keyword) { - auto const token = current_token(); - - if (token->is_keyword() && token->keyword() == keyword) { - next_token(); - return true; - } - - return false; + bool Parser::check(TokenType token_type) { + return current_token()->type() == token_type; } - bool Parser::expect_token_type(const TokenType& token_type, bool advance) { - if (current_token()->type() == token_type) { - if (advance) { - next_token(); - } - - return true; + bool Parser::expect(TokenType token_type) { + if (!check(token_type)) { + return false; } - return false; - } - - bool Parser::expect_type(const std::string& name) { - const Token* const token = current_token(); - - if (token->type() == TokenType::Keyword && token->value() == name) { - next_token(); - return true; - } + next_token(); - return false; + return true; } - void Parser::emit_parser_error(const char* const format, ...) { - // TODO: Save errors in a list as with the other ast visitors - _failed = true; - ++_error_count; - - SourceLocation loc = current_token()->location(); - - std::cerr << ColorAttribute::Bold << Color::Red - << "[error:parser(" - << loc.lnum() - << "," - << loc.format_columns() - << ")]" - << ColorAttribute::Reset - << " "; - - std::cerr << _scanner.source_name() << std::endl; - + void Parser::emit_diagnostic(const char* const format, ...) { + std::string message; va_list args; va_start(args, format); - std::vfprintf(stderr, format, args); + size_t size = std::vsnprintf(0, 0, format, args); va_end(args); - - std::cerr - << std::endl - << format_error_at_line(_scanner.current_line(), current_token()->location()) - << std::endl; + message.resize(size + 1); // Need space for NUL + vsnprintf(&message[0], size + 1, format, args); + message.resize(size); // Remove the NUL + + _diagnostics.emplace_back( + Diagnostic( + 0, + message, + DiagnosticGroup::Parser, + DiagnosticLevel::Error, + current_token()->location() + ) + ); advance_to_next_statement_boundary(); } - void Parser::set_module_name(const std::string& module_name) { - _module_name = module_name; - } - - void Parser::add_statement(Statement* const parent, Owned statement) { - // Use the statement that will contain enclosed statements (such as an if - // statement containing a brach of statements), otherwise use the top-level - // ast node - if (parent) { - parent->add_statement(std::move(statement)); - } else { - _ast->add_statement(std::move(statement)); - } - } - - void Parser::parse_statement(Statement* const parent) { + Owned Parser::parse_statement() { trace_parser("statement"); auto token = current_token(); - - if (token->is_keyword()) { - switch (token->keyword()) { - case Keyword::Return: - parse_return(parent); - break; - - case Keyword::If: - parse_if_statement(parent); - break; - - default: - break; - } - } else { - if (valid_declaration_start(token)) { - parse_declaration(parent); - } else if (token->type() == TokenType::LeftBrace) { - parse_block(parent); - } + auto token_type = token->type(); + + if (token_type == TokenType::Return) { + return parse_return(); + } else if (token_type == TokenType::If) { + return parse_if_statement(); + } else if (valid_declaration_start(token)) { + return parse_declaration(); } + + return nullptr; } - void Parser::parse_statement_list(Statement* const parent) { + std::vector> Parser::parse_statement_list() { trace_parser("statement list"); + std::vector> statements; + while (valid_statement_start(current_token())) { - parse_statement(parent); + statements.push_back(parse_statement()); } + + return statements; } void Parser::advance_to_next_statement_boundary() { trace_parser("advance to next statement boundary"); while (!_scanner.eof()) { - next_token(); - auto token = current_token(); - /* auto token_type = token->type(); */ + auto token = next_token(); + auto token_type = token->type(); - /* if (token_type == TokenType::newline || token_type == TokenType::semicolon) { */ - /* return; */ - /* } */ + if (token_type == TokenType::RightBrace) { + next_token(); + return; + } if (valid_statement_start(token)) { break; @@ -267,87 +166,45 @@ namespace kore { } } - std::string Parser::module_name() const { - return _module_name; - } - + // TODO: Remove ParseErrorNode? Owned Parser::make_parser_error(const std::string& msg) { - emit_parser_error(msg.c_str()); + emit_diagnostic(msg.c_str()); return Expression::make(msg, current_token()->location()); } - void Parser::parse_module() { - trace_parser("module"); - - auto token = current_token(); - - if (expect_keyword(Keyword::Module)) { - token = current_token(); - - if (token->is_identifier()) { - /* _ast->set_module_name(token->value()); */ - set_module_name(token->value()); - add_statement(nullptr, Statement::make_statement(*token)); - } else { - emit_parser_error("Module name must be an identifier"); - } - } else { - emit_parser_error("File must start with a module declaration"); - } - - next_token(); - } - - void Parser::parse_import_decl() { - parse_import_spec(); - } - - void Parser::parse_import_spec() { + Owned Parser::parse_import_decl() { if (current_token()->type() != TokenType::Identifier) { - emit_parser_error("Expected an identifier after 'import' keyword"); - return; + emit_diagnostic("Expected an identifier after 'import' keyword"); + return nullptr; } auto module_name = parse_maybe_qualified_identifier(); - add_statement(nullptr, Statement::make_statement(std::move(module_name))); + + return Statement::make_statement(std::move(module_name)); } bool Parser::valid_statement_start(const Token* const token) { if (token->is_identifier()) { return true; - } else if (token->type() == TokenType::LeftBrace) { - return true; - } else if (token->is_keyword()) { - switch (token->keyword()) { - case Keyword::Return: - case Keyword::If: - return true; - - default: - return false; - } - } else if (token->category() == TokenCategory::literal) { - return true; } - return false; + switch (token->type()) { + case TokenType::Return: + case TokenType::If: { + return true; + } + + default: { + return false; + } + } } bool Parser::valid_declaration_start(const Token* const token) { return token->is_identifier(); } - bool Parser::valid_function_start(const Token* const token) { - if (token->is_keyword()) { - auto value = token->value(); - - return value == "export" || value == "func"; - } - - return false; - } - /* void Parser::parse_type_alias(Statement* const parent) { */ /* // TODO: Handle 'export' keyword */ /* auto token = current_token(); */ @@ -378,10 +235,10 @@ namespace kore { Owned Parser::parse_index_expression(Owned indexed_expr) { trace_parser("index expression"); - auto expr = parse_expression(operator_base_precedence()); + auto expr = parse_expression(); - if (!expect_token_type(TokenType::RightBracket)) { - emit_parser_error("Expect ']' in index expression"); + if (!expect(TokenType::RightBracket)) { + emit_diagnostic("Expect ']' in index expression"); } auto location = SourceLocation( @@ -389,22 +246,18 @@ namespace kore { current_token()->location() ); - auto index_expr = Expression::make( + return Expression::make( std::move(expr), std::move(indexed_expr), location ); - - next_token(); - - return index_expr; } Owned Parser::parse_field_access_expression(Owned expr) { trace_parser("field access expression"); - if (!expect_token_type(TokenType::Dot, true)) { - // TODO: Or return an error expression instead? + if (!expect(TokenType::Dot)) { + emit_diagnostic("Expected a dot for field access expression"); return nullptr; } @@ -416,298 +269,227 @@ namespace kore { ); } - void Parser::parse_declaration(Statement* const parent) { + Owned Parser::parse_declaration() { trace_parser("declaration"); - auto lhs_exprs = parse_lhs_expression_list(); + bool exported = expect(TokenType::Export); - if (lhs_exprs.empty()) { - return; - } else if (lhs_exprs.size() == 1 && lhs_exprs[0]->expr_type() == ExpressionType::Call) { - add_statement( - parent, - Statement::make_statement( - parse_function_call(std::move(lhs_exprs[0])) - ) - ); + if (expect(TokenType::Import)) { + parse_import_decl(); + } else if (check(TokenType::Func)) { + return parse_function(exported); + } else if (expect(TokenType::Var)) { + auto lhs_exprs = parse_lhs_expression_list(); - return; - } + if (lhs_exprs.empty()) { + return nullptr; + } - auto rhs_exprs = parse_expression_list(); + auto rhs_exprs = parse_expression_list(); - if (rhs_exprs.empty()) { - return; - } + if (rhs_exprs.empty()) { + return nullptr; + } - add_statement( - parent, - Statement::make_statement( + return Statement::make_statement( std::move(lhs_exprs), std::move(rhs_exprs) - ) - ); + ); + } else if (expect_identifier()) { + // This may be the start of a list of constant variable + // declarations or a function call + auto lhs_exprs = parse_lhs_expression_list(); + + if (lhs_exprs.empty()) { + return nullptr; + } else if (lhs_exprs.size() == 1 && lhs_exprs[0]->expr_type() == ExpressionType::Call) { + return Statement::make_statement( + parse_function_call(std::move(lhs_exprs[0])) + ); + } else { + auto rhs_exprs = parse_expression_list(); - /* auto token = current_token(); */ + if (rhs_exprs.empty()) { + return nullptr; + } - /* if (!token->is_identifier()) { */ - /* return; */ - /* } */ + return Statement::make_statement( + std::move(lhs_exprs), + std::move(rhs_exprs) + ); + } + } - /* bool is_mutable = token->value() == "var"; */ + emit_diagnostic( + "Expected a declaration but got unexpected token '%s'", + current_token()->value().c_str() + ); - /* if (is_mutable) { */ - /* token = next_token(); */ - /* } */ + return nullptr; - // TODO: Parse an expression list here to support things like - // // a.lol, b[0], var c = d() // var c i32, b bool, d str = 1, false, "hello" // var c, b, d = 1, false, "hello" + // var c, b, d = 1, false, "hello" // // If the token after the first expression is comma, continue parsing // as an expression list, otherwise parse as a function call. The lhs // expression list can only contain valid targets for an assignment so // 1 + 2 = d() is not valid - - /* auto identifier_token = *token; */ - /* token = next_token(); */ - - /* if (expect_token_type(TokenType::LeftParenthesis, false)) { */ - /* if (is_mutable) { */ - /* emit_parser_error("Function calls cannot be declared variable"); */ - /* return; */ - /* } */ - - /* auto identifier = Expression::make_expression(identifier_token); */ - - /* add_statement( */ - /* parent, */ - /* Statement::make_statement( */ - /* parse_function_call(std::move(identifier)) */ - /* ) */ - /* ); */ - - /* return; */ - /* } */ - - /* Owned lhs_expr; */ - /* Owned rhs_expr; */ - /* const Type* decl_type; */ - /* auto identifier = Expression::make_expression(identifier_token); */ - - /* if (expect_token_type(TokenType::LeftBracket, true)) { */ - /* lhs_expr = parse_index_expression(std::move(identifier)); */ - /* } else { */ - /* lhs_expr = std::move(identifier); */ - /* decl_type = parse_type(); */ - /* } */ - - /* if (!expect_token_type(TokenType::Assign)) { */ - /* emit_parser_error("Variable declarations must be initialised"); */ - /* return; */ - /* } */ - - /* rhs_expr = parse_expression(operator_base_precedence()); */ - - /* add_statement( */ - /* parent, */ - /* Statement::make_statement( */ - /* is_mutable, */ - /* decl_type, */ - /* std::move(lhs_expr), */ - /* std::move(rhs_expr) */ - /* ) */ - /* ); */ } - void Parser::parse_if_statement(Statement* const parent) { + Owned Parser::parse_if_statement() { trace_parser("if"); - if (expect_keyword(Keyword::If)) { - auto if_statement = Statement::make_statement(); - auto condition = parse_expression(operator_base_precedence()); - - if (condition->is_error()) { - advance_to_next_statement_boundary(); - add_statement(parent, std::move(if_statement)); - return; - } - - parse_block(if_statement.get()); - if_statement->add_branch(std::move(condition)); - - // Parse multiple 'else-if' statements - while (expect_keyword(Keyword::Else)) { - if (expect_keyword(Keyword::If)) { - auto elseif_condition = parse_expression(operator_base_precedence()); - - if (elseif_condition->is_error()) { - advance_to_next_statement_boundary(); - break; - } - - parse_block(if_statement.get()); - if_statement->add_branch(std::move(elseif_condition)); - } else { - parse_block(if_statement.get()); - if_statement->add_else_branch(); - } - } - - add_statement(parent, std::move(if_statement)); + if (!expect(TokenType::If)) { + emit_diagnostic("Expected 'if' keyword"); + return nullptr; } - } - void Parser::parse_toplevel(Statement* const parent) { - /* parse_module(); */ + auto if_statement = Statement::make_statement(); + auto condition = parse_expression(); - while (expect_keyword(Keyword::Import)) { - parse_import_decl(); + // TODO: Return a nullptr and emit a parse error instead + if (condition->is_error()) { + advance_to_next_statement_boundary(); + return if_statement; } - auto token = current_token(); + if_statement->add_branch(std::move(condition)); + if_statement->add_statements(parse_block()); - while (token->type() != TokenType::Eof) { - if (valid_declaration_start(token)) { - parse_declaration(parent); - } else if (valid_function_start(token)) { - parse_function(parent); - } else { - if (token->is_keyword() && token->keyword() == Keyword::Return) { - emit_parser_error("Can only return from within a function"); - } else { - emit_parser_error("Expected a declaration"); + // Parse multiple 'else-if' statements + while (expect(TokenType::Else)) { + if (expect(TokenType::If)) { + auto elseif_condition = parse_expression(); + + if (elseif_condition->is_error()) { + advance_to_next_statement_boundary(); + break; } - break; + if_statement->add_branch(std::move(elseif_condition)); + if_statement->add_statements(parse_block()); + } else { + if_statement->add_else_branch(); + if_statement->add_statements(parse_block()); } } + + return if_statement; } - void Parser::parse_function(Statement* const parent) { + Owned Parser::parse_function(bool exported) { trace_parser("function"); - bool exported = expect_keyword(Keyword::Export); - bool is_func = expect_keyword(Keyword::Func); + bool is_func = expect(TokenType::Func); if (exported) { if (!is_func) { - emit_parser_error("Expected 'func' after 'export' keyword"); - next_token(); // TODO: Remove? - return; + emit_diagnostic("Expected 'func' after 'export' keyword"); + return nullptr; } } else if (!is_func) { - emit_parser_error("Expected 'func' keyword for valid function declaration"); - return; + emit_diagnostic("Expected 'func' keyword for valid function declaration"); + return nullptr; } - if (!expect_identifier("Expected function name identifier after 'func'")) { - next_token(); - return; + if (!expect_identifier("Expected function name after 'func'")) { + return nullptr; } trace_parser("function name"); - const auto func_name(*current_token()); - next_token(); - - auto func = Statement::make_statement(exported, func_name); - - // Create a function type for the function we are about to parse. - // Parsing will fill it with parameter and return types and we save it - // in the type cache afterwards - auto func_type = std::make_unique(); - func->set_type(func_type.get()); - - parse_function_signature(func.get()); - parse_block(func.get()); - - Type::set_function_type(std::move(func_type)); + const auto func_name(*next_token()); - add_statement(parent, std::move(func)); - } - - void Parser::parse_function_signature(Function* const func) { - trace_parser("function signature"); - - if (expect_token_type(TokenType::LeftParenthesis)) { - if (!expect_token_type(TokenType::RightParenthesis)) { - // TODO: Return parameter types from this method for constructing the function type - parse_function_parameters(func); - } - - auto return_types = parse_type_list(); - - if (return_types.empty()) { - // If no return type was specified, mark it as unknown and - // infer it in the type inference pass - func->type()->add_return_type(Type::unknown()); - } else { - for (auto type : return_types) { - func->type()->add_return_type(type); - } - } + if (!expect(TokenType::LeftParenthesis)) { + emit_diagnostic("Expected '(' for function signature"); + return nullptr; } + + auto parameters = parse_function_parameters(); + auto return_types = parse_type_list(); + auto body = parse_block(); + + // TODO: Need the right locations for the function + return Statement::make_statement( + exported, + func_name.value(), + std::move(parameters), + std::move(return_types), + std::move(body) + ); } - void Parser::parse_function_parameters(Function* const func) { + std::vector> Parser::parse_function_parameters() { trace_parser("function parameters"); - parse_parameter_list(func); + return parse_parameter_list(); } - void Parser::parse_parameter_decl(Function* const func) { + Owned Parser::parse_parameter() { trace_parser("parameter declaration"); - auto token = *current_token(); + auto param_token = *current_token(); - if (token.type() == TokenType::Identifier) { - auto parameter = Expression::make(token, Type::unknown()); - token = *next_token(); + if (expect_identifier()) { + auto type_expr = parse_type(); - if (token.type() != TokenType::Comma && token.type() != TokenType::RightParenthesis) { - parameter->set_type(parse_type()); + if (!type_expr) { + return nullptr; } - // TODO: Need to fix this - func->type()->add_parameter_type(parameter->type()); - func->add_parameter(std::move(parameter)); - } else { - emit_parser_error("Unexpected token '%s' in function parameter", token.type()); + return Expression::make(param_token.value(), std::move(type_expr), param_token.location()); } + + emit_diagnostic("Unexpected token '%s' in function parameter", param_token.type()); + + return nullptr; } - void Parser::parse_parameter_list(Function* const func) { + std::vector> Parser::parse_parameter_list() { trace_parser("parameter list"); SourceLocation loc = current_token()->location(); + std::vector> parameters; + + if (expect(TokenType::RightParenthesis)) { + return parameters; + } do { - parse_parameter_decl(func); + auto parameter = parse_parameter(); + + if (!parameter) { + break; + } + + parameters.push_back(parameter); auto token_type = current_token()->type(); - if (token_type == TokenType::Comma) { + if (expect(TokenType::Comma)) { next_token(); - } else if (token_type == TokenType::RightParenthesis) { + } else if (expect(TokenType::RightParenthesis)) { next_token(); break; } else { - emit_parser_error( + emit_diagnostic( "Expected ',' or ')' after parameter declaration but got unexpected token '%s'", token_type ); break; } } while (true); + + return parameters; } - void Parser::parse_return(Statement* const parent) { + Owned Parser::parse_return() { trace_parser("return"); next_token(); auto expr_list = parse_expression_list(); - add_statement(parent, Statement::make_statement(std::move(expr_list))); + + return Statement::make_statement(std::move(expr_list)); } IdentifierList Parser::parse_identifier_list() { @@ -726,7 +508,7 @@ namespace kore { token = next_token(); } else { - emit_parser_error("Expected an identifier but got '%s'", token->value().c_str()); + emit_diagnostic("Expected an identifier but got '%s'", token->value().c_str()); break; } } while (true); @@ -734,65 +516,67 @@ namespace kore { return identifiers; } - void Parser::parse_block(Statement* const parent) { + std::vector> Parser::parse_block() { trace_parser("block"); - if (expect_token_type(TokenType::LeftBrace)) { - parse_statement_list(parent); + if (!expect(TokenType::LeftBrace)) { + emit_diagnostic("Expected '{' at start of block"); + return {}; + } - if (!expect_token_type(TokenType::RightBrace)) { - emit_parser_error("Expected '}' to close block"); - } + auto statements = parse_statement_list(); + + if (!expect(TokenType::RightBrace)) { + emit_diagnostic("Expected '}' to close block"); + return {}; } + + return statements; } - const Type* Parser::parse_type() { + Owned Parser::parse_type() { trace_parser("type"); - auto token = current_token(); + if (!expect_identifier("Expected a type")) { + return nullptr; + } - if (token->is_type()) { - auto type = Type::from_token(*token); - next_token(); - ArrayType* array_type = nullptr; - - do { - if (expect_token_type(TokenType::LeftBracket)) { - // Array type (possibly nested) - if (!expect_token_type(TokenType::RightBracket)) { - emit_parser_error("Expected ']' after '[' in array type declaration"); - return nullptr; - } - - if (!array_type) { - array_type = Type::make_array_type(type); - } else { - array_type->increase_rank(); - } - } else if (expect_token_type(TokenType::QuestionMark)) { - return Type::make_optional_type(type); - } else { - return array_type ? array_type : type; - } - } while (true); + bool optional = false; + int rank = 0; + std::vector tokens{ *current_token() }; + + // Possibly nested array type + while (expect(TokenType::LeftBracket)) { + if (!expect(TokenType::RightBracket)) { + emit_diagnostic("Expected ']' after '[' in array type declaration"); + return nullptr; + } + + ++rank; } - return Type::unknown(); + if (expect(TokenType::QuestionMark)) { + optional = true; + } + + auto type_expr_value = join_on(tokens, ""); + auto location = SourceLocation::merged(tokens[0].location(), tokens.back().location()); + + return Expression::make(type_expr_value, location, optional, rank); } - std::vector Parser::parse_type_list() { - std::vector types; + std::vector> Parser::parse_type_list() { + std::vector> types; do { - auto type = parse_type(); - types.push_back(type); + types.push_back(parse_type()); - if (expect_token_type(TokenType::LeftBrace, false)) { + if (check(TokenType::LeftBrace)) { break; - } else if (expect_token_type(TokenType::Comma)) { + } else if (expect(TokenType::Comma)) { // Moves to next token } else { - emit_parser_error("Expected ',' or '{' when parsing function return type"); + emit_diagnostic("Expected ',' or '{' when parsing function return type"); break; } } while (true); @@ -881,17 +665,17 @@ namespace kore { /* auto& token = *current_token(); */ next_token(); - auto first_expr = parse_expression(operator_base_precedence()); + auto first_expr = parse_expression(); Owned result = nullptr; - if (expect_token_type(TokenType::Colon)) { + if (expect(TokenType::Colon)) { result = parse_array_fill_expression(lbracket_token, std::move(first_expr)); - } else if (expect_token_type(TokenType::Comma)) { + } else if (expect(TokenType::Comma)) { result = parse_normal_array_expression(lbracket_token, std::move(first_expr)); - } else if (expect_token_type(TokenType::RightBracket)) { + } else if (expect(TokenType::RightBracket)) { /* result = parse_index_expression(&token); */ } else { - emit_parser_error("Expected ':' or ',' in array literal"); + emit_diagnostic("Expected ':' or ',' in array literal"); } return result; @@ -902,9 +686,9 @@ namespace kore { Owned size_expr ) { trace_parser("array fill"); - auto fill_expr = parse_expression(operator_base_precedence()); + auto fill_expr = parse_expression(); - if (!expect_token_type(TokenType::RightBracket)) { + if (!expect(TokenType::RightBracket)) { return make_parser_error("Expected ']' after array fill expression"); } @@ -926,12 +710,12 @@ namespace kore { array_expr->set_start_location(lbracket_token->location()); while (!_scanner.eof()) { - auto element_expr = parse_expression(operator_base_precedence()); + auto element_expr = parse_expression(); array_expr->add_element(std::move(element_expr)); - if (expect_token_type(TokenType::RightBracket, false)) { + if (check(TokenType::RightBracket)) { break; - } else if (!expect_token_type(TokenType::Comma)) { + } else if (!expect(TokenType::Comma)) { array_expr->add_element(make_parser_error("Expected ',' after array element expression")); return array_expr; @@ -947,14 +731,13 @@ namespace kore { Owned Parser::parse_array_range_expression(const Token* const lbracket_token) { trace_parser("array range"); - int base_precedence = operator_base_precedence(); - auto start_expr = parse_expression(base_precedence); + auto start_expr = parse_expression(); - if (!expect_token_type(TokenType::Range)) { + if (!expect(TokenType::Range)) { return make_parser_error("Expected range '..' in array range expression"); } - auto end_expr = parse_expression(base_precedence); + auto end_expr = parse_expression(); return Expression::make( std::move(start_expr), @@ -997,22 +780,18 @@ namespace kore { auto token = current_token(); - if (token->is_keyword() && token->keyword() == Keyword::None) { + if (check(TokenType::None)) { auto expr = Expression::make(*token); next_token(); - + return expr; } expr = parse_maybe_qualified_identifier(); - /* if (!expr->is_error()) { */ - /* return expr; */ - /* } */ - - if (expect_token_type(TokenType::LeftParenthesis)) { + if (check(TokenType::LeftParenthesis)) { return parse_parenthesised_expression(); - } else if (expect_token_type(TokenType::LeftBracket)) { + } else if (expect(TokenType::LeftBracket)) { return parse_index_expression(std::move(expr)); } @@ -1056,7 +835,7 @@ namespace kore { } else { return expr; } - } else if (expect_token_type(TokenType::LeftBracket)) { + } else if (expect(TokenType::LeftBracket)) { return parse_index_expression(std::move(expr)); } } @@ -1078,9 +857,9 @@ namespace kore { Owned Parser::parse_function_call(Owned func_name) { trace_parser("call"); - if (!expect_token_type(TokenType::LeftParenthesis)) { + if (!expect(TokenType::LeftParenthesis)) { return {}; - } else if (expect_token_type(TokenType::RightParenthesis)) { + } else if (expect(TokenType::RightParenthesis)) { next_token(); return {}; } @@ -1108,9 +887,9 @@ namespace kore { trace_parser("exprlist:expr"); expr_list.push_back(std::move(expr)); - if (expect_token_type(TokenType::RightParenthesis)) { + if (expect(TokenType::RightParenthesis)) { break; - } else if (!expect_token_type(TokenType::Comma)) { + } else if (!expect(TokenType::Comma)) { break; } } @@ -1119,33 +898,31 @@ namespace kore { } std::pair, bool> Parser::parse_lhs_expression() { - auto token = current_token(); - bool is_mutable = expect_keyword(Keyword::Var); + bool is_mutable = expect(TokenType::Var); - if (!token->is_identifier()) { - emit_parser_error("Invalid left-hand side target for assignment"); + if (!expect_identifier("Invalid left-hand side target for assignment")) { return { nullptr, false }; } - auto identifier_token = *token; + auto identifier_token = *current_token(); Owned expr = Expression::make(identifier_token, is_mutable); bool parsed_index_or_field_expr = false; next_token(); // Continue parsing nested field access or index sub-expressions e.g. "a.b[0].c[1]" - // or we parse a function call e.g. "a.b[0].c[1]()" + // or until we parse a function call e.g. "a.b[0].c[1]()" do { - if (expect_token_type(TokenType::LeftBracket, false)) { + if (check(TokenType::LeftBracket)) { expr = parse_index_expression(std::move(expr)); parsed_index_or_field_expr = true; - } else if (expect_token_type(TokenType::Dot, false)) { + } else if (check(TokenType::Dot)) { expr = parse_field_access_expression(std::move(expr)); parsed_index_or_field_expr = true; - } else if (expect_token_type(TokenType::LeftParenthesis, false)) { + } else if (check(TokenType::LeftParenthesis)) { // TODO: Parse entire function call here if (is_mutable) { - emit_parser_error("Function calls cannot be declared variable"); + emit_diagnostic("Function calls cannot be declared variable"); return { nullptr, false }; } @@ -1157,13 +934,12 @@ namespace kore { if (parsed_index_or_field_expr) { if (is_mutable) { - emit_parser_error("Cannot declare left-hand side index or field access expression as variable"); + emit_diagnostic("Cannot declare left-hand side index or field access expression as variable"); return { nullptr, false }; } } else { - // TODO: Perhaps let parse_type see if the next thing is an identifier instead - if (!expect_token_type(TokenType::Comma, false) && !expect_token_type(TokenType::Assign, false)) { - expr->set_type(parse_type()); + if (!check(TokenType::Comma) && !check(TokenType::Assign)) { + auto type_expr = parse_type(); } } @@ -1184,15 +960,16 @@ namespace kore { lhs_exprs.emplace_back(std::move(lhs_expr)); - if (expect_token_type(TokenType::Assign)) { + if (expect(TokenType::Assign)) { if (contains_function_call) { - emit_parser_error("Function call cannot be the target of an assignment"); + emit_diagnostic("Function call cannot be the target of an assignment"); + return {}; } - return lhs_exprs; - } else if (!expect_token_type(TokenType::Comma)) { - emit_parser_error("Expected ',' or '=' after expression in left-hand side expression list"); - return lhs_exprs; + break; + } else if (!expect(TokenType::Comma)) { + emit_diagnostic("Expected ',' or '=' after expression in left-hand side expression list"); + return {}; } } while (true); @@ -1201,9 +978,11 @@ namespace kore { Owned Parser::parse_parenthesised_expression() { trace_parser("parenthesised expression"); - Owned expr = parse_expression(operator_base_precedence()); + + next_token(); // Consume left parenthesis + Owned expr = parse_expression(); - if (expect_token_type(TokenType::RightParenthesis)) { + if (expect(TokenType::RightParenthesis)) { expr->set_parenthesised(true); return expr; } @@ -1222,7 +1001,7 @@ namespace kore { while (true) { auto token = current_token(); - if (token->category() != TokenCategory::bin_op) { + if (token->is_binary_operator()) { return left; } @@ -1234,9 +1013,8 @@ namespace kore { } SourceLocation binop_location = token->location(); - std::string op = token->op(); + auto op = token->value(); next_token(); - auto right = parse_expression(right_precedence); if (op == "..") { @@ -1260,65 +1038,37 @@ namespace kore { return left; } - ParseResult Parser::parse_non_module( - const std::string& value, - const ParsedCommandLineArgs& args - ) { - Ast ast{""}; - - _ast = * - _args = &args; - _scanner.scan_string(value); - - next_token(); - parse_declaration(); - - return handle_parse_result(ast); - } - ParseResult Parser::parse_string( const std::string& value, const ParsedCommandLineArgs& args ) { - Ast ast{""}; - - _ast = * - _args = &args; _scanner.scan_string(value); + Ast ast{ "" }; - next_token(); - parse_toplevel(); - - return handle_parse_result(ast); + return parse(ast, args); } ParseResult Parser::parse_file( const std::string& path, const ParsedCommandLineArgs& args ) { - if (!_scanner.open_file(path)) { + if (!_scanner.scan_file(path)) { return {}; } - Ast ast{path}; - _ast = * - _args = &args; + Ast ast{ path }; - next_token(); - parse_toplevel(); - - return handle_parse_result(ast); + return parse(ast, args); } - ParseResult Parser::handle_parse_result(Ast& ast) { - if (failed()) { - reset(); - return {}; - } + ParseResult Parser::parse(Ast& ast, const ParsedCommandLineArgs& args) { + _trace = args.trace == TraceOption::Parse; + auto token = next_token(); - ParseResult result = std::make_optional(std::move(ast)); - reset(); + while (token->type() != TokenType::Eof) { + ast.add_statement(parse_declaration()); + } - return result; + return ParseResult{ std::move(ast), std::move(_diagnostics) }; } } diff --git a/src/ast/parser/parser.hpp b/src/ast/parser/parser.hpp index efdb331..394763c 100644 --- a/src/ast/parser/parser.hpp +++ b/src/ast/parser/parser.hpp @@ -1,45 +1,39 @@ #ifndef KORE_PARSER_HPP #define KORE_PARSER_HPP +#include #include #include "ast/ast.hpp" #include "ast/expressions/expression.hpp" #include "ast/expressions/identifier.hpp" -#include "ast/statements/function.hpp" +#include "ast/expressions/type_expression.hpp" +#include "ast/expressions/parameter.hpp" #include "ast/statements/statement.hpp" -#include "ast/scanner/keywords.hpp" #include "ast/scanner/scanner.hpp" #include "ast/scanner/token.hpp" #include "bin/korec/options.hpp" +#include "diagnostics/diagnostic2.hpp" + +namespace fs = std::filesystem; namespace kore { using ExpressionList = std::vector>; using IdentifierList = std::vector>; - using ParseResult = std::optional; - /// The parser. Each parse method is annotated with a comment with the / - /// particular part of the grammar that it handles. It is a recursive descent - /// parser. + struct ParseResult { + Ast ast; + std::vector diagnostics; + }; + + /// Each parse method is annotated with a comment with the / particular + /// part of the grammar that it handles. It is a recursive descent parser. /// - /// See 'kore.grammer' for the full grammar + /// See 'kore.grammar' for the full grammar class Parser final { public: Parser(); - virtual ~Parser(); - - bool failed() const noexcept; - int error_count() const noexcept; - - std::string module_name() const; - - /// Parse a non-module string without requiring a module declaration - /// and normal program structure. This is used to parse expressions - /// and statements given on the command line or in the REPL - ParseResult parse_non_module( - const std::string& value, - const ParsedCommandLineArgs& args - ); + ~Parser(); /// Parse a program in a string ParseResult parse_string( @@ -53,55 +47,43 @@ namespace kore { const ParsedCommandLineArgs& args ); + /// Parse a program in a file + ParseResult parse_file( + const fs::path& path, + const ParsedCommandLineArgs& args + ); + private: - std::string _module_name; - bool _failed; - int _error_count; + std::vector _diagnostics; Scanner _scanner; Token _current_token; - bool _did_peek; Token _peek_token; - Ast* _ast; - const ParsedCommandLineArgs* _args; + bool _trace; static const std::string MUTABLE_PREFIX; + private: void trace_parser(const std::string& name); - /* std::vector _warnings; */ - /* std::vector _errors; */ - - ParseResult handle_parse_result(Ast& ast); - - void reset(); + /// TopLevelDecl = Declaration | Function . + ParseResult parse(Ast& ast, const ParsedCommandLineArgs& args); const Token* current_token(); - - const Token* peek_token(); - const Token* next_token(); - bool expect_named_identifier(const std::string& name); + /// Check that the current token matches the type but do not advance + bool check(TokenType token_type); - bool expect_identifier(const std::string& error_message); + /// Check that the current token matches the type and advance + bool expect(TokenType token_type); - bool expect_keyword(const Keyword& keyword); + bool expect_identifier(const std::string& error_message = ""); - bool expect_token_type( - const TokenType& token_type, - bool advance = true - ); - - bool expect_type(const std::string& name); - - void emit_parser_error(const char* const format, ...); + // TODO: Change to accept severity when we can use std::format + void emit_diagnostic(const char* const format, ...); Owned make_parser_error(const std::string& msg); - void set_module_name(const std::string& module_name); - - void add_statement(Statement* const parent, Owned statement); - // When we encounter an error, this method is called to advance the // parser (and scanner) to the beginning of the next expression. // Otherwise, we will continue to parse the erroreous expression which @@ -109,24 +91,17 @@ namespace kore { void advance_to_next_statement_boundary(); /// Statement = Declaration | SimpleStmt | IfStmt | ForStmt | Block | ReturnStmt . - void parse_statement(Statement* const parent); + Owned parse_statement(); /// StatementList = { Statement ";" } . - void parse_statement_list(Statement* const parent); - - /// ModuleDecl = "module" ModuleName . - void parse_module(); + std::vector> parse_statement_list(); /// ImportDecl = "import" ModuleName { "." ModuleName } [ "{" ModuleList "}" ] . - void parse_import_decl(); - void parse_import_spec(); + Owned parse_import_decl(); bool valid_statement_start(const Token* const token); - bool valid_declaration_start(const Token* const token); - bool valid_function_start(const Token* const token); - /// TypeAlias = [ "export" ] "type" Identifier "=" Type . /* void parse_type_alias(Statement* const parent); */ @@ -134,41 +109,36 @@ namespace kore { Owned parse_field_access_expression(Owned expr); - void parse_declaration(Statement* const parent = nullptr); + Owned parse_declaration(); /// IfStmt = "if" [ SimpleStmt ] Block [ "else" ( IfStmt | Block ) ] . - void parse_if_statement(Statement* const parent); - - /// TopLevelDecl = Declaration | Function . - void parse_toplevel(Statement* const parent = nullptr); + Owned parse_if_statement(); /// Function = [ "export" ] "func" FunctionName FuncSignature [ FunctionBody ] . - void parse_function(Statement* const parent); - /// FuncSignature = Parameters [ Type ] . - void parse_function_signature(Function* const func); + Owned parse_function(bool exported); /// Parameters = "(" [ ParameterList ] ")" . - void parse_function_parameters(Function* const func); + std::vector> parse_function_parameters(); /// ParameterDecl = [ IdentifierList ] [ "..." ] Type . - void parse_parameter_decl(Function* const func); + Owned parse_parameter(); /// ParameterList = ParameterDecl { "," ParameterDecl } . - void parse_parameter_list(Function* const func); + std::vector> parse_parameter_list(); /// ReturnStmt = "return" [ ExpressionList ] . - void parse_return(Statement* const parent); + Owned parse_return(); /// IdentifierList = Identifier { "," Identifier } . IdentifierList parse_identifier_list(); - const Type* parse_type(); + Owned parse_type(); - std::vector parse_type_list(); + std::vector> parse_type_list(); /// Block = "{" StatementList "}" . - void parse_block(Statement* const parent); + std::vector> parse_block(); /// int_lit = decimal_lit | binary_lit | octal_lit | hex_lit . Owned parse_literal(); diff --git a/src/ast/scanner/integer_format.hpp b/src/ast/scanner/integer_format.hpp index c9e1710..eda33b4 100644 --- a/src/ast/scanner/integer_format.hpp +++ b/src/ast/scanner/integer_format.hpp @@ -4,10 +4,10 @@ namespace kore { // Different supported integer formats enum class IntegerFormat { - dec, - bin, - hex, - oct, + Decimal, + Binary, + Hexadecimal, + Octal, }; } diff --git a/src/ast/scanner/keywords.cpp b/src/ast/scanner/keywords.cpp index f5d056a..8517e0e 100644 --- a/src/ast/scanner/keywords.cpp +++ b/src/ast/scanner/keywords.cpp @@ -1,6 +1,7 @@ -#include "keywords.hpp" #include +#include "keywords.hpp" + namespace kore { /* A 1 in the table indicates that this ASCII character is the first character * in some keyword, 0 means that it is not. @@ -36,57 +37,65 @@ namespace kore { 0, // z }; - std::map _STRING_TO_KEYWORDS{ - {"bool", Keyword::Bool}, - {"byte", Keyword::Byte}, - {"char", Keyword::Char}, - {"else", Keyword::Else}, - {"export", Keyword::Export}, - {"enum", Keyword::Enum}, - {"f32", Keyword::F32}, - {"f64", Keyword::F64}, - {"false", Keyword::False}, - {"for", Keyword::For}, - {"func", Keyword::Func}, - {"i8", Keyword::I8}, - {"i16", Keyword::I16}, - {"i32", Keyword::I32}, - {"i64", Keyword::I64}, - {"if", Keyword::If}, - {"import", Keyword::Import}, - {"in", Keyword::In}, - {"is", Keyword::Is}, - {"match", Keyword::Match}, - {"module", Keyword::Module}, - {"None", Keyword::None}, - {"return", Keyword::Return}, - {"Some", Keyword::Some}, - {"str", Keyword::Str}, - {"struct", Keyword::Struct}, - {"true", Keyword::True}, - {"try", Keyword::Try}, - {"type", Keyword::Type}, - {"u8", Keyword::U8}, - {"u16", Keyword::U16}, - {"u32", Keyword::U32}, - {"u64", Keyword::U64}, - {"var", Keyword::Var} + std::map _STRING_TO_KEYWORDS{ + {"bool", TokenType::Bool}, + {"byte", TokenType::Byte}, + {"char", TokenType::Char}, + {"else", TokenType::Else}, + {"export", TokenType::Export}, + {"enum", TokenType::Enum}, + {"f32", TokenType::F32}, + {"f64", TokenType::F64}, + {"false", TokenType::False}, + {"for", TokenType::For}, + {"func", TokenType::Func}, + {"i8", TokenType::I8}, + {"i16", TokenType::I16}, + {"i32", TokenType::I32}, + {"i64", TokenType::I64}, + {"if", TokenType::If}, + {"import", TokenType::Import}, + {"in", TokenType::In}, + {"is", TokenType::Is}, + {"match", TokenType::Match}, + {"None", TokenType::None}, + {"return", TokenType::Return}, + {"Some", TokenType::Some}, + {"str", TokenType::Str}, + {"struct", TokenType::Struct}, + {"true", TokenType::True}, + {"try", TokenType::Try}, + {"type", TokenType::Type}, + {"u8", TokenType::U8}, + {"u16", TokenType::U16}, + {"u32", TokenType::U32}, + {"u64", TokenType::U64}, + {"var", TokenType::Var} }; - bool is_keyword(const std::string& identifier) { - char byte = identifier[0]; + bool is_keyword(const std::string& value) { + char byte = value[0]; int idx = static_cast(byte - (byte <= 'Z' ? 'A' : 'a')); - if (idx >= 0) { - if (_KEYWORD_FIRST_CHAR_TABLE[idx]) { - return _STRING_TO_KEYWORDS.count(identifier); - } + if (idx >= 0 && _KEYWORD_FIRST_CHAR_TABLE[idx]) { + return _STRING_TO_KEYWORDS.count(value); } return false; } - Keyword keyword_from_string(const std::string& value) { - return _STRING_TO_KEYWORDS[value]; + TokenType resolve_keyword(const std::string& value) { + char byte = value[0]; + int idx = static_cast(byte - (byte <= 'Z' ? 'A' : 'a')); + + if (idx >= 0 && _KEYWORD_FIRST_CHAR_TABLE[idx]) { + auto it = _STRING_TO_KEYWORDS.find(value); + + if (it != _STRING_TO_KEYWORDS.end()) { + return it->second; + } + } + + return TokenType::Identifier; } } diff --git a/src/ast/scanner/keywords.hpp b/src/ast/scanner/keywords.hpp index 5afdb4e..0da4862 100644 --- a/src/ast/scanner/keywords.hpp +++ b/src/ast/scanner/keywords.hpp @@ -3,47 +3,12 @@ #include -namespace kore { - enum class Keyword { - Bool, - Byte, - Char, - Else, - Enum, - Export, - F32, - F64, - False, - For, - Func, - I16, - I32, - I64, - I8, - If, - Import, - In, - Is, - Match, - Module, - None, - Return, - Some, - Str, - Struct, - True, - Try, - Type, - U16, - U32, - U64, - U8, - Var - }; +#include "ast/scanner/token_type.hpp" - bool is_keyword(const std::string& identifier); +namespace kore { + bool is_keyword(const std::string& value); - Keyword keyword_from_string(const std::string& value); + TokenType resolve_keyword(const std::string& value); } #endif // KORE_KEYWORDS_HPP diff --git a/src/ast/scanner/scanner.cpp b/src/ast/scanner/scanner.cpp index c84d2eb..077efee 100644 --- a/src/ast/scanner/scanner.cpp +++ b/src/ast/scanner/scanner.cpp @@ -2,10 +2,9 @@ #include #include -#include "errors/errors.hpp" -#include "keywords.hpp" -#include "logging/logging.hpp" #include "scanner.hpp" +#include "ast/scanner/keywords.hpp" +#include "ast/scanner/token.hpp" #include "utf8/utf8.hpp" // NOTE: No KORE_DEBUG helpers here since it's easy to tell @@ -41,72 +40,50 @@ namespace kore { } Scanner::Scanner() - : lnum(0), - last_col(0), - col(0) { - /* toplevel(true), */ - /* in_function(false), */ - /* in_string(false) { */ + : _lnum(0), + _last_col(0), + _col(0) { } - Scanner::~Scanner() { + Scanner::~Scanner() {} + + std::string Scanner::extract_current_characters() { + return _line.substr(_last_col, _col - _last_col + 1); } Token Scanner::make_one_char_token(TokenType type) { - auto token = Token::make_token( - type, - lnum, - col, - col, - line.substr(col, 1) - ); - - ++col; + auto token = Token(type, _lnum, _col, _col, _line.substr(_col, 1)); + ++_col; return token; } Token Scanner::make_inline_token(TokenType type, std::size_t end_col, std::size_t advance) { - auto token = Token::make_token( + auto token = Token( type, - lnum, - last_col, + _lnum, + _last_col, end_col, - line.substr(last_col, end_col - last_col + 1) + _line.substr(_last_col, end_col - _last_col + 1) ); - col += advance; - last_col = col; + _col += advance; + _last_col = _col; return token; } - Token Scanner::make_inline_int_token(IntegerFormat format, std::size_t end_col, std::size_t advance) { - auto token = Token::make_int_token( - format, - lnum, - last_col, - end_col, - line.substr(last_col, end_col - last_col + 1) + Token Scanner::make_char_token(std::size_t end_col, std::size_t advance) { + auto token = Token( + TokenType::Character, + _lnum, + _last_col, + _last_col, + _line.substr(_last_col, end_col - _last_col + 1) ); - col += advance; - last_col = col; - - return token; - } - - Token Scanner::make_char_token(codepoint cp, std::size_t end_col, std::size_t advance) { - auto token = Token::make_char_token( - cp, - lnum, - last_col, - last_col, - line.substr(last_col, end_col - last_col + 1) - ); - - col += advance; - last_col = col; + _col += advance; + _last_col = _col; return token; } @@ -115,34 +92,29 @@ namespace kore { TokenType type, std::size_t start_lnum, std::size_t start_col, - /* std::size_t end_lnum, */ std::size_t end_col ) { - return Token::make_token( + return Token( type, start_lnum, start_col, - /* end_lnum, */ end_col, - line.substr(start_col, end_col - start_col + 1) + _line.substr(start_col, end_col - start_col + 1) ); } - bool Scanner::open_file(const fs::path& path) { + bool Scanner::scan_file(const fs::path& path) { _source_name = path; std::ifstream ifs{ path }; - stream = std::make_unique(path); + _stream = std::make_unique(path); read_line(); - // Get the first token - /* next_token(); */ - return ifs.is_open(); } void Scanner::scan_string(const std::string& string) { _source_name = ""; - stream = std::make_unique(string); + _stream = std::make_unique(string); read_line(); } @@ -150,23 +122,17 @@ namespace kore { return _source_name; } - void Scanner::throw_error(const std::string& msg) { - error_group("scanner", "%s", format_error(msg, line, lnum, last_col, col).c_str()); - - throw_error_for_line(msg, line, lnum, col, col); - } - void Scanner::read_line() { // TODO: Don't rely on eof (https://isocpp.org/wiki/faq/input-output#istream-and-eof) if (eof()) { return; } - std::string current_line = line; - std::getline(*stream, line); - ++lnum; - last_col = 0; - col = 0; + std::string current_line = _line; + std::getline(*_stream, _line); + ++_lnum; + _last_col = 0; + _col = 0; /* if (did_read) { */ /* ++lnum; */ @@ -183,18 +149,18 @@ namespace kore { } std::string Scanner::consume_line() { - auto value = line.substr(col); + auto value = _line.substr(_col); read_line(); return value; } Token Scanner::consume_until(TokenType type, const std::string& value) { - std::size_t start_lnum = lnum; - std::size_t start_col = col; + std::size_t start_lnum = _lnum; + std::size_t start_col = _col; while (true) { - std::size_t idx = line.find(value); + std::size_t idx = _line.find(value); if (idx != std::string::npos) { return make_multiline_token( @@ -214,7 +180,7 @@ namespace kore { std::size_t consumed = 0; while (!eol()) { - char byte = line[col]; + char byte = _line[_col]; if (is_whitespace(byte)) { break; @@ -224,33 +190,22 @@ namespace kore { return 0; } - ++col; + ++_col; ++consumed; } return consumed; } - /* bool Scanner::accept(const std::string& run) { */ - /* for (int i = 0; i < run.length() && !eol(); ++i) { */ - /* if (line[col + i] != run[i]) { */ - /* return false; */ - /* } */ - /* } */ - - /* if (valid.find(byte) != std::string::npos) { */ - /* ++col; */ - /* return true; */ - /* } */ - - /* return false; */ - /* } */ + void Scanner::advance(bool allow_newline) { + ++_col; + } bool Scanner::accept_any(const std::string& valid) { - char byte = line[col]; + char byte = _line[_col]; if (valid.find(byte) != std::string::npos) { - ++col; + ++_col; return true; } @@ -258,43 +213,57 @@ namespace kore { } bool Scanner::expect(char byte) { - return line[col] == byte; + if(_line[_col] == byte) { + ++_col; + return true; + } + + return false; } - bool Scanner::expect_peek(char byte) { - if (!eol()) { - if (line[col + 1] == byte) { - ++col; - return true; - } + bool Scanner::peek(char byte, int offset) { + if (eol(offset)) { + return false; } - return false; + return _line[_col + offset] == byte; } - inline bool Scanner::eol() const { - return col >= line.length(); + inline bool Scanner::eol(int offset) const { + return _col + offset >= _line.length(); } bool Scanner::eof() const { - return stream && stream->eof(); + return _stream && _stream->eof(); } std::string Scanner::current_line() const { - return line; + return _line; } void Scanner::skip_whitespace() { while (!eol()) { - if (is_whitespace(line[col])) { - ++col; + if (is_whitespace(_line[_col])) { + ++_col; } else { - last_col = col; + _last_col = _col; return; } } } + void Scanner::emit_diagnostic(const std::string& message) { + _diagnostics.emplace_back( + Diagnostic( + 0, + message, + DiagnosticGroup::Scan, + DiagnosticLevel::Error, + SourceLocation(_lnum, _col, _col) + ) + ); + } + bool Scanner::is_whitespace(char byte) const noexcept { return byte == ' ' || byte == '\t' || byte == '\r'; } @@ -332,20 +301,22 @@ namespace kore { } Token Scanner::scan_number() { - char byte = line[col]; + if (peek('0', false)) { + if (eol(1)) { + emit_diagnostic("Unexpected end-of-line when scanning non-decimal number"); - if (byte == '0') { - ++col; + return make_one_char_token(TokenType::Invalid); + } - if (accept_any("xX")) { + if (peek('x', false, 1) || peek('X', false, 1)) { return scan_hex_number(); - } else if (accept_any("bB")) { + } else if (peek('b', false, 1) || peek('B', false, 1)) { return scan_binary_number(); - } else if (accept_any("01234567")) { + } else if (peek('o', false, 1) || peek('O', false, 1)) { return scan_octal_number(); } - --col; + --_col; } return scan_decimal_number(); @@ -353,91 +324,111 @@ namespace kore { Token Scanner::scan_hex_number() { if (consume_while(&Scanner::is_hex_digit) < 1) { - throw_error("Invalid hexadecimal number"); + emit_diagnostic("Invalid hexadecimal number"); + + return make_one_char_token(TokenType::Invalid); } - return make_inline_int_token(IntegerFormat::hex, col-1); + return make_inline_token(TokenType::Integer, _col - 1); } Token Scanner::scan_binary_number() { if (consume_while(&Scanner::is_binary_digit) < 1) { - throw_error("Invalid binary number"); + emit_diagnostic("Invalid binary number"); + + return make_one_char_token(TokenType::Invalid); } - return make_inline_int_token(IntegerFormat::bin, col-1); + return make_inline_token(TokenType::Integer, _col - 1); } Token Scanner::scan_octal_number() { if (consume_while(&Scanner::is_octal_digit) < 1) { - throw_error("Invalid octal number"); + emit_diagnostic("Invalid octal number"); + + return make_one_char_token(TokenType::Invalid); } - return make_inline_int_token(IntegerFormat::oct, col-1); + return make_inline_token(TokenType::Integer, _col - 1); } Token Scanner::scan_decimal_number() { - bool dot = false; - while (!eol()) { - char byte = line[col]; + char byte = _line[_col++]; - if (byte == '.') { - if (dot) { - throw_error("Invalid decimal number (multiple '.' found)"); - } - - dot = true; - } else if (!is_digit(byte)) { + if (!is_digit(byte) || byte != '_') { break; } + } + + bool dot = peek('.'); + bool exp = peek('e') || peek('E'); - ++col; + if (dot || exp) { + scan_float_suffix(); } - if (dot) { - return make_inline_token(TokenType::Float, col-1); - } else { - return make_inline_int_token(IntegerFormat::dec, col-1); + scan_number_suffix(); + + return make_inline_token(dot ? TokenType::Float : TokenType::Integer, _col - 1); + } + + void Scanner::scan_float_suffix() { + if (peek('.')) { + while (!eol() && is_digit(_line[_col++])) {} + } + + if (peek('e') || peek('E')) { + if (peek('-') || peek('+')) { + if (!is_digit(_line[_col])) { + emit_diagnostic("Expected digits after exponent"); + } else { + while (!eol() && is_digit(_line[_col++])) {} + } + } } } - Token Scanner::scan_identifier() { + void Scanner::scan_number_suffix() { + if (!expect('_')) { + return; + } + while (!eol()) { - if (!is_valid_identifier(line[col])) { + char byte = _line[_col++]; + + if (!is_digit(byte) || !std::isalpha(byte)) { break; } - - ++col; } + } - Token token = make_inline_token(TokenType::Identifier, col-1); - - if (is_keyword(token.value())) { - token.as_keyword(); + Token Scanner::scan_identifier() { + while (!eol() && is_valid_identifier(_line[_col])) { + ++_col; } - return token; + auto text = _line.substr(_last_col, _col - 1 - _last_col + 1); + auto token_type = resolve_keyword(text); + + return make_inline_token(token_type, _col - 1); } Token Scanner::scan_string() { // Skip first double quote - ++col; + advance(); while (!eol()) { if (expect('"')) { - auto token = make_inline_token(TokenType::String, col); - ++col; - - return token; + return make_inline_token(TokenType::String, _col, 1); } else { scan_utf8_encoded_codepoint(); } } - throw_error("End of string not encountered"); + emit_diagnostic("Unterminated string literal"); - // Make the compiler happy, even though throw_error will always throw - return Token(); + return make_one_char_token(TokenType::Invalid); } /* Token Scanner::scan_format_string() { */ @@ -464,43 +455,38 @@ namespace kore { Token Scanner::scan_character() { // Skip first single quote - last_col = ++col; + _last_col = ++_col; - codepoint cp = scan_utf8_encoded_codepoint(); + if (peek('\'')) { + emit_diagnostic("Empty character literal"); - if (expect('\'')) { - return make_char_token(cp, (col++)-1); - } else { - throw_error("Character contains more than one character, should be a string?"); + return make_one_char_token(TokenType::Invalid); } - // Make the compiler happy, even though throw_error will always throw - return Token(); - } + scan_utf8_encoded_codepoint(); - codepoint Scanner::scan_utf8_encoded_codepoint() { - if (col >= line.length()) { - throw_error("Scanned codepoint at end-of-line"); + if (expect('\'')) { + return make_char_token(_col - 1, 1); } - int num_bytes = 0; - DecodeError error = DecodeError::None; - codepoint cp = utf8_decode_string_pos(line, col, num_bytes, error); + return make_one_char_token(TokenType::Invalid); + } - if (error != DecodeError::None) { - throw_error("Invalid unicode character"); + void Scanner::scan_utf8_encoded_codepoint() { + if (eol()) { + emit_diagnostic("Scanned codepoint at end-of-line"); + } else { + _col += utf8_codepoint_bytes(_line[_col]); } - - col += num_bytes; - - return cp; } Token Scanner::scan_equal_or_arrow() { - if (expect_peek('>')) { - return make_inline_token(TokenType::Arrow, col, 2); - } else if (expect_peek('=')) { - return make_inline_token(TokenType::Equal, col, 2); + advance(); + + if (expect('>')) { + return make_inline_token(TokenType::Arrow, _col); + } else if (expect('=')) { + return make_inline_token(TokenType::Equal, _col); } return make_one_char_token(TokenType::Assign); @@ -511,7 +497,7 @@ namespace kore { return scan_multiline_comment(); } - auto token = make_inline_token(TokenType::SingleLineComment, line.length() - 1); + auto token = make_inline_token(TokenType::SingleLineComment, _line.length() - 1); // Just read the next line now to skip whitespace on the new line read_line(); @@ -520,35 +506,37 @@ namespace kore { } Token Scanner::scan_mult_or_exp() { - if (expect_peek('*')) { - return make_inline_token(TokenType::Exp, col, 1); + if (peek('*', 1)) { + return make_inline_token(TokenType::Exp, _col, 1); } return make_one_char_token(TokenType::Mult); } Token Scanner::scan_dot_or_range() { - if (expect_peek('.')) { - return make_inline_token(TokenType::Range, col, 1); + if (peek('.')) { + return make_inline_token(TokenType::Range, _col, 1); } return make_one_char_token(TokenType::Dot); } Token Scanner::scan_op_maybe_equal(TokenType op, TokenType equal_op) { - if (expect_peek('=')) { - return make_inline_token(equal_op, col, 2); + if (peek('=')) { + return make_inline_token(equal_op, _col, 2); } return make_one_char_token(op); } Token Scanner::scan_not_equal() { - if (!expect_peek('=')) { - throw_error("Expected '=' after '!'"); + if (!peek('=')) { + emit_diagnostic("Expected '=' after '!'"); + + return make_one_char_token(TokenType::Invalid); } - return make_inline_token(TokenType::NotEqual, col, 2); + return make_inline_token(TokenType::NotEqual, _col, 2); } Token Scanner::scan_multiline_comment() { @@ -556,11 +544,7 @@ namespace kore { } Token Scanner::next_token() { - /* if (in_string) { */ - /* // In the process of scanning a, possibly formatted, string */ - /* scan_string(); */ - /* } */ - + // TODO: Refactor skip_whitespace(); // If we are at the end of the line but not the file, keep reading lines @@ -571,7 +555,7 @@ namespace kore { skip_whitespace(); if (eol() && eof()) { - auto eof_token = Token::make_eof(lnum, col, col); + auto eof_token = Token::make_eof(_lnum, _col, _col); // Free the input stream as soon as we hit the eof token. This does not // really matter for string inputs but for file inputs we should @@ -581,7 +565,7 @@ namespace kore { return eof_token; } - char byte = line[col]; + char byte = _line[_col]; if (is_digit(byte)) { return scan_number(); @@ -614,13 +598,13 @@ namespace kore { case '\'': return scan_character(); case '=': return scan_equal_or_arrow(); - default: - throw_error("Unknown character in stream '" + std::string(byte, 1) + "'"); + default: { + emit_diagnostic("Unknown character in stream '" + std::string(byte, 1) + "'"); + break; + } } - // Make the compiler happy, even though this method will throw and neveer - // get here - return Token(); + return make_one_char_token(TokenType::Invalid); } Scanner::token_iterator Scanner::begin() { diff --git a/src/ast/scanner/scanner.hpp b/src/ast/scanner/scanner.hpp index e2f08c5..7f59765 100644 --- a/src/ast/scanner/scanner.hpp +++ b/src/ast/scanner/scanner.hpp @@ -2,7 +2,9 @@ #define KORE_SCANNER_HPP #include +#include +#include "diagnostics/diagnostic2.hpp" #include "pointer_types.hpp" #include "ast/scanner/token.hpp" @@ -37,9 +39,9 @@ namespace kore { public: Scanner(); - virtual ~Scanner(); + ~Scanner(); - bool open_file(const fs::path& path); + bool scan_file(const fs::path& path); void scan_string(const std::string& string); std::string source_name() const; bool eof() const; @@ -50,34 +52,34 @@ namespace kore { token_iterator end(); private: - std::size_t lnum; - std::size_t last_col; - std::size_t col; + std::size_t _lnum; + std::size_t _last_col; + std::size_t _col; std::string _source_name; - Owned stream; - std::string line; - - // True if we are at the top-level where scanning starts - /* bool toplevel; */ - - /* bool in_function; */ - /* bool in_string; */ + Owned _stream; + std::string _line; + std::vector _diagnostics; // Predicate type for consuming a run of characters using Predicate = bool (Scanner::*)(char) const; - void throw_error(const std::string& msg); - void read_line(); std::string consume_line(); Token consume_until(TokenType type, const std::string& value); std::size_t consume_while(Predicate predicate); + void advance(bool allow_newline = true); /* bool accept(const std::string& run); */ bool accept_any(const std::string& any_valid); + + // Expect a character and advance if it matches bool expect(char byte); - bool expect_peek(char byte); - inline bool eol() const; + + // Peek a character at an offset but do not advance + bool peek(char byte, int offset = 1); + + inline bool eol(int offset = 0) const; void skip_whitespace(); + void emit_diagnostic(const std::string& message); bool is_whitespace(char byte) const noexcept; bool is_digit_start(char byte) const noexcept; @@ -93,23 +95,24 @@ namespace kore { Token scan_binary_number(); Token scan_octal_number(); Token scan_decimal_number(); + void scan_float_suffix(); + void scan_number_suffix(); Token scan_identifier(); Token scan_string(); - /* Token scan_format_string(); */ Token scan_character(); Token scan_equal_or_arrow(); - codepoint scan_utf8_encoded_codepoint(); Token scan_comment(); Token scan_multiline_comment(); Token scan_mult_or_exp(); Token scan_dot_or_range(); Token scan_op_maybe_equal(TokenType op, TokenType equal_op); Token scan_not_equal(); + void scan_utf8_encoded_codepoint(); + std::string extract_current_characters(); Token make_one_char_token(TokenType type); Token make_inline_token(TokenType type, std::size_t end_col, std::size_t advance = 0); - Token make_inline_int_token(IntegerFormat format, std::size_t end_col, std::size_t advance = 0); - Token make_char_token(codepoint cp, std::size_t end_col, std::size_t advance = 0); + Token make_char_token(std::size_t end_col, std::size_t advance = 0); Token make_multiline_token( TokenType type, std::size_t start_lnum, diff --git a/src/ast/scanner/token.cpp b/src/ast/scanner/token.cpp index 2dfe463..0c6fdc1 100644 --- a/src/ast/scanner/token.cpp +++ b/src/ast/scanner/token.cpp @@ -1,30 +1,15 @@ -#include #include #include "ast/scanner/token.hpp" namespace kore { - Token::Token() : Token(TokenType::Invalid, SourceLocation::unknown) { - } + Token::Token() : Token(TokenType::Invalid, SourceLocation::unknown) {} Token::Token(const Token& token) : _type(token.type()), _location(token.location()), - _value(token.value()), - _category(token.category()) - { - if (_type == TokenType::Integer) { - _internal_value.integer = token.int_value(); - } else if (_type == TokenType::Float) { - _internal_value.float32 = token.float32_value(); - } else if (_type == TokenType::Keyword) { - _internal_value.keyword = token.keyword(); - } else if (_type == TokenType::Character) { - _internal_value.integer = token.int_value(); - } else { - _internal_value.str = token.string_value(); - } - } + _value(token.value()) + {} Token::Token(TokenType type, const SourceLocation& location) : Token(type, location, "") { @@ -34,29 +19,14 @@ namespace kore { : _type(type), _location(location), _value(value) { - _category = token_category_from_type(type); } - Token::~Token() { - } + Token::~Token() {} Token& Token::operator=(const Token& token) { _type = token.type(); _location = token.location(); _value = token.value(); - _category = token.category(); - - if (_type == TokenType::Integer) { - _internal_value.integer = token.int_value(); - } else if (_type == TokenType::Float) { - _internal_value.float32 = token.float32_value(); - } else if (_type == TokenType::Keyword) { - _internal_value.keyword = token.keyword(); - } else if (_type == TokenType::Character) { - _internal_value.integer = token.int_value(); - } else { - _internal_value.str = token.string_value(); - } return *this; } @@ -73,8 +43,8 @@ namespace kore { return _value; } - TokenCategory Token::category() const { - return _category; + bool Token::is_binary_operator() const noexcept { + return _type >= TokenType::Div && _type <= TokenType::Plus; } bool Token::is_eof() const noexcept { @@ -86,160 +56,15 @@ namespace kore { } bool Token::is_keyword() const noexcept { - return _type == TokenType::Keyword; - } - - bool Token::is_type() const noexcept { - return is_keyword() && ( - _value == "bool" || - _value == "byte" || - _value == "char" || - _value == "f32" || - _value == "f64" || - _value == "i8" || - _value == "i16" || - _value == "i32" || - _value == "i64" || - _value == "str" || - _value == "u8" || - _value == "u16" || - _value == "u32" || - _value == "u64" - ); - } - - bool Token::is_boolean_keyword() const noexcept { - if (type() == TokenType::Keyword) { - auto kw = keyword(); - - return kw == Keyword::True || kw == Keyword::False; - } - - return false; - } - - void Token::as_keyword() { - if (type() == TokenType::Identifier) { - _type = TokenType::Keyword; - _internal_value.keyword = keyword_from_string(value()); - } - } - - i32 Token::int_value() const { - return _internal_value.integer; - } - - f32 Token::float32_value() const { - return _internal_value.float32; - } - - std::string Token::string_value() const { - return _internal_value.str; - } - - Keyword Token::keyword() const { - return _internal_value.keyword; - } - - std::string Token::op() const { - return value(); - } - - bool Token::is_op() const noexcept { - return operator_precedence(value()) != -1; - } - - Token Token::make_invalid_token() { - return Token(TokenType::Invalid, SourceLocation::unknown); + return Token::is_keyword(_type); } Token Token::make_eof(std::size_t lnum, std::size_t start, std::size_t end) { return Token(TokenType::Eof, SourceLocation(lnum, start, end)); } - Token Token::make_token(TokenType type, const SourceLocation& location, const std::string& value) { - return Token(type, location, value); - } - - Token Token::make_int_token( - IntegerFormat format, - std::size_t lnum, - std::size_t start, - std::size_t end, - const std::string& value - ) { - auto token = Token::make_token(TokenType::Integer, SourceLocation(lnum, start, end), value); - - switch (format) { - case IntegerFormat::dec: - token._internal_value.integer = std::stoi(value, nullptr, 10); - break; - - case IntegerFormat::bin: - token._internal_value.integer = std::stoi(value.substr(2), nullptr, 2); - break; - - case IntegerFormat::hex: - token._internal_value.integer = std::stoi(value, nullptr, 16); - break; - - case IntegerFormat::oct: - token._internal_value.integer = std::stoi(value, nullptr, 8); - break; - } - - return token; - } - - Token Token::make_char_token( - codepoint cp, - std::size_t lnum, - std::size_t start, - std::size_t end, - const std::string& value - ) { - auto token = Token::make_token(TokenType::Character, SourceLocation(lnum, start, end), value); - token._internal_value.integer = cp; - - return token; - } - - Token Token::make_token( - TokenType type, - std::size_t lnum, - std::size_t start, - std::size_t end, - const std::string& value - ) { - auto token = Token::make_token(type, SourceLocation(lnum, start, end), value); - - if (type == TokenType::Float) { - token._internal_value.float32 = std::stof(value); - } else if (type == TokenType::Keyword) { - token._internal_value.keyword = keyword_from_string(value); - } - - return token; - } - - std::ostream& Token::ostream_value(std::ostream& os) const { - if (type() == TokenType::Integer) { - os << _internal_value.integer; - } else if (type() == TokenType::Float) { - os << _internal_value.float32; - } else { - os << _internal_value.str; - } - - return os; - } - - std::ostream& Token::column_format(std::ostream& os, int spacing) const { - // TODO: Move this to a templated utility function - return os - << std::setw(spacing) << std::left << type() - << std::setw(spacing) << std::left << value() - << location().colon_format(); + bool Token::is_keyword(TokenType type) { + return type >= TokenType::Bool && type <= TokenType::Var; } std::ostream& operator<<(std::ostream& os, const Token& token) { diff --git a/src/ast/scanner/token.hpp b/src/ast/scanner/token.hpp index 19dc114..f6af59f 100644 --- a/src/ast/scanner/token.hpp +++ b/src/ast/scanner/token.hpp @@ -1,16 +1,10 @@ #ifndef KORE_TOKEN_HPP #define KORE_TOKEN_HPP -#include #include -#include "internal_value_types.hpp" #include "ast/source_location.hpp" -#include "ast/scanner/integer_format.hpp" -#include "ast/scanner/keywords.hpp" #include "ast/scanner/token_type.hpp" -#include "ast/scanner/token_category.hpp" -#include "ast/parser/operator.hpp" namespace kore { /// A token generated by the scanner @@ -20,84 +14,33 @@ namespace kore { Token(const Token& token); Token(TokenType type, const SourceLocation& location); Token(TokenType type, const SourceLocation& location, const std::string& value); - virtual ~Token(); + Token( + TokenType type, + std::size_t lnum, + std::size_t start, + std::size_t end, + const std::string& value + ); + ~Token(); Token& operator=(const Token& token); TokenType type() const; SourceLocation location() const; std::string value() const; - TokenCategory category() const; + bool is_binary_operator() const noexcept; bool is_eof() const noexcept; bool is_identifier() const noexcept; bool is_keyword() const noexcept; - bool is_type() const noexcept; - bool is_boolean_keyword() const noexcept; - - void as_keyword(); - - i32 int_value() const; - f32 float32_value() const; - std::string string_value() const; - Keyword keyword() const; - std::string op() const; - bool is_op() const noexcept; - - std::ostream& ostream_value(std::ostream& os) const; - std::ostream& column_format(std::ostream& os, int spacing = 20) const; - static Token make_invalid_token(); static Token make_eof(std::size_t lnum, std::size_t start, std::size_t end); - static Token make_int_token( - IntegerFormat format, - std::size_t lnum, - std::size_t start, - std::size_t end, - const std::string& value - ); - static Token make_char_token( - codepoint cp, - std::size_t lnum, - std::size_t start, - std::size_t end, - const std::string& value - ); - static Token make_token( - TokenType type, - const SourceLocation& location, - const std::string& value - ); - static Token make_token( - TokenType type, - std::size_t lnum, - std::size_t start, - std::size_t end, - const std::string& value - ); + static bool is_keyword(TokenType type); private: TokenType _type; SourceLocation _location; std::string _value; - TokenCategory _category; - - private: - union TokenValue { - std::string str; - i32 integer; - f32 float32; - /* f64 float64; */ - Keyword keyword; - - // Union does not contain PODs only so the default constructor - // is deleted and must be explicitly defined - TokenValue() : str() {} - - ~TokenValue() {} - }; - - TokenValue _internal_value; }; std::ostream& operator<<(std::ostream& os, const Token& token); diff --git a/src/ast/scanner/token_type.cpp b/src/ast/scanner/token_type.cpp index 13f8b6e..fd93214 100644 --- a/src/ast/scanner/token_type.cpp +++ b/src/ast/scanner/token_type.cpp @@ -3,42 +3,76 @@ namespace kore { namespace token_names { const std::map mapping{ - {TokenType::Arrow, "arrow"}, - {TokenType::Assign, "assign"}, - {TokenType::At, "at"}, - {TokenType::Bar, "bar"}, - {TokenType::Character, "character"}, - {TokenType::Colon, "colon"}, - {TokenType::Comma, "comma"}, - {TokenType::Div, "div"}, - {TokenType::Dot, "dot"}, - {TokenType::Eof, "eof"}, - {TokenType::Equal, "equal"}, - {TokenType::Exp, "exp"}, - {TokenType::Float, "float"}, - {TokenType::GreaterThanEqual, "greater_than_equal"}, - {TokenType::GreaterThan, "greater_than"}, - {TokenType::Identifier, "identifier"}, - {TokenType::Integer, "integer"}, - {TokenType::Invalid, "invalid"}, - {TokenType::Keyword , "keyword"}, - {TokenType::LeftBrace, "left_brace"}, - {TokenType::LeftBracket, "left_bracket"}, - {TokenType::LessThanEqual, "less_than_equal"}, - {TokenType::LeftParenthesis, "left_parenthesis"}, - {TokenType::LessThan, "less_than"}, - {TokenType::Minus, "minus"}, - {TokenType::Mult, "mult"}, - {TokenType::MultiLineComment , "multi_line_comment"}, - {TokenType::NotEqual, "not_equal"}, - {TokenType::Plus, "plus"}, - {TokenType::QuestionMark, "question_mark"}, - {TokenType::Range, "range"}, - {TokenType::RightBrace, "right_brace"}, - {TokenType::RightBracket, "right_bracket"}, - {TokenType::RightParenthesis, "right_parenthesis"}, - {TokenType::SingleLineComment , "single_line_comment"}, - {TokenType::String, "string"}, + {TokenType::Arrow, "arrow"}, + {TokenType::Assign, "assign"}, + {TokenType::At, "at"}, + {TokenType::Bar, "bar"}, + {TokenType::Character, "character"}, + {TokenType::Colon, "colon"}, + {TokenType::Comma, "comma"}, + {TokenType::Div, "div"}, + {TokenType::Dot, "dot"}, + {TokenType::Eof, "eof"}, + {TokenType::Equal, "equal"}, + {TokenType::Exp, "exp"}, + {TokenType::Float, "float"}, + {TokenType::GreaterThanEqual, "greater_than_equal"}, + {TokenType::GreaterThan, "greater_than"}, + {TokenType::Identifier, "identifier"}, + {TokenType::Integer, "integer"}, + {TokenType::Invalid, "invalid"}, + {TokenType::LeftBrace, "left_brace"}, + {TokenType::LeftBracket, "left_bracket"}, + {TokenType::LessThanEqual, "less_than_equal"}, + {TokenType::LeftParenthesis, "left_parenthesis"}, + {TokenType::LessThan, "less_than"}, + {TokenType::Minus, "minus"}, + {TokenType::Mult, "mult"}, + {TokenType::MultiLineComment, "multi_line_comment"}, + {TokenType::NotEqual, "not_equal"}, + {TokenType::Plus, "plus"}, + {TokenType::QuestionMark, "question_mark"}, + {TokenType::Range, "range"}, + {TokenType::RightBrace, "right_brace"}, + {TokenType::RightBracket, "right_bracket"}, + {TokenType::RightParenthesis, "right_parenthesis"}, + {TokenType::SingleLineComment, "single_line_comment"}, + {TokenType::String, "string"}, + + // Keywords + {TokenType::Bool, "bool"}, + {TokenType::Byte, "byte"}, + {TokenType::Char, "char"}, + {TokenType::Else, "else"}, + {TokenType::Enum, "enum"}, + {TokenType::Export, "export"}, + {TokenType::F32, "f32"}, + {TokenType::F64, "f64"}, + {TokenType::False, "false"}, + {TokenType::For, "for"}, + {TokenType::Func, "func"}, + {TokenType::I16, "i16"}, + {TokenType::I32, "i32"}, + {TokenType::I64, "i64"}, + {TokenType::I8, "i8"}, + {TokenType::If, "if"}, + {TokenType::Import, "import"}, + {TokenType::In, "in"}, + {TokenType::Is, "is"}, + {TokenType::Match, "match"}, + {TokenType::None, "none"}, + {TokenType::Return, "return"}, + {TokenType::Some, "some"}, + {TokenType::Str, "str"}, + {TokenType::Struct, "struct"}, + {TokenType::True, "true"}, + {TokenType::Try, "try"}, + {TokenType::Type, "type"}, + {TokenType::U16, "u16"}, + {TokenType::U32, "u32"}, + {TokenType::U64, "u64"}, + {TokenType::U8, "u8"}, + {TokenType::Var, "var"} }; } diff --git a/src/ast/scanner/token_type.hpp b/src/ast/scanner/token_type.hpp index 6d4e4cd..4f2c719 100644 --- a/src/ast/scanner/token_type.hpp +++ b/src/ast/scanner/token_type.hpp @@ -14,28 +14,18 @@ namespace kore { Character, Colon, Comma, - Div, Dot, Eof, - Equal, Exp, Float, - GreaterThanEqual, - GreaterThan, Identifier, Integer, Invalid, - Keyword, LeftBrace, LeftBracket, LessThanEqual, LeftParenthesis, - LessThan, - Minus, - Mult, MultiLineComment, - NotEqual, - Plus, QuestionMark, Range, RightBrace, @@ -43,6 +33,52 @@ namespace kore { RightParenthesis, SingleLineComment, String, + + // Binary operators + Div, + Equal, + GreaterThanEqual, + GreaterThan, + LessThan, + Minus, + Mult, + NotEqual, + Plus, + + // Keywords + Bool, + Byte, + Char, + Else, + Enum, + Export, + F32, + F64, + False, + For, + Func, + I16, + I32, + I64, + I8, + If, + Import, + In, + Is, + Match, + None, + Return, + Some, + Str, + Struct, + True, + Try, + Type, + U16, + U32, + U64, + U8, + Var }; namespace token_names { diff --git a/src/ast/source_location.cpp b/src/ast/source_location.cpp index 3f53696..9eb6cb6 100644 --- a/src/ast/source_location.cpp +++ b/src/ast/source_location.cpp @@ -74,6 +74,16 @@ namespace kore { _end_col = std::max(_end_col, location.end()); } + SourceLocation SourceLocation::merged( + const SourceLocation& location1, + const SourceLocation& location2 + ) { + SourceLocation location = location1; + location.merge(location2); + + return location; + } + const SourceLocation SourceLocation::unknown = SourceLocation(); std::ostream& operator<<(std::ostream& os, const SourceLocation& location) { diff --git a/src/ast/source_location.hpp b/src/ast/source_location.hpp index db678d1..a93e6db 100644 --- a/src/ast/source_location.hpp +++ b/src/ast/source_location.hpp @@ -24,6 +24,11 @@ namespace kore { void merge(const SourceLocation& location); + static SourceLocation merged( + const SourceLocation& location1, + const SourceLocation& location2 + ); + static const SourceLocation unknown; private: diff --git a/src/ast/statements/function.cpp b/src/ast/statements/function.cpp index 19ee7a0..91175fa 100644 --- a/src/ast/statements/function.cpp +++ b/src/ast/statements/function.cpp @@ -3,32 +3,39 @@ #include "ast/ast_writer.hpp" #include "ast/ast_visitor.hpp" #include "ast/statements/function.hpp" -#include "types/function_type.hpp" -#include "types/type.hpp" -#include "types/unknown_type.hpp" namespace kore { Function::Function() : Statement(SourceLocation::unknown, StatementType::Function), _name(""), - _exported(false), - _type(nullptr) { + _exported(false) { } Function::Function(bool exported, const Token& token) : Statement(token.location(), StatementType::Function), _name(token), - _exported(exported), - _type(nullptr) { + _exported(exported) { } Function::Function(bool exported) : Statement(SourceLocation::unknown, StatementType::Function), _name(""), - _exported(exported), - _type(nullptr) { + _exported(exported) { } + Function::Function( + bool exported, + const std::string& name, + std::vector> parameters, + std::vector> return_types, + StatementList body + ) : Statement(SourceLocation::unknown, StatementType::Function), + _name(name), + _exported(exported), + _parameters(parameters), + _return_types(return_types), + _body(body) { + } Function::~Function() {} std::string Function::name() const { @@ -44,23 +51,7 @@ namespace kore { } int Function::arity() const { - return _type->arity(); - } - - const FunctionType* Function::type() const { - return _type; - } - - FunctionType* Function::type() { - return _type; - } - - void Function::set_type(FunctionType* type) { - _type = type; - - // The function's name (identifier) has the type of the function itself. - // Identifiers are what is stored in the scope stack - _name.set_type(type); + return _parameters.size(); } const Parameter* Function::parameter(int idx) const { diff --git a/src/ast/statements/function.hpp b/src/ast/statements/function.hpp index d342614..3456d25 100644 --- a/src/ast/statements/function.hpp +++ b/src/ast/statements/function.hpp @@ -3,15 +3,14 @@ #include +#include "ast/expressions/identifier.hpp" #include "ast/expressions/parameter.hpp" +#include "ast/expressions/type_expression.hpp" #include "ast/statements/statement.hpp" #include "ast/statements/statement_list.hpp" #include "pointer_types.hpp" namespace kore { - class Identifier; - class Type; - class Function : public Statement { public: using body_iterator = StatementList::iterator; @@ -20,20 +19,25 @@ namespace kore { Function(); Function(bool exported); Function(bool exported, const Token& token); + Function( + bool exported, + const std::string& name, + std::vector> _parameters, + std::vector> _return_types, + StatementList body + ); virtual ~Function(); std::string name() const; const Identifier* identifier() const; bool exported() const noexcept; int arity() const; - FunctionType* type(); - const FunctionType* type() const; - void set_type(FunctionType* type); const Parameter* parameter(int idx) const; void add_parameter(Owned&& parameter); /* void set_return_type(const Type* type); */ void add_statement(Owned statement) override; + void add_statements(std::vector>&& statements); Statement* last_statement(); body_iterator begin(); @@ -44,10 +48,8 @@ namespace kore { private: Identifier _name; bool _exported; - FunctionType* _type; - // TODO: Should we remove or duplicate parameter types in function - // type? std::vector> _parameters; + std::vector> _return_types; StatementList _body; std::vector _returns; }; diff --git a/src/ast/statements/if_statement.cpp b/src/ast/statements/if_statement.cpp index 1b25a1d..185fa4b 100644 --- a/src/ast/statements/if_statement.cpp +++ b/src/ast/statements/if_statement.cpp @@ -32,6 +32,10 @@ namespace kore { _statement_accumulator.emplace_back(std::move(statement)); } + void IfStatement::add_statements(std::vector>&& statements) { + _statement_accumulator.insert(_statement_accumulator.end(), statements.begin(), statements.end()); + } + bool IfStatement::has_else_branch() const { return _has_else_branch; } diff --git a/src/ast/statements/if_statement.hpp b/src/ast/statements/if_statement.hpp index 496117b..249cd7a 100644 --- a/src/ast/statements/if_statement.hpp +++ b/src/ast/statements/if_statement.hpp @@ -20,6 +20,7 @@ namespace kore { void add_else_branch(); void add_statement(Owned statement) override; + void add_statements(std::vector>&& statements); bool has_else_branch() const; int branch_count() const; diff --git a/src/bin/korec/main.cpp b/src/bin/korec/main.cpp index 292dfe8..b7f470b 100644 --- a/src/bin/korec/main.cpp +++ b/src/bin/korec/main.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -43,14 +44,17 @@ namespace kore { if (execute) { scanner.scan_string(expr); } else { - if (!scanner.open_file(path)) { + if (!scanner.scan_file(path)) { error_group("scan", "failed to open file: '%s'", path.c_str()); return 1; } } for (auto& token : scanner) { - token.column_format(std::cerr) << std::endl; + std::cerr + << std::setw(20) << std::left + << token.type() << token.value() + << token.location().colon_format(); } return 0; diff --git a/src/compiler/compiler.cpp b/src/compiler/compiler.cpp index 550976f..b81bba2 100644 --- a/src/compiler/compiler.cpp +++ b/src/compiler/compiler.cpp @@ -21,7 +21,7 @@ namespace kore { print_errors(pass, result); return 1; } else { - kore::success_group(1, verbosity, pass.name, ""); + success_group(1, verbosity, pass.name, ""); } if (!result.proceed) { diff --git a/src/utf8/utf8.cpp b/src/utf8/utf8.cpp index c4a2f1e..a54d962 100644 --- a/src/utf8/utf8.cpp +++ b/src/utf8/utf8.cpp @@ -1,9 +1,23 @@ #include "utf8.hpp" -#include - namespace kore { - i32 utf8_decode_string(const std::string& str, DecodeError& error) { + int utf8_codepoint_bytes(char chr) { + std::size_t byte_count = 1; + + if ((chr & 0x80) == 0) { + byte_count = 1; + } else if ((chr & 0xe0) == 0xc0) { + byte_count = 2; + } else if ((chr & 0xf0) == 0xe0) { + byte_count = 3; + } else if ((chr & 0xf8) == 0xf0) { + byte_count = 4; + } + + return byte_count; + } + + codepoint utf8_decode_string(const std::string& str, DecodeError& error) { error = DecodeError::None; if (str.length() < 1 || str.length() > 4) { @@ -54,7 +68,7 @@ namespace kore { return codepoint; } - i32 utf8_decode_string_pos(const std::string& str, std::size_t pos, int& num_bytes, DecodeError& error) { + codepoint utf8_decode_string_pos(const std::string& str, std::size_t pos, int& num_bytes, DecodeError& error) { if (str.empty()) { error = DecodeError::EndOfData; num_bytes = 1; diff --git a/src/utf8/utf8.hpp b/src/utf8/utf8.hpp index 78a72b2..8dac140 100644 --- a/src/utf8/utf8.hpp +++ b/src/utf8/utf8.hpp @@ -14,9 +14,11 @@ namespace kore { EndOfData, }; - i32 utf8_decode_string(const std::string& str, DecodeError& error); + int utf8_codepoint_bytes(char chr); - i32 utf8_decode_string_pos(const std::string& str, std::size_t pos, int& num_bytes, DecodeError& error); + codepoint utf8_decode_string(const std::string& str, DecodeError& error); + + codepoint utf8_decode_string_pos(const std::string& str, std::size_t pos, int& num_bytes, DecodeError& error); } #endif // KORE_UTF8_HPP diff --git a/tests/scanner/test_literals.cpp b/tests/scanner/test_literals.cpp index fec546f..7f1f4a6 100644 --- a/tests/scanner/test_literals.cpp +++ b/tests/scanner/test_literals.cpp @@ -7,7 +7,7 @@ namespace kore { TEST_CASE("Can scan literals", "[literals]") { Scanner scanner; - scanner.open_file("./tests/literals.kore"); + scanner.scan_file("./tests/literals.kore"); expect_token(scanner.next_token(), TokenType::Integer, "0xff", 1, 0, 3); expect_token(scanner.next_token(), TokenType::Integer, "0b1010011", 2, 0, 8); From 2882b122e30475346f37767b0b133f75ea9ba71f Mon Sep 17 00:00:00 2001 From: MisanthropicBit Date: Tue, 14 Jul 2026 20:41:20 +0200 Subject: [PATCH 2/2] More cleanup --- src/ast/expressions/expression.cpp | 1 + src/ast/parser/parser.cpp | 4 +- src/ast/scanner/scanner.cpp | 8 +- src/ast/statements/function.cpp | 7 +- src/ast/statements/function.hpp | 9 +- src/ast/statements/if_statement.cpp | 6 +- src/ast/statements/variable_declaration.cpp | 6 +- src/errors/error.cpp | 13 - src/errors/error.hpp | 51 ---- src/errors/errors.cpp | 237 ------------------ src/errors/errors.hpp | 77 ------ src/logging/color.hpp | 2 +- src/logging/color_attributes.hpp | 2 +- .../bytecode/bytecode_format_writer.hpp | 2 +- .../bytecode/codegen/bytecode_codegen2.hpp | 2 +- src/targets/bytecode/codegen/kir/kir.hpp | 8 +- .../codegen/kir/kir_lowering_pass.cpp | 11 +- src/targets/bytecode/codegen/kir/module.hpp | 2 +- src/targets/bytecode/compiled_object.hpp | 2 +- src/targets/bytecode/module.hpp | 2 +- src/types/scope.cpp | 52 ++-- src/types/scope.hpp | 52 ++-- src/types/type.cpp | 30 --- src/types/type.hpp | 1 - src/types/type_cache.hpp | 2 +- src/types/type_checker.cpp | 6 +- src/types/type_inferrer.cpp | 14 +- tests/test_utils.cpp | 6 +- 28 files changed, 123 insertions(+), 492 deletions(-) delete mode 100644 src/errors/error.cpp delete mode 100644 src/errors/error.hpp delete mode 100644 src/errors/errors.cpp delete mode 100644 src/errors/errors.hpp diff --git a/src/ast/expressions/expression.cpp b/src/ast/expressions/expression.cpp index 2ae727a..f1c5ada 100644 --- a/src/ast/expressions/expression.cpp +++ b/src/ast/expressions/expression.cpp @@ -13,6 +13,7 @@ namespace kore { case ExpressionType::Literal: return os << "literal"; case ExpressionType::Parameter: return os << "parameter"; case ExpressionType::Unary: return os << "unary"; + case ExpressionType::Type: return os << "type"; } } diff --git a/src/ast/parser/parser.cpp b/src/ast/parser/parser.cpp index ff9efb0..177fbd8 100644 --- a/src/ast/parser/parser.cpp +++ b/src/ast/parser/parser.cpp @@ -3,6 +3,7 @@ #include "ast/expressions/array_expression.hpp" #include "ast/expressions/array_fill_expression.hpp" +#include "ast/expressions/binary_expression.hpp" #include "ast/expressions/index_expression.hpp" #include "ast/expressions/array_range_expression.hpp" #include "ast/expressions/bool_expression.hpp" @@ -15,6 +16,7 @@ #include "ast/expressions/type_expression.hpp" #include "ast/parser_error_node.hpp" #include "ast/statements/expression_statement.hpp" +#include "ast/statements/function.hpp" #include "ast/statements/if_statement.hpp" #include "ast/statements/import_statement.hpp" #include "ast/statements/variable_assignment.hpp" @@ -463,7 +465,7 @@ namespace kore { break; } - parameters.push_back(parameter); + parameters.push_back(std::move(parameter)); auto token_type = current_token()->type(); if (expect(TokenType::Comma)) { diff --git a/src/ast/scanner/scanner.cpp b/src/ast/scanner/scanner.cpp index 077efee..9b0af30 100644 --- a/src/ast/scanner/scanner.cpp +++ b/src/ast/scanner/scanner.cpp @@ -197,7 +197,7 @@ namespace kore { return consumed; } - void Scanner::advance(bool allow_newline) { + void Scanner::advance([[maybe_unused]] bool allow_newline) { ++_col; } @@ -308,11 +308,11 @@ namespace kore { return make_one_char_token(TokenType::Invalid); } - if (peek('x', false, 1) || peek('X', false, 1)) { + if (peek('x', 1) || peek('X', 1)) { return scan_hex_number(); - } else if (peek('b', false, 1) || peek('B', false, 1)) { + } else if (peek('b', 1) || peek('B', 1)) { return scan_binary_number(); - } else if (peek('o', false, 1) || peek('O', false, 1)) { + } else if (peek('o', 1) || peek('O', 1)) { return scan_octal_number(); } diff --git a/src/ast/statements/function.cpp b/src/ast/statements/function.cpp index 91175fa..23ae24f 100644 --- a/src/ast/statements/function.cpp +++ b/src/ast/statements/function.cpp @@ -32,10 +32,11 @@ namespace kore { ) : Statement(SourceLocation::unknown, StatementType::Function), _name(name), _exported(exported), - _parameters(parameters), - _return_types(return_types), - _body(body) { + _parameters(std::move(parameters)), + _return_types(std::move(return_types)), + _body(std::move(body)) { } + Function::~Function() {} std::string Function::name() const { diff --git a/src/ast/statements/function.hpp b/src/ast/statements/function.hpp index 3456d25..c7a343f 100644 --- a/src/ast/statements/function.hpp +++ b/src/ast/statements/function.hpp @@ -32,14 +32,19 @@ namespace kore { const Identifier* identifier() const; bool exported() const noexcept; int arity() const; + int return_arity() const; - const Parameter* parameter(int idx) const; void add_parameter(Owned&& parameter); - /* void set_return_type(const Type* type); */ void add_statement(Owned statement) override; void add_statements(std::vector>&& statements); Statement* last_statement(); + Parameter* parameter(int idx); + const Parameter* parameter(int idx) const; + TypeExpression* return_type(int idx); + const TypeExpression* return_type(int idx) const; + + body_iterator begin(); body_iterator end(); diff --git a/src/ast/statements/if_statement.cpp b/src/ast/statements/if_statement.cpp index 185fa4b..ba20a18 100644 --- a/src/ast/statements/if_statement.cpp +++ b/src/ast/statements/if_statement.cpp @@ -33,7 +33,11 @@ namespace kore { } void IfStatement::add_statements(std::vector>&& statements) { - _statement_accumulator.insert(_statement_accumulator.end(), statements.begin(), statements.end()); + _statement_accumulator.insert( + _statement_accumulator.end(), + std::make_move_iterator(statements.begin()), + std::make_move_iterator(statements.end()) + ); } bool IfStatement::has_else_branch() const { diff --git a/src/ast/statements/variable_declaration.cpp b/src/ast/statements/variable_declaration.cpp index 530096c..e458bd7 100644 --- a/src/ast/statements/variable_declaration.cpp +++ b/src/ast/statements/variable_declaration.cpp @@ -8,9 +8,9 @@ namespace kore { _type(type.value()) { _location = SourceLocation( - identifier.location().lnum(), - identifier.location().start(), - type.location().end() + identifier.location().start_lnum(), + identifier.location().start_col(), + type.location().end_col() ); } diff --git a/src/errors/error.cpp b/src/errors/error.cpp deleted file mode 100644 index a920218..0000000 --- a/src/errors/error.cpp +++ /dev/null @@ -1,13 +0,0 @@ -#include "error.hpp" - -namespace kore { - namespace errors { - Error::Error(ErrorType error_type, const std::string& message, const SourceLocation& location, const SourceLocation& end) - : type(error_type), message(message), location(location), end(end) { - } - - Error::Error(const std::string& message, const SourceLocation& location, const SourceLocation& end) - : type(ErrorType::General), message(message), location(location), end(end) { - } - } -} diff --git a/src/errors/error.hpp b/src/errors/error.hpp deleted file mode 100644 index f0a8c73..0000000 --- a/src/errors/error.hpp +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef KORE_ERROR_HPP -#define KORE_ERROR_HPP - -#include "ast/source_location.hpp" - -namespace kore { - namespace errors { - // TODO: Change this to Diagnostic so we can use it as a warning as well - enum class ErrorType { - General, - Scan, - Parser, - Typing, - Codegen, - }; - - struct Error { - Error(ErrorType error_type, const std::string& message, const SourceLocation& location, const SourceLocation& end = SourceLocation::unknown); - Error(const std::string& message, const SourceLocation& location, const SourceLocation& end = SourceLocation::unknown); - - ErrorType type; - std::string message; - - SourceLocation location; - - // Used for e.g. multiple declaration errors to point to the - // previous declaration - SourceLocation end; - }; - - namespace scan { - enum class Errors { - CharacterShouldBeString, - EndOfString, - EofCodepoint, - ExpectEqualAfterBang, - InvalidBinNumber, - InvalidDecNumberDots, - InvalidHexNumber, - InvalidOctNumber, - InvalidUnicode, - UnknownCharacter, - }; - - Error character_should_be_string(std::size_t col, std::size_t lnum); - } - } - -} - -#endif // KORE_ERROR_HPP diff --git a/src/errors/errors.cpp b/src/errors/errors.cpp deleted file mode 100644 index 48e9e84..0000000 --- a/src/errors/errors.cpp +++ /dev/null @@ -1,237 +0,0 @@ -#include -#include - -#include "errors.hpp" -#include "ast/expressions/call.hpp" -#include "ast/expressions/identifier.hpp" -#include "types/function_type.hpp" -/* #include "logging.hpp" */ - -namespace kore { - namespace errors { - /* namespace scan { */ - /* Error character_should_be_string(std::size_t col, std::size_t lnum) { */ - /* auto message = "Character contains more than one character, should be a string?"; */ - - /* return Error(ErrorType::Scan, message, SourceLocation(lnum, col, col)); */ - /* } */ - /* } */ - - namespace typing { - Error cannot_assign(const Type* lhs, const Type* rhs, const SourceLocation& location) { - auto message = "cannot assign expression of type " - + lhs->name() - + " to variable of type " - + rhs->name() - + " without conversion"; - - return Error(ErrorType::Typing, message, location); - } - - Error incompatible_binop(const Type* left, const Type* right, BinOp op, const SourceLocation& location) { - auto message = "cannot use binary operator '" - + binop_to_string(op) + "'" - + " with types " - + left->name() - + " and " - + right->name() - + " without conversion"; - - return Error(ErrorType::Typing, message, location); - } - - Error binop_operand(const Type* type, BinOp op, const std::string& which, const SourceLocation& location) { - std::string types_string = operand_types_string(op); - - auto message = "binary operator '" - + binop_to_string(op) - + "' must have " + types_string + " operands but got " - + type->name() + " for " + which + " argument"; - - return Error(ErrorType::Typing, message, location); - } - - Error variable_shadows(const Identifier* identifier, const Identifier* shadowed, const SourceLocation& location, const SourceLocation& prev_location) { - std::string message = "variable '" + identifier->name() + "' at (" + location.colon_format() + ") shadows variable "; - - if (shadowed) { - message += "'" + shadowed->name() + "' (at " + prev_location.colon_format() + ")"; - } else { - message += "in outer scope"; - } - - return Error(ErrorType::Typing, message, location); - } - - Error redeclaration_constant_variable(const Identifier& identifier, const SourceLocation& location, const Identifier& prev_declared) { - std::ostringstream oss; - - oss << "redeclaration of constant variable '" << identifier.name() << "', previously declared here: " << prev_declared.location(); - - return Error(ErrorType::Typing, oss.str(), location); - } - - Error cannot_declare_mutable_global(const Identifier& identifier, const SourceLocation& location) { - auto message = "cannot declare global mutable variables ('" + identifier.name() + "')"; - - return Error(ErrorType::Typing, message, location); - } - - Error cannot_assign_global_variable(const Identifier* identifier, const Identifier* shadowed, const SourceLocation& location, const SourceLocation& prev_location) { - std::string message = "assignment of '" + identifier->name() + "' at (" + location.colon_format() + ") cannot assign to global variable '" + shadowed->name() + "' previously declared at (" + std::to_string(prev_location.lnum()) + ")"; - - return Error(ErrorType::Typing, message, location); - } - - Error undefined_variable(const Identifier& identifier) { - return Error(ErrorType::Typing, "use of undefined variable " + identifier.name(), identifier.location()); - } - - Error unknown_call(const Call& call) { - return Error(ErrorType::Typing, "unknown function called: " + call.name(), call.location()); - } - - Error incorrect_parameter_type(const Expression* expression, const Type* arg_type, const Type* param_type, Call& call, int arg_index) { - std::ostringstream oss; - - oss << "call to function " << call.name() << " " - << "expected type of argument " << (arg_index + 1) << " to be " - << param_type->name() - << " but got " << arg_type->name() << " instead"; - - return Error(ErrorType::Typing, oss.str(), expression->location()); - } - - Error not_a_function(const Call& call, const Type* type) { - std::string message = "expected type of " + - call.name() + " in call to be " + call.expected_func_type_name() + - ", but got " + type->name(); - - return Error(ErrorType::Typing, message, call.location()); - } - - Error incorrect_arg_count(const Call& call, const Type* type) { - std::ostringstream oss; - - // FIX: - oss << "incorrect argument count for " - << call.name() - << ": expected " << call.arg_count() - << ", but got " << type->name(); - - return Error(ErrorType::Typing, oss.str(), call.location()); - } - - // Error return_type_mismatch(const Function* func, const Type* type, const SourceLocation& location) { - // auto message = "Trying to return " + type->name() + " from function '" + func->name() + "' returning " + func->return_type()->name(); - // - // return Error(ErrorType::Typing, message, location); - // } - - Error void_return_from_nonvoid_function(const Function* func, const SourceLocation& location) { - auto message = "Trying to return void from non-void function '" + func->name() + "'"; - - return Error(ErrorType::Typing, message, location); - } - } - - namespace kir { - Error moved_variable(Identifier& expr) { - return Error("cannot reference moved variable", expr.location()); - } - } - } - - std::string format_locations(const SourceLocation& start) { - std::ostringstream oss; - oss << start.lnum() << ":" << start.start(); - - if (!start.is_single_pos()) { - oss << "-" << start.end(); - } - - return oss.str(); - } - - /* void output_error( */ - /* const errors::Error& error, */ - /* const std::string& source_name */ - /* ) { */ - /* error_indent( */ - /* "[%s:%s]: %s", */ - /* source_name.c_str(), */ - /* format_locations(error.location), */ - /* error.message.c_str() */ - /* ); */ - /* } */ - - std::string format_error_at_line( - const std::string& line, - std::size_t start_col, - std::size_t end_col - ) { - std::ostringstream oss; - - const std::string indent = "|" + std::string(4, ' '); - - if (start_col == end_col) { - oss << indent << line << std::endl - << indent << std::setw(end_col + 1) << "^"; - } else { - oss << indent << line << std::endl - << indent - << std::string(start_col + 1, ' ') - << "^" - << std::setfill('-') - << std::string(end_col - start_col, ' ') - << "^"; - } - - return oss.str(); - } - - std::string format_error_at_line(const std::string& line, const SourceLocation& location) { - return format_error_at_line(line, location.start(), location.end()); - } - - std::string format_error( - const std::string& msg, - const std::string& line, - std::size_t lnum, - std::size_t start_col, - std::size_t end_col - ) { - std::ostringstream oss; - - if (start_col == end_col) { - oss << msg << " (line " << lnum << ", column " << end_col << ")" - << std::endl; - } else { - oss << msg << " (line " << lnum - << ", columns " << (start_col + 1) << "-" << (end_col + 1) << ")" - << std::endl; - } - - format_error_at_line(line, start_col, end_col); - - return oss.str(); - } - - std::string format_error( - const std::string& msg, - const std::string& line, - const SourceLocation& location - ) { - return format_error(msg, line, location.lnum(), location.start(), location.end()); - } - - void throw_error_for_line( - const std::string& msg, - const std::string& line, - std::size_t lnum, - std::size_t start_col, - std::size_t end_col - ) { - throw std::runtime_error(format_error(msg, line, lnum, start_col, end_col)); - } -} diff --git a/src/errors/errors.hpp b/src/errors/errors.hpp deleted file mode 100644 index e41061e..0000000 --- a/src/errors/errors.hpp +++ /dev/null @@ -1,77 +0,0 @@ -#ifndef KORE_ERRORS_HPP -#define KORE_ERRORS_HPP - -#include - -#include "ast/statements/function.hpp" -#include "errors/error.hpp" -#include "ast/expressions/binary_expression.hpp" -#include "ast/source_location.hpp" -#include "types/type.hpp" - -namespace kore { - class Call; - - namespace errors { - namespace typing { - Error cannot_assign(const Type* lhs, const Type* rhs, const SourceLocation& location); - Error incompatible_binop(const Type* left, const Type* right, BinOp op, const SourceLocation& location); - Error binop_operand(const Type* type, BinOp op, const std::string& which, const SourceLocation& location); - Error variable_shadows(const Identifier* identifier, const Identifier* shadowed, const SourceLocation& location, const SourceLocation& prev_location); - Error redeclaration_constant_variable(const Identifier& identifier, const SourceLocation& location, const Identifier& prev_declared); - Error cannot_declare_mutable_global(const Identifier& identifier, const SourceLocation& location); - Error cannot_assign_global_variable(const Identifier* identifier, const Identifier* shadowed, const SourceLocation& location, const SourceLocation& prev_location); - Error undefined_variable(const Identifier& identifier); - Error unknown_call(const class Call& call); - Error incorrect_parameter_type(const Expression* expression, const Type* arg_type, const Type* param_type, class Call& call, int arg_index); - Error not_a_function(const class Call& call, const Type* type); - Error incorrect_arg_count(const class Call& call, const Type* type); - // Error return_type_mismatch(const Function* func, const Type* type, const SourceLocation& location); - Error void_return_from_nonvoid_function(const Function* func, const SourceLocation& location); - } - - namespace kir { - Error moved_variable(Identifier& expr); - } - } - - std::string format_locations(const SourceLocation& start, const SourceLocation& end); - - /* void output_error( */ - /* const errors::Error& error, */ - /* const std::string& source_name */ - /* ); */ - - std::string format_error_at_line( - const std::string& line, - std::size_t lnum, - std::size_t start_col, - std::size_t end_col - ); - - std::string format_error_at_line(const std::string& line, const SourceLocation& location); - - std::string format_error( - const std::string& msg, - const std::string& line, - std::size_t lnum, - std::size_t start_col, - std::size_t end_col - ); - - std::string format_error( - const std::string& msg, - const std::string& line, - const SourceLocation& location - ); - - void throw_error_for_line( - const std::string& msg, - const std::string& line, - std::size_t lnum, - std::size_t start_col, - std::size_t end_col - ); -} - -#endif // KORE_ERRORS_HPP diff --git a/src/logging/color.hpp b/src/logging/color.hpp index 9150fe7..53ec64d 100644 --- a/src/logging/color.hpp +++ b/src/logging/color.hpp @@ -44,7 +44,7 @@ namespace kore { public: Color(); Color(AnsiColorCode ansi_code); - virtual ~Color(); + ~Color(); bool is_background() const noexcept; diff --git a/src/logging/color_attributes.hpp b/src/logging/color_attributes.hpp index a5b6214..4a27ca8 100644 --- a/src/logging/color_attributes.hpp +++ b/src/logging/color_attributes.hpp @@ -22,7 +22,7 @@ namespace kore { public: ColorAttribute(); ColorAttribute(AnsiAttributeCode ansi_code); - virtual ~ColorAttribute(); + ~ColorAttribute(); static const ColorAttribute None; static const ColorAttribute Reset; diff --git a/src/targets/bytecode/bytecode_format_writer.hpp b/src/targets/bytecode/bytecode_format_writer.hpp index 0f93146..25aa568 100644 --- a/src/targets/bytecode/bytecode_format_writer.hpp +++ b/src/targets/bytecode/bytecode_format_writer.hpp @@ -21,7 +21,7 @@ namespace kore { class BytecodeFormatWriter final { public: BytecodeFormatWriter(); - virtual ~BytecodeFormatWriter(); + ~BytecodeFormatWriter(); // TODO: Make it possible to write this to a bytecode stream that // can be directly executed by the vm or should the vm just be able diff --git a/src/targets/bytecode/codegen/bytecode_codegen2.hpp b/src/targets/bytecode/codegen/bytecode_codegen2.hpp index 74c85e9..22ddd90 100644 --- a/src/targets/bytecode/codegen/bytecode_codegen2.hpp +++ b/src/targets/bytecode/codegen/bytecode_codegen2.hpp @@ -29,7 +29,7 @@ namespace kore { public: BytecodeGenerator2(); - virtual ~BytecodeGenerator2(); + ~BytecodeGenerator2(); std::vector generate(kir::Kir& kir); diff --git a/src/targets/bytecode/codegen/kir/kir.hpp b/src/targets/bytecode/codegen/kir/kir.hpp index 97aee13..a84f640 100644 --- a/src/targets/bytecode/codegen/kir/kir.hpp +++ b/src/targets/bytecode/codegen/kir/kir.hpp @@ -1,12 +1,7 @@ #ifndef KORE_KIR_HPP #define KORE_KIR_HPP -#include "ast/ast.hpp" -#include "ast/ast_visitor.hpp" -#include "errors/errors.hpp" -#include "targets/bytecode/codegen/kir/function.hpp" #include "targets/bytecode/codegen/kir/module.hpp" -#include "types/scope.hpp" namespace kore { namespace kir { @@ -18,7 +13,7 @@ namespace kore { public: Kir(); - virtual ~Kir(); + ~Kir(); int globals_count(); /* const ConstantTable& i32_constant_table(); */ @@ -41,4 +36,3 @@ namespace kore { } #endif // KORE_KIR_HPP - diff --git a/src/targets/bytecode/codegen/kir/kir_lowering_pass.cpp b/src/targets/bytecode/codegen/kir/kir_lowering_pass.cpp index a80eb54..71ded4d 100644 --- a/src/targets/bytecode/codegen/kir/kir_lowering_pass.cpp +++ b/src/targets/bytecode/codegen/kir/kir_lowering_pass.cpp @@ -175,7 +175,7 @@ namespace kore { Reg dst; auto& func = current_function(); - auto entry = _scope_stack.find_inner(expr.name()); + auto entry = _scope_stack.find_local(expr.name()); if (!entry) { // NOTE: Is it always ok to directly use the rhs register? @@ -321,7 +321,7 @@ namespace kore { auto user_func = _functions[call.name()]; func_index = user_func.func_index; - return_register_count = user_func.func->type()->return_arity(); + return_register_count = user_func.func->return_arity(); } Regs return_registers = func.allocate_registers(return_register_count); @@ -386,7 +386,7 @@ namespace kore { void KirLoweringPass::enter_function(kore::Function& func) { add_kir_function(&func); - _scope_stack.enter_function_scope(&func); + _scope_stack.enter_function(&func); for (int i = 0; i < func.arity(); ++i) { auto parameter = func.parameter(i); @@ -396,7 +396,7 @@ namespace kore { void KirLoweringPass::exit_function() { _func_index_stack.pop(); - _scope_stack.leave_function_scope(); + _scope_stack.leave_function(); } void KirLoweringPass::add_kir_function(kore::Function* function) { @@ -421,12 +421,11 @@ namespace kore { return pop_register(); } - void KirLoweringPass::check_register_state(Identifier& expr, Reg reg) { + void KirLoweringPass::check_register_state([[maybe_unused]] Identifier& expr, Reg reg) { auto& func = current_function(); if (func.register_state(reg) != RegisterState::Available) { // TODO: Move push_error into a separate class or Error class - /*push_error(*/errors::kir::moved_variable(expr);/*);*/ } } diff --git a/src/targets/bytecode/codegen/kir/module.hpp b/src/targets/bytecode/codegen/kir/module.hpp index 18ae2e0..084c1df 100644 --- a/src/targets/bytecode/codegen/kir/module.hpp +++ b/src/targets/bytecode/codegen/kir/module.hpp @@ -20,7 +20,7 @@ namespace kore { public: Module(ModuleIndex index, const fs::path& path); - virtual ~Module(); + ~Module(); std::string path() const; ModuleIndex index() const noexcept; diff --git a/src/targets/bytecode/compiled_object.hpp b/src/targets/bytecode/compiled_object.hpp index f4cd1d1..bada25c 100644 --- a/src/targets/bytecode/compiled_object.hpp +++ b/src/targets/bytecode/compiled_object.hpp @@ -35,7 +35,7 @@ namespace kore { int reg_count, const std::vector& instructions ); - virtual ~CompiledObject(); + ~CompiledObject(); std::string name() const; int func_index() const; diff --git a/src/targets/bytecode/module.hpp b/src/targets/bytecode/module.hpp index c6da8d0..8f0ba80 100644 --- a/src/targets/bytecode/module.hpp +++ b/src/targets/bytecode/module.hpp @@ -38,7 +38,7 @@ namespace kore { Module(); Module(Module&& module) = default; Module(ModuleIndex idx, const fs::path& path); - virtual ~Module(); + ~Module(); Version get_compiler_version(); Version get_bytecode_version(); diff --git a/src/types/scope.cpp b/src/types/scope.cpp index b3ad219..26e0602 100644 --- a/src/types/scope.cpp +++ b/src/types/scope.cpp @@ -1,11 +1,13 @@ #include "types/scope.hpp" +#include "ast/expressions/identifier.hpp" +#include "ast/expressions/parameter.hpp" namespace kore { - bool ScopeEntry::is_global_scope() const { + bool Symbol::is_global_scope() const { return level == 1; } - ScopeEntry* Scope::find(const std::string& name) { + Symbol* Scope::find(const std::string& name) { auto entry = _map.find(name); return entry != _map.end() ? &entry->second : nullptr; @@ -26,7 +28,7 @@ namespace kore { _scopes.emplace_back(false); } - void ScopeStack::enter_function_scope(Function* func) { + void ScopeStack::enter_function(Function* func) { _scopes.emplace_back(true); _functions.push_back(func); } @@ -35,22 +37,22 @@ namespace kore { _scopes.pop_back(); } - void ScopeStack::leave_function_scope() { + void ScopeStack::leave_function() { _scopes.pop_back(); _functions.pop_back(); } - ScopeEntry* ScopeStack::find(const std::string& name) { + Symbol* ScopeStack::find(const std::string& name) { return find_in_range(name, levels(), 1); } - ScopeEntry* ScopeStack::find_inner(const std::string& name) { + Symbol* ScopeStack::find_local(const std::string& name) { int inner_scope = levels(); return find_in_range(name, inner_scope, inner_scope); } - ScopeEntry* ScopeStack::find_enclosing(const std::string& name) { + Symbol* ScopeStack::find_enclosing(const std::string& name) { return find_in_range(name, levels() - 1, 2); } @@ -63,15 +65,15 @@ namespace kore { } void ScopeStack::insert(const Identifier* identifier, Reg reg) { - auto& active_scope = _scopes.back(); + insert(reg, identifier->name(), identifier->location(), identifier->type()); + } - auto entry = ScopeEntry { - reg, - static_cast(_scopes.size()), - identifier, - }; + void ScopeStack::insert(const Parameter* parameter) { + insert(parameter, -1); + } - active_scope._map.emplace(identifier->name(), entry); + void ScopeStack::insert(const Parameter* parameter, Reg reg) { + insert(reg, parameter->name(), parameter->location(), parameter->type()); } bool ScopeStack::is_global_scope() const { @@ -83,7 +85,27 @@ namespace kore { enter(); } - ScopeEntry* ScopeStack::find_in_range( + void ScopeStack::insert( + Reg reg, + const std::string& name, + const SourceLocation& location, + const Type* type + ) { + auto& active_scope = _scopes.back(); + + auto symbol = Symbol { + static_cast(_scopes.size()), + reg, + name, + location, + type, + false + }; + + active_scope._map.emplace(name, symbol); + } + + Symbol* ScopeStack::find_in_range( const std::string& name, int start_lvl, int end_lvl diff --git a/src/types/scope.hpp b/src/types/scope.hpp index ebafed5..f14fa79 100644 --- a/src/types/scope.hpp +++ b/src/types/scope.hpp @@ -1,58 +1,63 @@ -#ifndef KORE_SCOPES_HPP -#define KORE_SCOPES_HPP +#ifndef KORE_SCOPE_HPP +#define KORE_SCOPE_HPP #include #include -#include "ast/expressions/identifier.hpp" #include "ast/statements/function.hpp" #include "targets/bytecode/register.hpp" namespace kore { - /// An entry for each variable with additional bookkeeping - struct ScopeEntry { - Reg reg; + /// A symbol defined in some level of scope (destructed parts of an + /// identifier and a function parameter in order to support both) + struct Symbol { int level; - const Identifier* identifier; + Reg reg; + std::string name; + SourceLocation location; + const Type* type; + bool constant; bool is_global_scope() const; }; namespace { - using ScopeMap = std::map; + using SymbolMap = std::map; struct Scope { Scope(bool func_scope_start) : func_scope_start(func_scope_start) {} - ScopeEntry* find(const std::string& name); + Symbol* find(const std::string& name); bool func_scope_start; - ScopeMap _map; + SymbolMap _map; }; } - // TODO: Convert to a linked list of scopes instead? /// A (lexical) scope keeps track of all variables at each level class ScopeStack final { + // TODO: Convert to a linked list of scopes instead? public: ScopeStack(); - virtual ~ScopeStack(); + ~ScopeStack(); int levels() const; void enter(); - void enter_function_scope(Function* func); + void enter_function(Function* func); void leave(); - void leave_function_scope(); - ScopeEntry* find(const std::string& name); - ScopeEntry* find_inner(const std::string& name); - ScopeEntry* find_enclosing(const std::string& name); + void leave_function(); + Symbol* find(const std::string& name); + Symbol* find_local(const std::string& name); + Symbol* find_enclosing(const std::string& name); Function* enclosing_function(); void insert(const Identifier* identifier); void insert(const Identifier* identifier, Reg reg); + void insert(const Parameter* parameter); + void insert(const Parameter* parameter, Reg reg); bool is_global_scope() const; - bool in_function_scope() const; + bool is_function_scope() const; void clear(); private: @@ -60,7 +65,14 @@ namespace kore { std::vector _scopes; std::vector _functions; - ScopeEntry* find_in_range( + void insert( + Reg reg, + const std::string& name, + const SourceLocation& location, + const Type* type + ); + + Symbol* find_in_range( const std::string& name, int start_lvl, int end_lvl @@ -68,4 +80,4 @@ namespace kore { }; } -#endif // KORE_SCOPES_HPP +#endif // KORE_SCOPE_HPP diff --git a/src/types/type.cpp b/src/types/type.cpp index e9d9280..6a913f3 100644 --- a/src/types/type.cpp +++ b/src/types/type.cpp @@ -185,34 +185,4 @@ namespace kore { const Type* Type::get_type_from_category(TypeCategory category) { return _type_cache.get_type(category); } - - const Type* Type::from_token(const Token& token) { - if (!token.is_type()) { - throw std::runtime_error("Cannot create type from non-type token '%s'"); - } - - if (token.value() == "i32") { - return _type_cache.get_type(TypeCategory::Integer32); - } else if (token.value() == "i64") { - return _type_cache.get_type(TypeCategory::Integer64); - } else if (token.value() == "f32") { - return _type_cache.get_type(TypeCategory::Float32); - } else if (token.value() == "f64") { - return _type_cache.get_type(TypeCategory::Float64); - } else if (token.value() == "byte") { - return _type_cache.get_type(TypeCategory::Byte); - } else if (token.value() == "char") { - return _type_cache.get_type(TypeCategory::Char); - } else if (token.value() == "str") { - return _type_cache.get_type(TypeCategory::Str); - } else if (token.value() == "bool") { - return _type_cache.get_type(TypeCategory::Bool); - } else if (token.value() == "void") { - return Type::void_type(); - } else if (token.value() == "unknown") { - return Type::unknown(); - } - - return Type::unknown(); - } } diff --git a/src/types/type.hpp b/src/types/type.hpp index de07705..314b56a 100644 --- a/src/types/type.hpp +++ b/src/types/type.hpp @@ -58,7 +58,6 @@ namespace kore { ); static void set_function_type(std::unique_ptr&& func_type); static const Type* get_type_from_category(TypeCategory category); - static const Type* from_token(const Token& token); template const T* as() const { diff --git a/src/types/type_cache.hpp b/src/types/type_cache.hpp index d6ad6e8..4ec9661 100644 --- a/src/types/type_cache.hpp +++ b/src/types/type_cache.hpp @@ -21,7 +21,7 @@ namespace kore { public: TypeCache(); - virtual ~TypeCache(); + ~TypeCache(); const Type* get_type(TypeCategory category); ArrayType* get_array_type(const Type* element_type); diff --git a/src/types/type_checker.cpp b/src/types/type_checker.cpp index e6763ad..3f58d2b 100644 --- a/src/types/type_checker.cpp +++ b/src/types/type_checker.cpp @@ -327,11 +327,11 @@ namespace kore { } void TypeChecker::visit(Function& func) { - trace_type_checker("function", func.type()); + trace_type_checker("function"); // Enter a new function scope and add all function // arguments to that scope - _scope_stack.enter_function_scope(&func); + _scope_stack.enter_function(&func); for (int i = 0; i < func.arity(); ++i) { auto parameter = func.parameter(i); @@ -342,7 +342,7 @@ namespace kore { statement->accept(*this); } - _scope_stack.leave_function_scope(); + _scope_stack.leave_function(); // After leaving the function scope, bind the function type // to the function name (not necessarily in the top-level scope diff --git a/src/types/type_inferrer.cpp b/src/types/type_inferrer.cpp index 9520a30..de805c6 100644 --- a/src/types/type_inferrer.cpp +++ b/src/types/type_inferrer.cpp @@ -70,10 +70,10 @@ namespace kore { } void TypeInferrer::visit(class Call& call) { - auto entry = _scope_stack.find(call.name()); + auto symbol = _scope_stack.find(call.name()); - if (entry) { - auto type = entry->identifier->type(); + if (symbol) { + auto type = symbol->type; if (type->is_function()) { auto func_type = static_cast(type); @@ -97,7 +97,7 @@ namespace kore { // If no identifier was found, this is an undefined variable which is // caught by the type checker if (entry) { - expr.set_type(entry->identifier->type()); + expr.set_type(entry->type); } else { expr.set_type(Type::unknown()); } @@ -157,11 +157,11 @@ namespace kore { /* } */ void TypeInferrer::visit(Function& func) { - trace_type_inference("function " + func.name(), func.type()); + trace_type_inference("function " + func.name()); // Enter a new function scope and add all function // arguments to that scope - _scope_stack.enter_function_scope(&func); + _scope_stack.enter_function(&func); // TODO: Move into enter_function_scope? for (int i = 0; i < func.arity(); ++i) { @@ -173,7 +173,7 @@ namespace kore { statement->accept(*this); } - _scope_stack.leave_function_scope(); + _scope_stack.leave_function(); // After leaving the function scope, bind the function type // to the function name (not necessarily in the top-level scope diff --git a/tests/test_utils.cpp b/tests/test_utils.cpp index 6f6638b..8f08dbf 100644 --- a/tests/test_utils.cpp +++ b/tests/test_utils.cpp @@ -14,9 +14,9 @@ namespace kore { REQUIRE(token.value() == value); if (!token.is_eof()) { - REQUIRE(token.location().lnum() == lnum); - REQUIRE(token.location().start() == start); - REQUIRE(token.location().end() == end); + REQUIRE(token.location().start_lnum() == lnum); + REQUIRE(token.location().start_col() == start); + REQUIRE(token.location().end_col() == end); } } }