Parser.cpp

Go to the documentation of this file.

#include <Ark/Compiler/AST/Parser.hpp>
 
#include <fmt/core.h>
 
namespace Ark::internal
{
    Parser::Parser(const unsigned debug, const ParserMode mode) :
        BaseParser(), m_mode(mode), m_logger("Parser", debug),
        m_ast(NodeType::List), m_imports({}), m_allow_macro_behavior(0),
        m_nested_nodes(0)
    {
        m_ast.push_back(Node(Keyword::Begin));
 
        m_parsers = {
            [this](FilePosition) {
                return wrapped(&Parser::letMutSet, "variable assignment or declaration");
            },
            [this](FilePosition) {
                return wrapped(&Parser::function, "function");
            },
            [this](FilePosition) {
                return wrapped(&Parser::condition, "condition");
            },
            [this](FilePosition) {
                return wrapped(&Parser::loop, "loop");
            },
            [this](const FilePosition filepos) {
                return import_(filepos);
            },
            [this](const FilePosition filepos) {
                return block(filepos);
            },
            [this](FilePosition) {
                return wrapped(&Parser::macroCondition, "$if");
            },
            [this](const FilePosition filepos) {
                return macro(filepos);
            },
            [this](FilePosition) {
                return wrapped(&Parser::del, "del");
            },
            [this](const FilePosition filepos) {
                return functionCall(filepos);
            },
            [this](const FilePosition filepos) {
                return list(filepos);
            }
        };
    }
 
    void Parser::process(const std::string& filename, const std::string& code)
    {
        m_logger.traceStart("process");
        initParser(filename, code);
 
        while (!isEOF())
        {
            std::string comment = newlineOrComment();
            if (isEOF())
            {
                if (!comment.empty())
                    m_ast.list().back().attachCommentAfter(comment);
                break;
            }
 
            const auto pos = getCount();
            if (auto n = node())
            {
                m_ast.push_back(n->attachNearestCommentBefore(n->comment() + comment));
                m_ast.list().back().attachCommentAfter(spaceComment());
            }
            else
            {
                backtrack(pos);
                std::string out = peek();
                std::string message;
                if (out == ")")
                    message = "Unexpected closing paren";
                else if (out == "}")
                    message = "Unexpected closing bracket";
                else if (out == "]")
                    message = "Unexpected closing square bracket";
                else
                    errorWithNextToken("invalid syntax, expected node");
                errorWithNextToken(message);
            }
        }
 
        m_logger.traceEnd();
    }
 
    const Node& Parser::ast() const noexcept
    {
        return m_ast;
    }
 
    const std::vector<Import>& Parser::imports() const
    {
        return m_imports;
    }
 
    Node Parser::positioned(Node node, const FilePosition cursor) const
    {
        const auto [row, col] = cursor;
        const auto [end_row, end_col] = getCursor();
 
        node.m_filename = m_filename;
        node.m_pos = FileSpan {
            .start = FilePos { .line = row, .column = col },
            .end = FilePos { .line = end_row, .column = end_col }
        };
        return node;
    }
 
    std::optional<Node>& Parser::positioned(std::optional<Node>& node, const FilePosition cursor) const
    {
        if (!node)
            return node;
 
        const auto [row, col] = cursor;
        const auto [end_row, end_col] = getCursor();
 
        node->m_filename = m_filename;
        node->m_pos = FileSpan {
            .start = FilePos { .line = row, .column = col },
            .end = FilePos { .line = end_row, .column = end_col }
        };
        return node;
    }
 
    std::optional<Node> Parser::node()
    {
        ++m_nested_nodes;
 
        if (m_nested_nodes > MaxNestedNodes)
            errorWithNextToken(fmt::format("Too many nested node while parsing, exceeds limit of {}. Consider rewriting your code by breaking it in functions and macros.", MaxNestedNodes));
 
        // save current position in buffer to be able to go back if needed
        const auto position = getCount();
        const auto filepos = getCursor();
        std::optional<Node> result = std::nullopt;
 
        for (auto&& parser : m_parsers)
        {
            result = parser(filepos);
 
            if (result)
                break;
            backtrack(position);
        }
 
        // return std::nullopt only on parsing error, nothing matched, the user provided terrible code
        --m_nested_nodes;
        return result;
    }
 
    std::optional<Node> Parser::letMutSet(const FilePosition filepos)
    {
        std::optional<Node> leaf { NodeType::List };
 
        std::string token;
        if (!oneOf({ "let", "mut", "set" }, &token))
            return std::nullopt;
 
        std::string comment = newlineOrComment();
        leaf->attachNearestCommentBefore(comment);
 
        if (token == "let")
            leaf->push_back(Node(Keyword::Let));
        else if (token == "mut")
            leaf->push_back(Node(Keyword::Mut));
        else  // "set"
            leaf->push_back(Node(Keyword::Set));
 
        if (m_allow_macro_behavior > 0)
        {
            const auto position = getCount();
            const auto value_pos = getCursor();
            if (const auto value = nodeOrValue(); value.has_value())
            {
                const Node& sym = value.value();
                if (sym.nodeType() == NodeType::List || sym.nodeType() == NodeType::Symbol || sym.nodeType() == NodeType::Macro || sym.nodeType() == NodeType::Spread)
                    leaf->push_back(sym);
                else
                    error(fmt::format("Can not use a {} as a symbol name, even in a macro", nodeTypes[static_cast<std::size_t>(sym.nodeType())]), value_pos);
            }
            else
                backtrack(position);
        }
 
        if (leaf->constList().size() == 1)
        {
            // we haven't parsed anything while in "macro state"
            std::string symbol_name;
            const auto value_pos = getCursor();
            if (!name(&symbol_name))
                errorWithNextToken(token + " needs a symbol");
 
            leaf->push_back(positioned(Node(NodeType::Symbol, symbol_name), value_pos));
        }
 
        comment = newlineOrComment();
        if (auto value = nodeOrValue(); value.has_value())
            leaf->push_back(value.value().attachNearestCommentBefore(comment));
        else
            errorWithNextToken("Expected a value");
 
        return positioned(leaf, filepos);
    }
 
    std::optional<Node> Parser::del(const FilePosition filepos)
    {
        std::optional<Node> leaf { NodeType::List };
 
        if (!oneOf({ "del" }))
            return std::nullopt;
        leaf->push_back(Node(Keyword::Del));
 
        const std::string comment = newlineOrComment();
 
        std::string symbol_name;
        if (!name(&symbol_name))
            errorWithNextToken("del needs a symbol");
 
        leaf->push_back(Node(NodeType::Symbol, symbol_name));
        leaf->list().back().attachNearestCommentBefore(comment);
 
        return positioned(leaf, filepos);
    }
 
    std::optional<Node> Parser::condition(const FilePosition filepos)
    {
        std::optional<Node> leaf { NodeType::List };
 
        if (!oneOf({ "if" }))
            return std::nullopt;
 
        std::string comment = newlineOrComment();
 
        leaf->push_back(Node(Keyword::If));
 
        if (auto cond_expr = nodeOrValue(); cond_expr.has_value())
            leaf->push_back(cond_expr.value().attachNearestCommentBefore(comment));
        else
            errorWithNextToken("`if' needs a valid condition");
 
        comment = newlineOrComment();
        if (auto value_if_true = nodeOrValue(); value_if_true.has_value())
            leaf->push_back(value_if_true.value().attachNearestCommentBefore(comment));
        else
            errorWithNextToken("Expected a node or value after condition");
 
        comment = newlineOrComment();
        if (auto value_if_false = nodeOrValue(); value_if_false.has_value())
        {
            leaf->push_back(value_if_false.value().attachNearestCommentBefore(comment));
            leaf->list().back().attachCommentAfter(newlineOrComment());
        }
        else if (!comment.empty())
            leaf->attachCommentAfter(comment);
 
        return positioned(leaf, filepos);
    }
 
    std::optional<Node> Parser::loop(const FilePosition filepos)
    {
        std::optional<Node> leaf { NodeType::List };
 
        if (!oneOf({ "while" }))
            return std::nullopt;
 
        std::string comment = newlineOrComment();
        leaf->push_back(Node(Keyword::While));
 
        if (auto cond_expr = nodeOrValue(); cond_expr.has_value())
            leaf->push_back(cond_expr.value().attachNearestCommentBefore(comment));
        else
            errorWithNextToken("`while' needs a valid condition");
 
        comment = newlineOrComment();
        if (auto body = nodeOrValue(); body.has_value())
            leaf->push_back(body.value().attachNearestCommentBefore(comment));
        else
            errorWithNextToken("Expected a node or value after loop condition");
 
        return positioned(leaf, filepos);
    }
 
    std::optional<Node> Parser::import_(const FilePosition filepos)
    {
        std::optional<Node> leaf { NodeType::List };
 
        auto context = generateErrorContextAtCurrentPosition();
        if (!accept(IsChar('(')))
            return std::nullopt;
 
        std::string comment = newlineOrComment();
        leaf->attachNearestCommentBefore(comment);
 
        if (!oneOf({ "import" }))
            return std::nullopt;
 
        comment = newlineOrComment();
        leaf->push_back(Node(Keyword::Import));
 
        Import import_data;
        import_data.col = filepos.col;
        import_data.line = filepos.row;
 
        const auto pos = getCount();
        if (!packageName(&import_data.prefix))
            errorWithNextToken("Import expected a package name");
 
        if (import_data.prefix.size() > 255)
        {
            backtrack(pos);
            errorWithNextToken(fmt::format("Import name too long, expected at most 255 characters, got {}", import_data.prefix.size()));
        }
        import_data.package.push_back(import_data.prefix);
 
        Node packageNode = positioned(Node(NodeType::List), getCursor()).attachNearestCommentBefore(comment);
        packageNode.push_back(Node(NodeType::Symbol, import_data.prefix));
 
        // first, parse the package name
        while (!isEOF())
        {
            const auto item_pos = getCursor();
 
            // parsing package folder.foo.bar.yes
            if (accept(IsChar('.')))
            {
                const auto package_pos = getCursor();
                std::string path;
                if (!packageName(&path))
                    errorWithNextToken("Package name expected after '.'");
                else
                {
                    packageNode.push_back(positioned(Node(NodeType::Symbol, path), package_pos));
 
                    import_data.package.push_back(path);
                    import_data.prefix = path;  // in the end we will store the last element of the package, which is what we want
 
                    if (path.size() > 255)
                    {
                        backtrack(pos);
                        errorWithNextToken(fmt::format("Import name too long, expected at most 255 characters, got {}", path.size()));
                    }
                }
            }
            else if (accept(IsChar(':')) && accept(IsChar('*')))  // parsing :*, terminal in imports
            {
                leaf->push_back(packageNode);
                leaf->push_back(positioned(Node(NodeType::Symbol, "*"), item_pos));
 
                space();
                expectSuffixOrError(')', fmt::format("in import `{}'", import_data.toPackageString()), context);
 
                // save the import data structure to know we encounter an import node, and retrieve its data more easily later on
                import_data.with_prefix = false;
                import_data.is_glob = true;
                m_imports.push_back(import_data);
 
                return positioned(leaf, filepos);
            }
            else
                break;
        }
 
        Node symbols = positioned(Node(NodeType::List), getCursor());
        // then parse the symbols to import, if any
        if (space())
        {
            comment = newlineOrComment();
 
            while (!isEOF())
            {
                if (accept(IsChar(':')))  // parsing potential :a :b :c
                {
                    const auto symbol_pos = getCursor();
                    std::string symbol_name;
                    if (!name(&symbol_name))
                        errorWithNextToken("Expected a valid symbol to import");
                    if (symbol_name == "*")
                        error(fmt::format("Glob patterns can not be separated from the package, use (import {}:*) instead", import_data.toPackageString()), symbol_pos);
 
                    if (symbol_name.size() >= 2 && symbol_name[symbol_name.size() - 2] == ':' && symbol_name.back() == '*')
                        error("Glob pattern can not follow a symbol to import", FilePosition { .row = symbol_pos.row, .col = symbol_pos.col + symbol_name.size() - 2 });
 
                    symbols.push_back(positioned(Node(NodeType::Symbol, symbol_name).attachNearestCommentBefore(comment), symbol_pos));
                    comment.clear();
 
                    import_data.symbols.push_back(symbol_name);
                    // we do not need the prefix when importing specific symbols
                    import_data.with_prefix = false;
                }
 
                if (!space())
                    break;
                comment = newlineOrComment();
            }
 
            if (!comment.empty() && !symbols.list().empty())
                symbols.list().back().attachCommentAfter(comment);
        }
 
        leaf->push_back(packageNode);
        leaf->push_back(symbols);
        // save the import data
        m_imports.push_back(import_data);
 
        comment = newlineOrComment();
        if (!comment.empty())
            leaf->list().back().attachCommentAfter(comment);
 
        expectSuffixOrError(')', fmt::format("in import `{}'", import_data.toPackageString()), context);
        return positioned(leaf, filepos);
    }
 
    std::optional<Node> Parser::block(const FilePosition filepos)
    {
        std::optional<Node> leaf { NodeType::List };
 
        auto context = generateErrorContextAtCurrentPosition();
        bool alt_syntax = false;
        std::string comment;
        if (accept(IsChar('(')))
        {
            comment = newlineOrComment();
            if (!oneOf({ "begin" }))
                return std::nullopt;
        }
        else if (accept(IsChar('{')))
            alt_syntax = true;
        else
            return std::nullopt;
 
        leaf->setAltSyntax(alt_syntax);
        leaf->push_back(Node(Keyword::Begin).attachNearestCommentBefore(comment));
 
        comment = newlineOrComment();
 
        while (!isEOF())
        {
            if (auto value = nodeOrValue(); value.has_value())
            {
                leaf->push_back(value.value().attachNearestCommentBefore(comment));
                comment = newlineOrComment();
            }
            else
                break;
        }
 
        comment += newlineOrComment();
        expectSuffixOrError(alt_syntax ? '}' : ')', "to close block", context);
        leaf->list().back().attachCommentAfter(comment);
        return positioned(leaf, filepos);
    }
 
    std::optional<Node> Parser::functionArgs(const FilePosition filepos)
    {
        expect(IsChar('('));
        std::optional<Node> args { NodeType::List };
 
        std::string comment = newlineOrComment();
        args->attachNearestCommentBefore(comment);
 
        bool has_captures = false;
 
        while (!isEOF())
        {
            const auto pos = getCursor();
            if (accept(IsChar('&')))  // captures
            {
                has_captures = true;
                std::string capture;
                if (!name(&capture))
                    error("No symbol provided to capture", pos);
 
                args->push_back(positioned(Node(NodeType::Capture, capture), pos));
            }
            else if (accept(IsChar('(')))
            {
                // attribute modifiers: mut, ref
                std::string modifier;
                std::ignore = newlineOrComment();
                if (!oneOf({ "mut", "ref" }, &modifier))
                    // We cannot return an error like this:
                    //   error("Expected an attribute modifier, either `mut' or `ref'", pos);
                    // Because it would break on macro instantiations like (fun ((suffix-dup a 3)) ())
                    return std::nullopt;
 
                NodeType type = NodeType::Unused;
                if (modifier == "mut")
                    type = NodeType::MutArg;
                else if (modifier == "ref")
                    type = NodeType::RefArg;
 
                Node arg_with_attr = Node(type);
                std::string comment2 = newlineOrComment();
                arg_with_attr.attachCommentAfter(comment2);
 
                std::string symbol_name;
                if (!name(&symbol_name))
                    error(fmt::format("Expected a symbol name for the attribute with modifier `{}'", modifier), pos);
                arg_with_attr.setString(symbol_name);
 
                args->push_back(positioned(arg_with_attr, pos));
                std::ignore = newlineOrComment();
                expect(IsChar(')'));
            }
            else
            {
                std::string symbol_name;
                if (!name(&symbol_name))
                    break;
                if (has_captures)
                    error("Captured variables should be at the end of the argument list", pos);
 
                args->push_back(positioned(Node(NodeType::Symbol, symbol_name), pos));
            }
 
            if (!comment.empty())
                args->list().back().attachNearestCommentBefore(comment);
            comment = newlineOrComment();
        }
 
        if (accept(IsChar(')')))
            return positioned(args, filepos);
        return std::nullopt;
    }
 
    std::optional<Node> Parser::function(const FilePosition filepos)
    {
        std::optional<Node> leaf { NodeType::List };
 
        if (!oneOf({ "fun" }))
            return std::nullopt;
        leaf->push_back(Node(Keyword::Fun));
 
        const std::string comment_before_args = newlineOrComment();
 
        while (m_allow_macro_behavior > 0)
        {
            const auto position = getCount();
 
            // args
            if (const auto value = nodeOrValue(); value.has_value())
            {
                // if value is nil, just add an empty argument bloc to prevent bugs when
                // declaring functions inside macros
                const Node& args = value.value();
                if (args.nodeType() == NodeType::Symbol && args.string() == "nil")
                    leaf->push_back(Node(NodeType::List));
                else
                    leaf->push_back(args);
            }
            else
            {
                backtrack(position);
                break;
            }
 
            const std::string comment = newlineOrComment();
            // body
            if (auto value = nodeOrValue(); value.has_value())
                leaf->push_back(value.value().attachNearestCommentBefore(comment));
            else
                errorWithNextToken("Expected a body for the function");
            return positioned(leaf, filepos);
        }
 
        const auto position = getCount();
        const auto args_file_pos = getCursor();
        if (auto args = functionArgs(args_file_pos); args.has_value())
            leaf->push_back(args.value().attachNearestCommentBefore(comment_before_args));
        else
        {
            backtrack(position);
 
            if (auto value = nodeOrValue(); value.has_value())
                leaf->push_back(value.value().attachNearestCommentBefore(comment_before_args));
            else
                errorWithNextToken("Expected an argument list");
        }
 
        const std::string comment = newlineOrComment();
 
        if (auto value = nodeOrValue(); value.has_value())
            leaf->push_back(value.value().attachNearestCommentBefore(comment));
        else
            errorWithNextToken("Expected a body for the function");
 
        return positioned(leaf, filepos);
    }
 
    std::optional<Node> Parser::macroCondition(const FilePosition filepos)
    {
        std::optional<Node> leaf { NodeType::Macro };
 
        if (!oneOf({ "$if" }))
            return std::nullopt;
        leaf->push_back(Node(Keyword::If));
 
        std::string comment = newlineOrComment();
        leaf->attachNearestCommentBefore(comment);
 
        if (const auto cond_expr = nodeOrValue(); cond_expr.has_value())
            leaf->push_back(cond_expr.value());
        else
            errorWithNextToken("$if need a valid condition");
 
        comment = newlineOrComment();
        if (auto value_if_true = nodeOrValue(); value_if_true.has_value())
            leaf->push_back(value_if_true.value().attachNearestCommentBefore(comment));
        else
            errorWithNextToken("Expected a node or value after condition");
 
        comment = newlineOrComment();
        if (auto value_if_false = nodeOrValue(); value_if_false.has_value())
        {
            leaf->push_back(value_if_false.value().attachNearestCommentBefore(comment));
            comment = newlineOrComment();
            leaf->list().back().attachCommentAfter(comment);
        }
 
        return positioned(leaf, filepos);
    }
 
    std::optional<Node> Parser::macroArgs(const FilePosition filepos)
    {
        if (!accept(IsChar('(')))
            return std::nullopt;
 
        std::optional<Node> args { NodeType::List };
 
        std::string comment = newlineOrComment();
        args->attachNearestCommentBefore(comment);
 
        std::vector<std::string> names;
        while (!isEOF())
        {
            const auto pos = getCount();
 
            std::string arg_name;
            if (!name(&arg_name))
                break;
 
            comment = newlineOrComment();
            args->push_back(Node(NodeType::Symbol, arg_name).attachNearestCommentBefore(comment));
 
            if (std::ranges::find(names, arg_name) != names.end())
            {
                backtrack(pos);
                errorWithNextToken(fmt::format("Argument names must be unique, can not reuse `{}'", arg_name));
            }
            names.push_back(arg_name);
        }
 
        const auto pos = getCount();
        if (sequence("..."))
        {
            std::string spread_name;
            if (!name(&spread_name))
                errorWithNextToken("Expected a name for the variadic arguments list");
 
            args->push_back(Node(NodeType::Spread, spread_name));
            args->list().back().attachCommentAfter(newlineOrComment());
 
            if (std::ranges::find(names, spread_name) != names.end())
            {
                backtrack(pos);
                errorWithNextToken(fmt::format("Argument names must be unique, can not reuse `{}'", spread_name));
            }
        }
 
        if (!accept(IsChar(')')))
            return std::nullopt;
 
        comment = newlineOrComment();
        if (!comment.empty())
            args->attachCommentAfter(comment);
 
        return positioned(args, filepos);
    }
 
    std::optional<Node> Parser::macro(const FilePosition filepos)
    {
        std::optional<Node> leaf { NodeType::Macro };
 
        auto context = generateErrorContextAtCurrentPosition();
        if (!accept(IsChar('(')))
            return std::nullopt;
 
        if (!oneOf({ "macro" }))
            return std::nullopt;
        std::string comment = newlineOrComment();
        leaf->attachNearestCommentBefore(comment);
 
        std::string symbol_name;
        if (!name(&symbol_name))
            errorWithNextToken("Expected a symbol to declare a macro");
 
        comment = newlineOrComment();
        leaf->push_back(Node(NodeType::Symbol, symbol_name).attachNearestCommentBefore(comment));
 
        const auto position = getCount();
        const auto args_file_pos = getCursor();
        if (const auto args = macroArgs(args_file_pos); args.has_value())
            leaf->push_back(args.value());
        else
        {
            // if we couldn't parse arguments, then we have a value
            backtrack(position);
 
            ++m_allow_macro_behavior;
            const auto value = nodeOrValue();
            --m_allow_macro_behavior;
 
            if (value.has_value())
                leaf->push_back(value.value());
            else
                errorWithNextToken(fmt::format("Expected an argument list, atom or node while defining macro `{}'", symbol_name));
 
            leaf->list().back().attachCommentAfter(newlineOrComment());
            expectSuffixOrError(')', fmt::format("to close macro `{}'", symbol_name), context);
            return positioned(leaf, filepos);
        }
 
        ++m_allow_macro_behavior;
        const auto value = nodeOrValue();
        --m_allow_macro_behavior;
 
        if (value.has_value())
            leaf->push_back(value.value());
        else if (leaf->list().size() == 2)  // the argument list is actually a function call and it's okay
        {
            leaf->list().back().attachCommentAfter(newlineOrComment());
 
            expectSuffixOrError(')', fmt::format("to close macro `{}'", symbol_name), context);
            return positioned(leaf, filepos);
        }
        else
        {
            backtrack(position);
            errorWithNextToken(fmt::format("Expected a value while defining macro `{}'", symbol_name), context);
        }
 
        leaf->list().back().attachCommentAfter(newlineOrComment());
 
        expectSuffixOrError(')', fmt::format("to close macro `{}'", symbol_name), context);
        return positioned(leaf, filepos);
    }
 
    std::optional<Node> Parser::functionCall(const FilePosition filepos)
    {
        auto context = generateErrorContextAtCurrentPosition();
        if (!accept(IsChar('(')))
            return std::nullopt;
        std::string comment = newlineOrComment();
 
        const auto func_name_pos = getCursor();
        std::optional<Node> func;
        if (auto sym_or_field = anyAtomOf({ NodeType::Symbol, NodeType::Field }); sym_or_field.has_value())
            func = sym_or_field->attachNearestCommentBefore(comment);
        else if (auto nested = node(); nested.has_value())
            func = nested->attachNearestCommentBefore(comment);
        else
            return std::nullopt;
 
        if (func.value().nodeType() == NodeType::Symbol && func.value().string() == "ref")
            error("`ref' can not be used outside a function's arguments list.", func_name_pos);
 
        std::optional<Node> leaf { NodeType::List };
        leaf->push_back(positioned(func.value(), func_name_pos));
 
        comment = newlineOrComment();
 
        while (!isEOF())
        {
            if (auto arg = nodeOrValue(); arg.has_value())
            {
                leaf->push_back(arg.value().attachNearestCommentBefore(comment));
                comment = newlineOrComment();
            }
            else
                break;
        }
 
        leaf->list().back().attachCommentAfter(comment);
        comment = newlineOrComment();
        if (!comment.empty())
            leaf->list().back().attachCommentAfter(comment);
 
        expectSuffixOrError(')', fmt::format("in function call to `{}'", func.value().repr()), context);
        return positioned(leaf, filepos);
    }
 
    std::optional<Node> Parser::list(const FilePosition filepos)
    {
        std::optional<Node> leaf { NodeType::List };
 
        auto context = generateErrorContextAtCurrentPosition();
        if (!accept(IsChar('[')))
            return std::nullopt;
        leaf->setAltSyntax(true);
        leaf->push_back(Node(NodeType::Symbol, "list"));
 
        std::string comment = newlineOrComment();
        leaf->attachNearestCommentBefore(comment);
 
        while (!isEOF())
        {
            if (auto value = nodeOrValue(); value.has_value())
            {
                leaf->push_back(value.value().attachNearestCommentBefore(comment));
                comment = newlineOrComment();
            }
            else
                break;
        }
        leaf->list().back().attachCommentAfter(comment);
 
        expectSuffixOrError(']', "to end list definition", context);
        return positioned(leaf, filepos);
    }
 
    std::optional<Node> Parser::number(const FilePosition filepos)
    {
        std::string res;
        if (signedNumber(&res))
        {
            double output;
            if (Utils::isDouble(res, &output))
                return positioned(Node(output), filepos);
 
            error("Is not a valid number", filepos);
        }
        return std::nullopt;
    }
 
    std::optional<Node> Parser::string(const FilePosition filepos)
    {
        std::string res;
        if (accept(IsChar('"')))
        {
            while (true)
            {
                const auto pos = getCursor();
 
                if (accept(IsChar('\\')))
                {
                    if (m_mode != ParserMode::Interpret)
                        res += '\\';
 
                    if (accept(IsChar('"')))
                        res += '"';
                    else if (accept(IsChar('\\')))
                        res += '\\';
                    else if (accept(IsChar('n')))
                        res += m_mode == ParserMode::Interpret ? '\n' : 'n';
                    else if (accept(IsChar('t')))
                        res += m_mode == ParserMode::Interpret ? '\t' : 't';
                    else if (accept(IsChar('v')))
                        res += m_mode == ParserMode::Interpret ? '\v' : 'v';
                    else if (accept(IsChar('r')))
                        res += m_mode == ParserMode::Interpret ? '\r' : 'r';
                    else if (accept(IsChar('a')))
                        res += m_mode == ParserMode::Interpret ? '\a' : 'a';
                    else if (accept(IsChar('b')))
                        res += m_mode == ParserMode::Interpret ? '\b' : 'b';
                    else if (accept(IsChar('f')))
                        res += m_mode == ParserMode::Interpret ? '\f' : 'f';
                    else if (accept(IsChar('u')))
                    {
                        std::string seq;
                        if (hexNumber(4, &seq))
                        {
                            if (m_mode == ParserMode::Interpret)
                            {
                                char utf8_str[5];
                                utf8::decode(seq.c_str(), utf8_str);
                                if (*utf8_str == '\0')
                                    error("Invalid escape sequence", pos);
                                res += utf8_str;
                            }
                            else
                                res += "u" + seq;
                        }
                        else
                            error("Invalid escape sequence, expected 4 hex digits: \\uabcd", pos);
                    }
                    else if (accept(IsChar('U')))
                    {
                        std::string seq;
                        if (hexNumber(8, &seq))
                        {
                            if (m_mode == ParserMode::Interpret)
                            {
                                std::size_t begin = 0;
                                for (; seq[begin] == '0'; ++begin)
                                    ;
                                char utf8_str[5];
                                utf8::decode(seq.c_str() + begin, utf8_str);
                                if (*utf8_str == '\0')
                                    error("Invalid escape sequence", pos);
                                res += utf8_str;
                            }
                            else
                                res += "U" + seq;
                        }
                        else
                            error("Invalid escape sequence, expected 8 hex digits: \\UABCDEF78", pos);
                    }
                    else
                    {
                        backtrack(getCount() - 1);
                        error("Unknown escape sequence", pos);
                    }
                }
                else
                    accept(IsNot(IsEither(IsChar('\\'), IsChar('"'))), &res);
 
                if (accept(IsChar('"')))
                    break;
                if (isEOF())
                    expectSuffixOrError('"', "after string");
            }
 
            return positioned(Node(NodeType::String, res), filepos);
        }
        return std::nullopt;
    }
 
    std::optional<Node> Parser::field(const FilePosition filepos)
    {
        std::string sym;
        if (!name(&sym))
            return std::nullopt;
 
        std::optional<Node> leaf { Node(NodeType::Field) };
        leaf->push_back(Node(NodeType::Symbol, sym));
 
        while (true)
        {
            if (leaf->list().size() == 1 && !accept(IsChar('.')))  // Symbol:abc
                return std::nullopt;
 
            if (leaf->list().size() > 1 && !accept(IsChar('.')))
                break;
 
            const auto filepos_inner = getCursor();
            std::string res;
            if (!name(&res))
                errorWithNextToken("Expected a field name: <symbol>.<field>");
            leaf->push_back(positioned(Node(NodeType::Symbol, res), filepos_inner));
        }
 
        return positioned(leaf, filepos);
    }
 
    std::optional<Node> Parser::symbol(const FilePosition filepos)
    {
        std::string res;
        if (!name(&res))
            return std::nullopt;
        return positioned(Node(NodeType::Symbol, res), filepos);
    }
 
    std::optional<Node> Parser::spread(const FilePosition filepos)
    {
        std::string res;
        if (sequence("..."))
        {
            if (!name(&res))
                errorWithNextToken("Expected a name for the variadic");
            return positioned(Node(NodeType::Spread, res), filepos);
        }
        return std::nullopt;
    }
 
    std::optional<Node> Parser::nil(const FilePosition filepos)
    {
        if (!accept(IsChar('(')))
            return std::nullopt;
 
        const std::string comment = newlineOrComment();
        if (!accept(IsChar(')')))
            return std::nullopt;
 
        if (m_mode == ParserMode::Interpret)
            return positioned(Node(NodeType::Symbol, "nil").attachNearestCommentBefore(comment), filepos);
        return positioned(Node(NodeType::List).attachNearestCommentBefore(comment), filepos);
    }
 
    std::optional<Node> Parser::atom()
    {
        const auto pos = getCount();
        const auto filepos = getCursor();
 
        if (auto res = Parser::number(filepos); res.has_value())
            return res;
        backtrack(pos);
 
        if (auto res = Parser::string(filepos); res.has_value())
            return res;
        backtrack(pos);
 
        if (auto res = Parser::spread(filepos); m_allow_macro_behavior > 0 && res.has_value())
            return res;
        backtrack(pos);
 
        if (auto res = Parser::field(filepos); res.has_value())
            return res;
        backtrack(pos);
 
        if (auto res = Parser::symbol(filepos); res.has_value())
            return res;
        backtrack(pos);
 
        if (auto res = Parser::nil(filepos); res.has_value())
            return res;
        backtrack(pos);
 
        return std::nullopt;
    }
 
    std::optional<Node> Parser::anyAtomOf(const std::initializer_list<NodeType> types)
    {
        if (auto value = atom(); value.has_value())
        {
            for (const auto type : types)
            {
                if (value->nodeType() == type)
                    return value;
            }
        }
        return std::nullopt;
    }
 
    std::optional<Node> Parser::nodeOrValue()
    {
        if (auto value = atom(); value.has_value())
            return value;
        if (auto sub_node = node(); sub_node.has_value())
            return sub_node;
 
        return std::nullopt;
    }
 
    std::optional<Node> Parser::wrapped(std::optional<Node> (Parser::*parser)(FilePosition), const std::string& name)
    {
        const auto cursor = getCursor();
        auto context = generateErrorContextAtCurrentPosition();
        if (!prefix('('))
            return std::nullopt;
 
        const std::string comment = newlineOrComment();
 
        if (auto result = (this->*parser)(cursor); result.has_value())
        {
            result->attachNearestCommentBefore(result->comment() + comment);
            result.value().attachCommentAfter(newlineOrComment());
 
            if (name == "function")
                expectSuffixOrError(')', "after function body. Did you forget to wrap the body with `{}'?", context);
            else
                expectSuffixOrError(')', "after " + name, context);
 
            result.value().attachCommentAfter(spaceComment());
            return result;
        }
 
        return std::nullopt;
    }
}