diff --git a/dang-utils/include/dang-utils/parsing/lexer.h b/dang-utils/include/dang-utils/parsing/lexer.h new file mode 100644 index 00000000..1a1bd861 --- /dev/null +++ b/dang-utils/include/dang-utils/parsing/lexer.h @@ -0,0 +1,258 @@ +#pragma once + +#include +#include +#include +#include + +namespace dang::utils { + +class LexerError : public std::runtime_error { + using std::runtime_error::runtime_error; +}; + +// The Lexer concept: +// - using Char = ...; +// - using Token = ...; +// - Lexer(std::basic_string_view) +// - std::optional next() + +template +struct LexerToken { + using Char = TChar; + using TextView = std::basic_string_view; + + TextView text; +}; + +/// @brief Tokenizes a series of characters one by one. +template +class BasicLexer { +public: + using Char = TChar; + using Token = LexerToken; + + using TextView = std::basic_string_view; + + constexpr explicit BasicLexer(TextView text) + : text_(text) + {} + + constexpr TextView textView() const { return text_; } + + constexpr std::optional next() + { + if (text_.empty()) + return std::nullopt; + auto token = text_.substr(0, 1); + text_.remove_prefix(1); + return Token{token}; + } + +private: + TextView text_; +}; + +/// @brief Tokenizes a series of UTF-8 code units one by one. +class Utf8Lexer { +public: + using Char = char; // TODO: C++20 char8_t + using Token = LexerToken; + + using TextView = std::basic_string_view; + + constexpr explicit Utf8Lexer(TextView text) + : text_(text) + , has_bom_(scanBOM()) + {} + + constexpr bool hasBOM() const { return has_bom_; } + + constexpr TextView textView() const { return text_; } + + constexpr std::optional next() + { + if (text_.empty()) + return std::nullopt; + + std::size_t code_unit_length = [&] { + auto code_point = text_.front(); + if ((code_point & 0b1000'0000) == 0b0000'0000) + return 1; + if ((code_point & 0b1110'0000) == 0b1100'0000) + return 2; + if ((code_point & 0b1111'0000) == 0b1110'0000) + return 3; + if ((code_point & 0b1111'1000) == 0b1111'0000) + return 4; + throw LexerError("Invalid initial UTF-8 code point."); + }(); + + if (text_.size() < code_unit_length) + throw LexerError("Incomplete UTF-8 code unit."); + + for (std::size_t i = 1; i < code_unit_length; i++) { + if ((text_[i] & 0b1100'0000) != 0b1000'0000) + throw LexerError("Invalid UTF-8 code point."); + } + + auto result = text_.substr(0, code_unit_length); + text_.remove_prefix(code_unit_length); + return Token{result}; + } + +private: + constexpr bool scanBOM() + { + if (text_[0] != '\xEF') + return false; + if (text_.size() < 3 || text_[1] != '\xBB' || text_[2] != '\xBF') + throw LexerError("Invalid UTF-8 BOM."); + text_.remove_prefix(3); + return true; + } + + TextView text_; + bool has_bom_; +}; + +namespace lex { + +/// @brief Reads a single char from the char lexer if it matches. +template > +struct Char : LexerToken { + using Base = LexerToken; + using CharLexer = TCharLexer; + + static_assert(std::is_same_v); + + static constexpr std::optional match(CharLexer& char_lexer) + { + auto next_char = char_lexer.next(); + if (!next_char || next_char->text.size() != 1 || next_char->text.front() != v_char) + return std::nullopt; + return Char{next_char->text}; + } +}; + +/// @brief Reads chars as long as the predicate holds true. +/// @remark Allows both TextView and plain chars in the predicate. +template , typename = void> +struct TakeWhile; + +template +struct TakeWhile>> + : LexerToken { + using Base = LexerToken; + using CharLexer = TCharLexer; + + static constexpr std::optional match(CharLexer& char_lexer) + { + auto original_text = char_lexer.textView(); + std::size_t count = 0; + while (true) { + auto previous_lexer = char_lexer; + auto next_char = char_lexer.next(); + if (!next_char || !v_predicate(next_char->text)) { + char_lexer = previous_lexer; + break; + } + count += next_char->text.size(); + } + if (count == 0) + return std::nullopt; + return TakeWhile{original_text.substr(0, count)}; + } +}; + +namespace detail { + +/// @brief Turns a bool(char) predicate into a bool(TextView) predicate. +template +constexpr bool singleCharValidator(std::basic_string_view c) +{ + return c.size() == 1 && v_predicate(c.front()); +} + +} // namespace detail + +template +struct TakeWhile>> + : TakeWhile, TCharLexer> { + using Base = TakeWhile, TCharLexer>; + using CharLexer = TCharLexer; + + static constexpr std::optional match(CharLexer& char_lexer) + { + auto result = Base::match(char_lexer); + if (result) + return TakeWhile{result->text}; + return std::nullopt; + } +}; + +/// @brief Reads a single char from the char lexer. +template > +struct Any : LexerToken { + using Base = LexerToken; + using CharLexer = TCharLexer; + + static constexpr std::optional match(CharLexer& char_lexer) + { + auto next_char = char_lexer.next(); + if (!next_char) + return std::nullopt; + return Any{next_char->text}; + } +}; + +} // namespace lex + +/// @brief A lexer that can processes the given variant of tokens. +template > +class AutoLexer; + +template +class AutoLexer, TCharLexer> { +public: + using CharLexer = TCharLexer; + using Char = typename TCharLexer::Char; + using Token = std::variant; + + using TextView = typename CharLexer::TextView; + + constexpr explicit AutoLexer(TextView text) + : char_lexer_(text) + {} + + constexpr CharLexer charLexer() const { return char_lexer_; } + + constexpr std::optional next() { return tryTokens(); } + +private: + template + constexpr std::optional tryTokens() + { + auto old_lexer = char_lexer_; + auto maybe_token = TFirst::match(char_lexer_); + if (!maybe_token) { + char_lexer_ = old_lexer; + if constexpr (sizeof...(TRest) > 0) + return tryTokens(); + else { + if (char_lexer_.next()) + throw LexerError("Invalid lexer token."); + return std::nullopt; + } + } + return *maybe_token; + } + + CharLexer char_lexer_; +}; + +} // namespace dang::utils diff --git a/dang-utils/tests/CMakeLists.txt b/dang-utils/tests/CMakeLists.txt index 3f36097a..36eed0d6 100644 --- a/dang-utils/tests/CMakeLists.txt +++ b/dang-utils/tests/CMakeLists.txt @@ -5,7 +5,7 @@ find_package(Catch2 CONFIG REQUIRED) include(Catch) -add_executable(${PROJECT_NAME} test-event.cpp test-stub.cpp) +add_executable(${PROJECT_NAME} parsing/test-lexer.cpp test-event.cpp test-parsing.cpp test-stub.cpp) target_precompile_headers(${PROJECT_NAME} PRIVATE ) diff --git a/dang-utils/tests/parsing/test-lexer.cpp b/dang-utils/tests/parsing/test-lexer.cpp new file mode 100644 index 00000000..1a6e1f02 --- /dev/null +++ b/dang-utils/tests/parsing/test-lexer.cpp @@ -0,0 +1,215 @@ +#include +#include +#include +#include +#include + +#include "dang-utils/parsing/lexer.h" + +#include "catch2/catch.hpp" + +namespace dutils = dang::utils; +using Catch::Matchers::Message; +using dutils::lex::Any; +using dutils::lex::Char; +using dutils::lex::TakeWhile; + +TEST_CASE("The basic lexer uses characters as tokens.", "[lexer]") +{ + SECTION("It returns all characters one by one.") + { + auto lexer = dutils::BasicLexer<>("true"); + CHECK(lexer.next().value().text == "t"); + CHECK(lexer.next().value().text == "r"); + CHECK(lexer.next().value().text == "u"); + CHECK(lexer.next().value().text == "e"); + CHECK_FALSE(lexer.next()); + CHECK_FALSE(lexer.next()); + } + SECTION("It can be used constexpr.") + { + constexpr auto tokens = [] { + auto lexer = dutils::BasicLexer<>("const"); + return std::array{ + lexer.next(), lexer.next(), lexer.next(), lexer.next(), lexer.next(), lexer.next(), lexer.next()}; + }(); + STATIC_REQUIRE(tokens[0].value().text == "c"); + STATIC_REQUIRE(tokens[1].value().text == "o"); + STATIC_REQUIRE(tokens[2].value().text == "n"); + STATIC_REQUIRE(tokens[3].value().text == "s"); + STATIC_REQUIRE(tokens[4].value().text == "t"); + STATIC_REQUIRE_FALSE(tokens[5]); + STATIC_REQUIRE_FALSE(tokens[6]); + } +} + +TEST_CASE("The UTF-8 lexer uses UTF-8 code points as tokens.", "[lexer]") +{ + // TODO: C++20 will need u8 literals. + SECTION("It can lex ASCII.") + { + auto lexer = dutils::Utf8Lexer("true"); + CHECK_FALSE(lexer.hasBOM()); + CHECK(lexer.next().value().text == "t"); + CHECK(lexer.next().value().text == "r"); + CHECK(lexer.next().value().text == "u"); + CHECK(lexer.next().value().text == "e"); + CHECK_FALSE(lexer.next()); + CHECK_FALSE(lexer.next()); + } + SECTION("It can lex Japanese code points.") + { + auto lexer = dutils::Utf8Lexer("ごきげんよう"); + CHECK_FALSE(lexer.hasBOM()); + CHECK(lexer.next().value().text == "ご"); + CHECK(lexer.next().value().text == "き"); + CHECK(lexer.next().value().text == "げ"); + CHECK(lexer.next().value().text == "ん"); + CHECK(lexer.next().value().text == "よ"); + CHECK(lexer.next().value().text == "う"); + CHECK_FALSE(lexer.next()); + CHECK_FALSE(lexer.next()); + } + SECTION("It can lex a combination of ASCII and Japanese.") + { + auto lexer = dutils::Utf8Lexer("AあIいUうEえOお"); + CHECK_FALSE(lexer.hasBOM()); + CHECK(lexer.next().value().text == "A"); + CHECK(lexer.next().value().text == "あ"); + CHECK(lexer.next().value().text == "I"); + CHECK(lexer.next().value().text == "い"); + CHECK(lexer.next().value().text == "U"); + CHECK(lexer.next().value().text == "う"); + CHECK(lexer.next().value().text == "E"); + CHECK(lexer.next().value().text == "え"); + CHECK(lexer.next().value().text == "O"); + CHECK(lexer.next().value().text == "お"); + CHECK_FALSE(lexer.next()); + CHECK_FALSE(lexer.next()); + } + SECTION("It skips a potential UTF-8 BOM.") + { + SECTION("Only BOM.") + { + auto lexer = dutils::Utf8Lexer("\xEF\xBB\xBF"); + CHECK(lexer.hasBOM()); + CHECK_FALSE(lexer.next()); + CHECK_FALSE(lexer.next()); + } + SECTION("BOM with one character.") + { + auto lexer = dutils::Utf8Lexer("\xEF\xBB\xBFX"); + CHECK(lexer.hasBOM()); + CHECK(lexer.next().value().text == "X"); + CHECK_FALSE(lexer.next()); + CHECK_FALSE(lexer.next()); + } + SECTION("BOM with ASCII.") + { + auto lexer = dutils::Utf8Lexer("\xEF\xBB\xBFhi"); + CHECK(lexer.hasBOM()); + CHECK(lexer.next().value().text == "h"); + CHECK(lexer.next().value().text == "i"); + CHECK_FALSE(lexer.next()); + CHECK_FALSE(lexer.next()); + } + SECTION("BOM with Japanese.") + { + auto lexer = dutils::Utf8Lexer("\xEF\xBB\xBFはい"); + CHECK(lexer.hasBOM()); + CHECK(lexer.next().value().text == "は"); + CHECK(lexer.next().value().text == "い"); + CHECK_FALSE(lexer.next()); + CHECK_FALSE(lexer.next()); + } + } + SECTION("It can be used constexpr.") + { + constexpr auto lexer_and_tokens = [] { + auto lexer = dutils::Utf8Lexer("\xEF\xBB\xBFコンスト"); + return std::tuple{ + lexer, std::array{lexer.next(), lexer.next(), lexer.next(), lexer.next(), lexer.next(), lexer.next()}}; + }(); + constexpr auto lexer = std::get<0>(lexer_and_tokens); + constexpr auto tokens = std::get<1>(lexer_and_tokens); + STATIC_REQUIRE(lexer.hasBOM()); + STATIC_REQUIRE(tokens[0].value().text == "コ"); + STATIC_REQUIRE(tokens[1].value().text == "ン"); + STATIC_REQUIRE(tokens[2].value().text == "ス"); + STATIC_REQUIRE(tokens[3].value().text == "ト"); + STATIC_REQUIRE_FALSE(tokens[4]); + STATIC_REQUIRE_FALSE(tokens[5]); + } + SECTION("It throws a LexerError for invalid UTF-8 code points.") + { + auto lex = [](auto text) { dutils::Utf8Lexer(text).next(); }; + + CHECK_THROWS_MATCHES(lex("\x80"), dutils::LexerError, Message("Invalid initial UTF-8 code point.")); + CHECK_THROWS_MATCHES(lex("\xC0\x01"), dutils::LexerError, Message("Invalid UTF-8 code point.")); + CHECK_THROWS_MATCHES(lex("\xC0"), dutils::LexerError, Message("Incomplete UTF-8 code unit.")); + CHECK_THROWS_MATCHES(lex("\xE0\x80\x01"), dutils::LexerError, Message("Invalid UTF-8 code point.")); + CHECK_THROWS_MATCHES(lex("\xE0\x80"), dutils::LexerError, Message("Incomplete UTF-8 code unit.")); + CHECK_THROWS_MATCHES(lex("\xF0\x80\x80\x01"), dutils::LexerError, Message("Invalid UTF-8 code point.")); + CHECK_THROWS_MATCHES(lex("\xF0\x80\x80"), dutils::LexerError, Message("Incomplete UTF-8 code unit.")); + } +} + +constexpr bool isWhitespace(char c) { return c == ' '; } +constexpr bool isAlpha(char c) { return c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z'; } + +using WhitespaceToken = TakeWhile; +using CommaToken = Char<','>; +using StringToken = TakeWhile; +using InvalidToken = Any<>; + +using StringListToken = std::variant; +using StringListLexer = dutils::AutoLexer; + +using StringListTokenWithInvalid = std::variant; +using StringListLexerWithInvalid = dutils::AutoLexer; + +TEST_CASE("The auto lexer uses a variant of lexer tokens.", "[lexer]") +{ + SECTION("It can process a valid series of tokens.") + { + auto lexer = StringListLexer("Hello, World"); + CHECK(std::get(lexer.next().value()).text == "Hello"); + CHECK(std::get(lexer.next().value()).text == ","); + CHECK(std::get(lexer.next().value()).text == " "); + CHECK(std::get(lexer.next().value()).text == "World"); + CHECK_FALSE(lexer.next()); + CHECK_FALSE(lexer.next()); + } + SECTION("It skips over invalid characters, throwing a LexerError in the process.") + { + auto lexer = StringListLexer("Hello 7,"); + CHECK(std::get(lexer.next().value()).text == "Hello"); + CHECK(std::get(lexer.next().value()).text == " "); + CHECK_THROWS_MATCHES(lexer.next(), dutils::LexerError, Message("Invalid lexer token.")); + CHECK(std::get(lexer.next().value()).text == ","); + CHECK_FALSE(lexer.next()); + CHECK_FALSE(lexer.next()); + } + SECTION("To avoid exceptions, a final 'Any' token can be used instead.") + { + auto lexer = StringListLexerWithInvalid("Hello 7,"); + CHECK(std::get(lexer.next().value()).text == "Hello"); + CHECK(std::get(lexer.next().value()).text == " "); + CHECK(std::get(lexer.next().value()).text == "7"); + CHECK(std::get(lexer.next().value()).text == ","); + CHECK_FALSE(lexer.next()); + CHECK_FALSE(lexer.next()); + } + SECTION("It can be used constexpr as long as all tokens are constexpr.") + { + constexpr auto tokens = [] { + auto lexer = StringListLexer("Hello, World"); + return std::array{lexer.next(), lexer.next(), lexer.next(), lexer.next(), lexer.next()}; + }(); + STATIC_REQUIRE(std::get(tokens[0].value()).text == "Hello"); + STATIC_REQUIRE(std::get(tokens[1].value()).text == ","); + STATIC_REQUIRE(std::get(tokens[2].value()).text == " "); + STATIC_REQUIRE(std::get(tokens[3].value()).text == "World"); + STATIC_REQUIRE_FALSE(tokens[4]); + } +}