Skip to content

Commit c31ac1a

Browse files
shewitt-aushewitt-au
committed
Migrate code
1 parent f97999d commit c31ac1a

File tree

10 files changed

+712
-606
lines changed

10 files changed

+712
-606
lines changed

‎.gitmodules

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,6 @@
1313
path = external/throwing_ptr
1414
url = https://github.com/rockdreamer/throwing_ptr
1515
ignore = dirty
16+
[submodule "external/lexertl17"]
17+
path = external/lexertl17
18+
url = https://github.com/BenHanson/lexertl17.git

‎external/lexertl17

Submodule lexertl17 added at 5507eed

‎lib/CMakeLists.txt

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,22 @@ else ()
1111
message(STATUS "libpl static library is being created")
1212
endif ()
1313

14+
add_subdirectory(lexer_gen)
15+
16+
set(PL_STATICLEXERPATH "${CMAKE_CURRENT_SOURCE_DIR}/include/pl/core/lexer_static.hpp")
17+
18+
add_custom_command(
19+
OUTPUT ${PL_STATICLEXERPATH}
20+
COMMAND $<TARGET_FILE:lexer_gen> ${PL_STATICLEXERPATH}
21+
DEPENDS lexer_gen
22+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
23+
COMMENT "Generating lexertl17 static lexer"
24+
)
25+
26+
add_custom_target(run_lexer_gen
27+
DEPENDS ${PL_STATICLEXERPATH}
28+
)
29+
1430
add_library(libpl ${LIBRARY_TYPE}
1531
source/pl/helpers/utils.cpp
1632

@@ -52,7 +68,11 @@ add_library(libpl ${LIBRARY_TYPE}
5268

5369
source/pl/core/token.cpp
5470
source/pl/core/evaluator.cpp
71+
5572
source/pl/core/lexer.cpp
73+
$<$<NOT:$<CONFIG:Debug>>:${PL_STATICLEXERPATH}>
74+
$<$<CONFIG:Debug>:source/pl/core/lexer_sm.cpp>
75+
5676
source/pl/core/parser.cpp
5777
source/pl/core/preprocessor.cpp
5878
source/pl/core/validator.cpp
@@ -102,6 +122,7 @@ endif ()
102122

103123
target_include_directories(libpl PUBLIC include ../external/throwing_ptr/include)
104124
target_include_directories(libpl_includes INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include ../external/throwing_ptr/include)
125+
target_include_directories(libpl PRIVATE include ../external/lexertl17/include)
105126
target_link_libraries(libpl PRIVATE ${FMT_LIBRARIES})
106127
target_link_libraries(libpl PUBLIC wolv::types wolv::io wolv::utils wolv::hash wolv::containers)
107128

‎lib/include/pl/core/errors/error.hpp

Lines changed: 42 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -130,33 +130,27 @@ namespace pl::core::err {
130130
std::vector<Location> m_trace;
131131
};
132132

133-
class ErrorCollector {
133+
class ErrorCollectorExplicitLocation
134+
{
134135
public:
135-
136-
virtual ~ErrorCollector() = default;
137-
138-
virtual Location location() = 0;
136+
virtual ~ErrorCollectorExplicitLocation() = default;
139137

140138
template <typename... Args>
141-
void error(const fmt::format_string<Args...>& fmt, Args&&... args) {
142-
this->m_errors.emplace_back(fmt::format(fmt, std::forward<Args>(args)...), location());
139+
void error(const Location &location, const fmt::format_string<Args...> &fmt, Args&&... args) {
140+
this->m_errors.emplace_back(fmt::format(fmt, std::forward<Args>(args)...), location);
143141
}
144142

145-
void error(const std::string &message) {
146-
this->m_errors.emplace_back(message, location());
147-
}
148-
149-
void errorDesc(const std::string &message, const std::string &description) {
150-
this->m_errors.emplace_back(message, description, location());
143+
void errorDesc(const Location &location, const std::string &message, const std::string &description) {
144+
this->m_errors.emplace_back(message, description, location);
151145
}
152146

153147
template<typename... Args>
154-
void errorDesc(const fmt::format_string<Args...>& message, const std::string &description, Args&&... args) {
155-
this->m_errors.emplace_back(fmt::format(message, std::forward<Args>(args)...), description, location());
148+
void errorDesc(const Location &location, const fmt::format_string<Args...>& message, const std::string &description, Args&&... args) {
149+
this->m_errors.emplace_back(fmt::format(message, std::forward<Args>(args)...), description, location);
156150
}
157151

158-
void error(CompileError& error) {
159-
error.getTrace().push_back(location());
152+
void error(const Location &location, CompileError& error) {
153+
error.getTrace().push_back(location);
160154
this->m_errors.push_back(std::move(error));
161155
}
162156

@@ -188,8 +182,39 @@ namespace pl::core::err {
188182
void clear() {
189183
this->m_errors.clear();
190184
}
185+
191186
private:
192187
std::vector<CompileError> m_errors;
193188
};
194189

190+
class ErrorCollector : public ErrorCollectorExplicitLocation {
191+
public:
192+
193+
virtual ~ErrorCollector() = default;
194+
195+
virtual Location location() = 0;
196+
197+
template <typename... Args>
198+
void error(const fmt::format_string<Args...> &fmt, Args&&... args) {
199+
this->ErrorCollectorExplicitLocation::error(location(), fmt, std::forward<Args>(args)...);
200+
}
201+
202+
void error(const std::string &message) {
203+
this->errorAt(location(), message);
204+
}
205+
206+
void errorDesc(const std::string &message, const std::string &description) {
207+
this->ErrorCollectorExplicitLocation::errorDesc(location(), message, description);
208+
}
209+
210+
template<typename... Args>
211+
void errorDesc(const fmt::format_string<Args...>& message, const std::string &description, Args&&... args) {
212+
this->ErrorCollectorExplicitLocation::errorDesc(location(), message, description, std::forward<Args>(args)...);
213+
}
214+
215+
void error(CompileError& error) {
216+
this->ErrorCollectorExplicitLocation::error(location(), error);
217+
}
218+
};
219+
195220
}

‎lib/include/pl/core/lexer.hpp

Lines changed: 17 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,72 +1,34 @@
1+
// "Lexer.hpp"
12
#pragma once
23

3-
#include <pl/helpers/types.hpp>
4-
#include <pl/core/errors/error.hpp>
5-
6-
#include <pl/core/token.hpp>
7-
8-
#include <fmt/core.h>
9-
4+
#include <cstddef>
105
#include <optional>
11-
#include <string>
126
#include <vector>
13-
7+
#include <pl/core/token.hpp>
8+
#include <pl/core/errors/error.hpp>
149
#include <pl/core/errors/result.hpp>
10+
// Debugging
11+
#include <string>
1512

1613
namespace pl::core {
1714

18-
class Lexer : err::ErrorCollector {
15+
class Lexer : err::ErrorCollectorExplicitLocation {
1916
public:
20-
Lexer() = default;
17+
Lexer();
18+
19+
void reset();
2120

2221
hlp::CompileResult<std::vector<Token>> lex(const api::Source *source);
2322
size_t getLongestLineLength() const { return m_longestLineLength; }
24-
void reset();
2523

2624
private:
27-
[[nodiscard]] char peek(size_t p = 1) const;
28-
bool processToken(auto parserFunction, const std::string_view& identifier);
29-
Location location() override;
30-
31-
std::optional<char> parseCharacter();
32-
std::optional<Token> parseOperator();
33-
std::optional<Token> parseSeparator();
34-
std::optional<Token> parseOneLineComment();
35-
std::optional<Token> parseOneLineDocComment();
36-
std::optional<Token> parseMultiLineComment();
37-
std::optional<Token> parseMultiLineDocComment();
38-
std::optional<Token> parseKeyword(const std::string_view &identifier);
39-
std::optional<Token> parseType(const std::string_view &identifier);
40-
std::optional<Token> parseDirectiveName(const std::string_view &identifier);
41-
std::optional<Token> parseNamedOperator(const std::string_view &identifier);
42-
std::optional<Token> parseConstant(const std::string_view &identifier);
43-
std::optional<Token> parseStringLiteral();
44-
std::optional<Token> parseDirectiveArgument();
45-
std::optional<Token> parseDirectiveValue();
46-
std::optional<Token::Literal> parseIntegerLiteral(std::string_view literal);
25+
std::optional<Token::Literal> parseInteger(std::string_view literal, const auto &location);
26+
std::optional<double> parseFloatingPoint(std::string_view literal, const char suffix, const auto &location);
27+
std::optional<char> parseCharacter(const char* &pchar, const char* e, const auto &location);
28+
std::optional<Token> parseStringLiteral(std::string_view literal, const auto &location);
4729

48-
std::optional<double> parseFloatingPoint(std::string_view literal, char suffix);
49-
std::optional<u128> parseInteger(std::string_view literal);
50-
51-
Token makeToken(const Token& token, size_t length = 1);
52-
static Token makeTokenAt(const Token& token, Location& location, size_t length = 1);
53-
void addToken(const Token& token);
54-
bool hasTheLineEnded(const char &ch) {
55-
if(ch == '\n') {
56-
m_longestLineLength = std::max(m_longestLineLength, m_cursor - m_lineBegin);
57-
m_line++;
58-
m_lineBegin = m_cursor;
59-
return true;
60-
}
61-
return false;
62-
}
63-
std::string m_sourceCode;
64-
const api::Source* m_source = nullptr;
6530
std::vector<Token> m_tokens;
66-
size_t m_cursor = 0;
67-
u32 m_line = 0;
68-
u32 m_lineBegin = 0;
69-
size_t m_longestLineLength = 0;
70-
u32 m_errorLength = 0;
31+
std::size_t m_longestLineLength = 0;
7132
};
72-
}
33+
34+
} // namespace pl::core

‎lib/include/pl/core/lexer_sm.hpp

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#pragma once
2+
3+
#include <lexertl/state_machine.hpp>
4+
5+
namespace pl::core {
6+
7+
namespace {
8+
9+
namespace LexerToken {
10+
11+
enum {
12+
EndOfFile, NewLine, KWNamedOpTypeConstIdent, SingleLineComment,
13+
MultiLineCommentOpen, MultiLineCommentClose, String, Separator,
14+
Directive, DirectiveType, DirectiveParam, Operator, Char,
15+
Integer, FPNumber
16+
};
17+
18+
}
19+
}
20+
21+
void newLexerBuild(lexertl::state_machine &sm);
22+
23+
}

‎lib/lexer_gen/CMakeLists.txt

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
set(SOURCES
2+
../source/pl/core/lexer_sm.cpp
3+
main.cpp
4+
)
5+
6+
add_executable(lexer_gen ${SOURCES})
7+
8+
target_include_directories(lexer_gen PRIVATE
9+
${CMAKE_CURRENT_SOURCE_DIR}/../include
10+
)
11+
12+
target_include_directories(lexer_gen PRIVATE include ../../external/lexertl17/include)

‎lib/lexer_gen/main.cpp

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
Build a "static" lexer in release builds.
3+
4+
Static in the sense that the state machine is built in a pre-build
5+
step to optimize application start-up time.
6+
*/
7+
#include <pl/core/lexer_sm.hpp>
8+
9+
#include <lexertl/state_machine.hpp>
10+
#include <lexertl/generate_cpp.hpp>
11+
12+
#include <fstream>
13+
#include <iostream>
14+
15+
int main(int argc, char *argv[])
16+
{
17+
if (argc!=2)
18+
return 1;
19+
20+
lexertl::state_machine sm;
21+
22+
pl::core::newLexerBuild(sm);
23+
sm.minimise();
24+
25+
std::ofstream ofs(argv[1]);
26+
lexertl::table_based_cpp::generate("lookup", sm, false, ofs);
27+
28+
return 0;
29+
}

0 commit comments

Comments
 (0)