Regex_Search: Add tokenizer, postfix conversion, and NFA construction

libpz/include/RegexPostfix.hpp

@@ -0,0 +1,28 @@
+#ifndef REGEX_POSTFIX_HPP
+#define REGEX_POSTFIX_HPP
+
+#include <RegexTokenizer.hpp>
+#include <pz_cxx_std.hpp>
+#include <pz_types.hpp>
+
+/**
+ * @brief Converts regex tokens from infix to postfix (RPN) form.
+ *
+ * This conversion is used as a preprocessing step before NFA construction.
+ * The class is stateless and intended to be used via its static methods.
+ */
+class Postfix {
+public:
+  /**
+   * @brief Convert an infix token sequence into postfix order.
+   */
+  static std::vector<Token> convert(const std::vector<Token> &infix);
+
+private:
+  /**
+   * @brief Returns precedence of a regex operator token.
+   */
+  static st32 get_precedence(TokenType type);
+};
+
+#endif // REGEX_POSTFIX_HPP

libpz/include/RegexTokenizer.hpp

@@ -0,0 +1,145 @@
+#ifndef REGEX_TOKENIZER_HPP
+#define REGEX_TOKENIZER_HPP
+
+#include <pz_cxx_std.hpp>
+#include <pz_types.hpp>
+
+/**
+ * @brief Types of tokens produced by the regex tokenizer.
+ */
+enum class TokenType {
+  /** Literal character like 'a', 'b', etc. */
+  LITERAL,
+
+  /** '.' wildcard */
+  DOT,
+
+  /** '*' operator */
+  STAR,
+
+  /** '+' operator */
+  PLUS,
+
+  /** '?' operator */
+  QUESTION,
+
+  /** '|' alternation */
+  ALTERNATION,
+
+  /** '(' opening group */
+  LPAREN,
+
+  /** ')' closing group */
+  RPAREN,
+
+  /** '^' start anchor */
+  CARET,
+
+  /** '$' end anchor */
+  DOLLAR,
+
+  /** Character class: '[...]', \d, \w, \s, etc */
+  CHAR_CLASS,
+
+  /** Quantifier range: '{m,n}', '{m,}', '{m}' */
+  QUANTIFIER_RANGE,
+
+  /** End of pattern */
+  END,
+
+  /** Implicit concatenation */
+  CONCAT
+};
+
+/**
+ * @brief Represents a character range [lo, hi].
+ */
+struct CharRange {
+  /** Lower bound */
+  ut8 lo;
+
+  /** Upper bound */
+  ut8 hi;
+};
+
+/**
+ * @brief A single token in the regex.
+ */
+struct Token {
+  /** Token category */
+  TokenType type;
+  /** Position in pattern (for error reporting) */
+  size_t pos;
+  /** Group ID for parentheses */
+  st32 group_id = -1;
+
+  /** Literal character value */
+  ut8 literal = '\0';
+
+  /** Whether character class is negated */
+  bool negated = false;
+  /** Character ranges for character class */
+  std::vector<CharRange> ranges{};
+
+  /** Minimum repetitions for quantifier */
+  st32 min = 0;
+  /** Maximum repetitions (-1 means unbounded) */
+  st32 max = 0;
+};
+
+/**
+ * @brief Converts a regex pattern into a sequence of tokens.
+ */
+class Tokenizer {
+public:
+  /**
+   * @brief Construct tokenizer for a pattern.
+   * @param pat Regex pattern.
+   */
+  explicit Tokenizer(std::string_view pat);
+
+  /**
+   * @brief Tokenize the entire pattern.
+   * @return Vector of tokens ending with END token.
+   */
+  std::vector<Token> tokenize();
+
+private:
+  /** Input regex pattern */
+  std::string_view pattern;
+  /** Current cursor position */
+  size_t i = 0;
+  /** Counter for assigning group IDs */
+  st32 group_counter = 0;
+  /** Stack for nested group tracking */
+  std::stack<st32> group_stack;
+
+  /** Peek next character without consuming */
+  ut8 peek() const;
+  /** Consume next character */
+  ut8 get();
+  /** Check for end of input */
+  bool eof() const;
+
+  /** Read next token */
+  Token next_token();
+  /** Read literal character */
+  Token read_literal(ut8);
+  /** Read escape sequence */
+  Token read_escape();
+  /** Read character class */
+  Token read_char_class();
+  /** Read quantifier range */
+  Token read_quantifier();
+
+  /** @brief Populates a token with ranges for \d, \w, \s, etc. */
+  void add_shorthand_ranges(ut8, Token &);
+
+  /** @brief Inserts implicit CONCAT tokens where concatenation occurs. */
+  void add_concat_tokens(std::vector<Token> &);
+
+  /** @brief Sorts and merges overlapping ranges for efficient NFA matching. */
+  void normalize_ranges(std::vector<CharRange> &);
+};
+
+#endif // REGEX_TOKENIZER_HPP

libpz/include/pz_cxx_std.hpp

@@ -11,6 +11,7 @@
 #include <optional>
 #include <set>
 #include <sstream>
+#include <stack>
 #include <string>
 #include <string_view>
 #include <unordered_map>

libpz/regex/RegexPostfix.cpp

@@ -0,0 +1,126 @@
+#include "RegexPostfix.hpp"
+#include "pz_error.hpp"
+
+st32 Postfix::get_precedence(TokenType type) {
+  switch (type) {
+  case TokenType::STAR:
+  case TokenType::PLUS:
+  case TokenType::QUESTION:
+  case TokenType::QUANTIFIER_RANGE:
+    return 3; // Unary postfix operators
+  case TokenType::CONCAT:
+    return 2; // Implicit concatenation
+  case TokenType::ALTERNATION:
+    return 1; // Lowest precedence
+  default:
+    return 0;
+  }
+}
+
+std::vector<Token> Postfix::convert(const std::vector<Token> &infix) {
+  std::vector<Token> postfix;
+  std::stack<Token> operators;
+  TokenType last_type = TokenType::END; // Tracks previous token for validation
+
+  for (const auto &t : infix) {
+    switch (t.type) {
+    // Operands go directly to output
+    case TokenType::LITERAL:
+    case TokenType::DOT:
+    case TokenType::CHAR_CLASS:
+    case TokenType::CARET:
+    case TokenType::DOLLAR:
+      postfix.push_back(t);
+      break;
+
+    // '(' is pushed to operator stack and output (for NFA grouping)
+    case TokenType::LPAREN: {
+      postfix.push_back(t);
+      operators.push(t);
+      break;
+    }
+
+    // Pop operators until matching '(' is found
+    case TokenType::RPAREN: {
+      if (last_type == TokenType::LPAREN)
+        PzError::report_error(PzError::PzErrorType::PZ_INVALID_INPUT,
+                              "Empty Parentheses at position " +
+                                  std::to_string(t.pos));
+      while (!operators.empty() && operators.top().type != TokenType::LPAREN) {
+        postfix.push_back(operators.top());
+        operators.pop();
+      }
+      if (operators.empty())
+        PzError::report_error(PzError::PzErrorType::PZ_INVALID_INPUT,
+                              "Mismatched ')' at position " +
+                                  std::to_string(t.pos));
+      operators.pop(); // Discard '('
+      postfix.push_back(t);
+      break;
+    }
+    // Unary postfix operators must follow a valid expression
+    case TokenType::STAR:
+    case TokenType::PLUS:
+    case TokenType::QUESTION:
+    case TokenType::QUANTIFIER_RANGE:
+      if (last_type != TokenType::LITERAL && last_type != TokenType::DOT &&
+          last_type != TokenType::CHAR_CLASS &&
+          last_type != TokenType::RPAREN) {
+        PzError::report_error(PzError::PzErrorType::PZ_INVALID_INPUT,
+                              "Quantifier used without a valid preceding "
+                              "expression at position " +
+                                  std::to_string(t.pos));
+      }
+      postfix.push_back(t);
+      break;
+
+    case TokenType::ALTERNATION:
+      // '|' must separate two valid expressions
+      if (last_type == TokenType::END || last_type == TokenType::LPAREN ||
+          last_type == TokenType::ALTERNATION) {
+        PzError::report_error(PzError::PzErrorType::PZ_INVALID_INPUT,
+                              "Invalid '|' at position " +
+                                  std::to_string(t.pos) +
+                                  ". It must separate two expressions.");
+      }
+      goto push_operator;
+
+    // Binary operators handled via precedence rules
+    case TokenType::CONCAT:
+    push_operator:
+      while (!operators.empty() && operators.top().type != TokenType::LPAREN &&
+             get_precedence(operators.top().type) >= get_precedence(t.type)) {
+        postfix.push_back(operators.top());
+        operators.pop();
+      }
+      operators.push(t);
+      break;
+
+    default:
+      break;
+    }
+
+    if (t.type != TokenType::END)
+      last_type = t.type;
+  }
+
+  // Pattern must not end with a binary operator
+  if (last_type == TokenType::ALTERNATION || last_type == TokenType::CONCAT) {
+    PzError::report_error(
+        PzError::PzErrorType::PZ_INVALID_INPUT,
+        "Trailing binary operator at end of pattern at position " +
+            std::to_string(infix.back().pos));
+  }
+
+  // Drain remaining operators
+  while (!operators.empty()) {
+    if (operators.top().type == TokenType::LPAREN)
+      PzError::report_error(PzError::PzErrorType::PZ_INVALID_INPUT,
+                            "Unmatched '(' at position " +
+                                std::to_string(operators.top().pos));
+    postfix.push_back(operators.top());
+    operators.pop();
+  }
+
+  return postfix;
+}