Revert "Add tekken to the tokenize tool (#119)" (#123)

This reverts commit 1c2481d.

Author: jackzhxng

examples/tokenize_tool/main.cpp

@@ -22,7 +22,6 @@
 #include "hf_tokenizer.h"
 #include "sentencepiece.h"
 #include "tiktoken.h"
-#include "tekken.h"
 
 using namespace tokenizers;
 
@@ -65,8 +64,6 @@ int main(int argc, char* argv[]) {
     tok_ptr.reset(new Tiktoken());
   } else if (tokenizer_type == "hf_tokenizer") {
     tok_ptr.reset(new HFTokenizer());
-  } else if (tokenizer_type == "tekken") {
-    tok_ptr.reset(new Tekken());
   } else {
     std::stringstream ss;
     ss << "ERROR: Invalid tokenizer type: " << tokenizer_type << std::endl