Since Pull Request ggml-org#14029 is pending

I have merged Release b6300 (https://github.com/ggml-org/llama.cpp/releases/tag/b6300)
into
https://github.com/ngxson/llama.cpp/tree/xsn/qwen3_embd_rerank

Author: sfallah

common/common.h

@@ -413,6 +413,7 @@ struct common_params {
     std::string embd_out   = "";    // empty = default, "array" = [[],[]...], "json" = openai style, "json+" = same "json" + cosine similarity matrix
     std::string embd_sep   = "\n";  // separator of embeddings
     std::string cls_sep    = "\t";  // separator of classification sequences
+    bool reranking         = false; // enable reranking support on server
 
     // server params
     int32_t port              = 8080;         // server listens on this network port

convert_hf_to_gguf.py

@@ -887,9 +887,6 @@ def get_vocab_base_pre(self, tokenizer) -> str:
         if chkhsh == "a1e163ecab2e718a4c829d1148b6e86824ec36163bb71941c3dca9cd5ac25756":
             # ref: https://huggingface.co/JetBrains/Mellum-4b-base
             res = "mellum"
-        if chkhsh == "d4540891389ea895b53b399da6ac824becc30f2fba0e9ddbb98f92e55ca0e97c":
-            # ref: https://huggingface.co/Qwen/Qwen3-Embedding-0.6B
-            res = "qwen2"
 
         if res is None:
             logger.warning("\n")
@@ -3665,6 +3662,8 @@ class Qwen3Model(Qwen2Model):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        hparams = ModelBase.load_hparams(self.dir_model, is_mistral_format=False)
+        self.origin_hf_arch = hparams.get('architectures', [None])[0]
         # a bit hacky, but currently the only way to detect if this is a rerank model
         # ref: https://huggingface.co/Qwen/Qwen3-Reranker-0.6B
         readme_path = self.dir_model / "README.md"
@@ -3674,6 +3673,8 @@ def __init__(self, *args, **kwargs):
                 readme_text = f.read()
         if "# Qwen3-Reranker" in readme_text:
             self._find_rerank_config()
+        else:
+            logger.info("gguf: not a rerank model")
 
     def _find_rerank_config(self):
         from transformers import AutoTokenizer
@@ -3688,6 +3689,7 @@ def _find_rerank_config(self):
     def set_gguf_parameters(self):
         super().set_gguf_parameters()
         is_rerank = self.token_false_id is not None and self.token_true_id is not None
+        logger.info(f"gguf: is_rerank = {is_rerank}")
         if is_rerank:
             self.gguf_writer.add_pooling_type(gguf.PoolingType.RANK)
             self.gguf_writer.add_classifier_output_labels(["yes", "no"])
@@ -3723,11 +3725,6 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
 
         return super().modify_tensors(data_torch, name, bid)
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        hparams = ModelBase.load_hparams(self.dir_model, is_mistral_format=False)
-        self.origin_hf_arch = hparams.get('architectures', [None])[0]
-
     def set_vocab(self):
         # deal with intern-s1-mini
         if self.origin_hf_arch == 'InternS1ForConditionalGeneration':

src/llama-graph.cpp

@@ -204,7 +204,7 @@ void llm_graph_input_cls::set_input(const llama_ubatch * ubatch) {
         std::vector<int> target_pos(n_seqs_unq, -1);
         std::vector<int> target_row(n_seqs_unq, -1);
 
-        bool last = cparams.pooling_type == LLAMA_POOLING_TYPE_LAST;
+        bool last = cparams.pooling_type == LLAMA_POOLING_TYPE_LAST || arch == LLM_ARCH_QWEN3; // qwen3 reranking & embedding
 
         for (int i = 0; i < n_tokens; ++i) {
             const llama_pos pos = ubatch->pos[i];

src/llama-model.cpp

@@ -8088,7 +8088,7 @@ struct llm_build_stablelm : public llm_graph_context {
 };
 
 struct llm_build_qwen : public llm_graph_context {
-    llm_build_qwen(const llama_model & model, const llm_graph_params & params, ggml_cgraph * gf) : llm_graph_context(params) {
+    llm_build_qwen(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
         const int64_t n_embd_head = hparams.n_embd_head_v;
 
         GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);

tools/server/server.cpp

@@ -4798,8 +4798,8 @@ int main(int argc, char ** argv) {
     };
 
     const auto handle_rerank = [&ctx_server, &res_error, &res_ok](const httplib::Request & req, httplib::Response & res) {
-        if (!ctx_server.params_base.embedding || ctx_server.params_base.pooling_type != LLAMA_POOLING_TYPE_RANK) {
-            res_error(res, format_error_response("This server does not support reranking. Start it with `--reranking`", ERROR_TYPE_NOT_SUPPORTED));
+        if (!ctx_server.params_base.reranking || ctx_server.params_base.embedding) {
+            res_error(res, format_error_response("This server does not support reranking. Start it with `--reranking` and without `--embedding`", ERROR_TYPE_NOT_SUPPORTED));
             return;
         }
 
@@ -4838,29 +4838,19 @@ int main(int argc, char ** argv) {
             return;
         }
 
-        std::vector<server_tokens> tokenized_queries = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, query, /* add_special */ false, true);
-        if (tokenized_queries.size() != 1) {
-            res_error(res, format_error_response("\"query\" must contain only a single prompt", ERROR_TYPE_INVALID_REQUEST));
-        }
-
         // create and queue the task
         json responses = json::array();
         bool error = false;
         std::unordered_set<int> task_ids;
         {
             std::vector<server_task> tasks;
-            auto tokenized_docs = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, documents, /* add_special */ false, true);
-            tasks.reserve(tokenized_docs.size());
-            for (size_t i = 0; i < tokenized_docs.size(); i++) {
-                auto tmp = format_rerank(ctx_server.vocab, tokenized_queries[0], tokenized_docs[i]);
             auto inputs = tokenize_rerank(ctx_server.model, query, documents);
             tasks.reserve(documents.size());
             for (size_t i = 0; i < inputs.size(); i++) {
                 server_task task   = server_task(SERVER_TASK_TYPE_RERANK);
                 task.id            = ctx_server.queue_tasks.get_new_id();
                 task.index         = i;
                 task.prompt_tokens = server_tokens(inputs[i], ctx_server.mctx != nullptr);
-                task.prompt_tokens = std::move(tmp);
                 tasks.push_back(std::move(task));
             }