feat: add regenerate all embeddings endpoint

Author: McPatate

init_db.sql

@@ -14,7 +14,7 @@ CREATE TABLE IF NOT EXISTS issues (
   number INT NOT NULL,
   html_url VARCHAR NOT NULL,
   url VARCHAR NOT NULL,
-  embedding vector(1024) NOT NULL,
+  embedding halfvec(2560) NOT NULL,
   created_at timestamp with time zone NOT NULL DEFAULT (current_timestamp AT TIME ZONE 'UTC'),
   updated_at timestamp with time zone NOT NULL DEFAULT (current_timestamp AT TIME ZONE 'UTC')
 );
@@ -31,14 +31,19 @@ CREATE TABLE IF NOT EXISTS comments (
 
 CREATE INDEX IF NOT EXISTS issues_source_id_idx ON issues (source_id);
 CREATE INDEX IF NOT EXISTS comments_source_id_idx ON comments (source_id);
-CREATE INDEX IF NOT EXISTS issues_embedding_hnsw_idx ON issues USING hnsw (embedding vector_cosine_ops);
+CREATE INDEX IF NOT EXISTS issues_embedding_hnsw_idx ON issues USING hnsw (embedding halfvec_cosine_ops);
+
+CREATE TYPE job_type AS ENUM ('embeddings_regeneration', 'issue_indexation');
 
 CREATE TABLE IF NOT EXISTS jobs (
   id SERIAL PRIMARY KEY,
-  repository_id VARCHAR NOT NULL UNIQUE,
+  job_type job_type NOT NULL,
+  repository_id VARCHAR UNIQUE,
   data JSONB NOT NULL,
   created_at timestamp with time zone NOT NULL DEFAULT (current_timestamp AT TIME ZONE 'UTC'),
   updated_at timestamp with time zone NOT NULL DEFAULT (current_timestamp AT TIME ZONE 'UTC')
 );
 
 CREATE INDEX IF NOT EXISTS jobs_repository_id_idx ON jobs (repository_id);
+CREATE INDEX IF NOT EXISTS jobs_job_type_idx ON jobs (job_type);
+CREATE UNIQUE INDEX jobs_type_special_idx ON jobs (job_type) WHERE job_type = 'embeddings_regeneration';

issue-bot/Cargo.lock

@@ -18,15 +18,20 @@ hex = "0.4"
 # hf-hub = { version = "0.4", features = ["tokio"] }
 hmac = "0.12"
 metrics = "0.24"
-metrics-exporter-prometheus = "0.16"
+metrics-exporter-prometheus = "0.17"
 nanoid = "0.4"
 once_cell = "1.20"
 pgvector = { version = "0.4", features = ["sqlx"] }
 reqwest = { version = "0.12", features = ["json"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1", features = ["raw_value"] }
 sha2 = "0.10"
-sqlx = { version = "0.8", features = ["chrono", "postgres", "runtime-tokio"] }
+sqlx = { version = "0.8", features = [
+  "chrono",
+  "macros",
+  "postgres",
+  "runtime-tokio",
+] }
 thiserror = "2"
 # tokenizers = { version = "0.21", default-features = false, features = ["onig"] }
 tokio = { version = "1.0", features = ["full"] }