diff --git a/helicone-cron/src/db/database.types.ts b/helicone-cron/src/db/database.types.ts index 23f7eaf2b5..3bb12db728 100644 --- a/helicone-cron/src/db/database.types.ts +++ b/helicone-cron/src/db/database.types.ts @@ -1521,6 +1521,45 @@ export type Database = { }, ] } + online_evaluators: { + Row: { + config: Json | null + created_at: string + evaluator: string + id: number + organization: string + } + Insert: { + config?: Json | null + created_at?: string + evaluator: string + id?: number + organization: string + } + Update: { + config?: Json | null + created_at?: string + evaluator?: string + id?: number + organization?: string + } + Relationships: [ + { + foreignKeyName: "public_online_evaluators_evaluator_fkey" + columns: ["evaluator"] + isOneToOne: false + referencedRelation: "evaluator" + referencedColumns: ["id"] + }, + { + foreignKeyName: "public_online_evaluators_organization_fkey" + columns: ["organization"] + isOneToOne: false + referencedRelation: "organization" + referencedColumns: ["id"] + }, + ] + } org_rate_limit_tracker: { Row: { created_at: string | null @@ -2963,7 +3002,6 @@ export type Database = { owner_id: string | null path_tokens: string[] | null updated_at: string | null - user_metadata: Json | null version: string | null } Insert: { @@ -2977,7 +3015,6 @@ export type Database = { owner_id?: string | null path_tokens?: string[] | null updated_at?: string | null - user_metadata?: Json | null version?: string | null } Update: { @@ -2991,7 +3028,6 @@ export type Database = { owner_id?: string | null path_tokens?: string[] | null updated_at?: string | null - user_metadata?: Json | null version?: string | null } Relationships: [ @@ -3013,7 +3049,6 @@ export type Database = { key: string owner_id: string | null upload_signature: string - user_metadata: Json | null version: string } Insert: { @@ -3024,7 +3059,6 @@ export type Database = { key: string owner_id?: string | null upload_signature: string - user_metadata?: Json | null version: string } Update: { @@ -3035,7 +3069,6 @@ export type Database = { key?: string owner_id?: string | null upload_signature?: string - user_metadata?: Json | null version?: string } Relationships: [ @@ -3132,7 +3165,7 @@ export type Database = { Args: { name: string } - Returns: unknown + Returns: string[] } get_size_by_bucket: { Args: Record @@ -3172,10 +3205,6 @@ export type Database = { updated_at: string }[] } - operation: { - Args: Record - Returns: string - } search: { Args: { prefix: string diff --git a/supabase/migrations/20241212204905_online_evaluators.sql b/supabase/migrations/20241212204905_online_evaluators.sql new file mode 100644 index 0000000000..84b08590c8 --- /dev/null +++ b/supabase/migrations/20241212204905_online_evaluators.sql @@ -0,0 +1,66 @@ +create table "public"."online_evaluators" ( + "id" bigint generated by default as identity not null, + "created_at" timestamp with time zone not null default now(), + "evaluator" uuid not null, + "config" jsonb, + "organization" uuid not null +); + + +alter table "public"."online_evaluators" enable row level security; + +CREATE UNIQUE INDEX online_evaluators_pkey ON public.online_evaluators USING btree (id); + +alter table "public"."online_evaluators" add constraint "online_evaluators_pkey" PRIMARY KEY using index "online_evaluators_pkey"; + +alter table "public"."online_evaluators" add constraint "public_online_evaluators_evaluator_fkey" FOREIGN KEY (evaluator) REFERENCES evaluator(id) not valid; + +alter table "public"."online_evaluators" validate constraint "public_online_evaluators_evaluator_fkey"; + +alter table "public"."online_evaluators" add constraint "public_online_evaluators_organization_fkey" FOREIGN KEY (organization) REFERENCES organization(id) not valid; + +alter table "public"."online_evaluators" validate constraint "public_online_evaluators_organization_fkey"; + +grant delete on table "public"."online_evaluators" to "anon"; + +grant insert on table "public"."online_evaluators" to "anon"; + +grant references on table "public"."online_evaluators" to "anon"; + +grant select on table "public"."online_evaluators" to "anon"; + +grant trigger on table "public"."online_evaluators" to "anon"; + +grant truncate on table "public"."online_evaluators" to "anon"; + +grant update on table "public"."online_evaluators" to "anon"; + +grant delete on table "public"."online_evaluators" to "authenticated"; + +grant insert on table "public"."online_evaluators" to "authenticated"; + +grant references on table "public"."online_evaluators" to "authenticated"; + +grant select on table "public"."online_evaluators" to "authenticated"; + +grant trigger on table "public"."online_evaluators" to "authenticated"; + +grant truncate on table "public"."online_evaluators" to "authenticated"; + +grant update on table "public"."online_evaluators" to "authenticated"; + +grant delete on table "public"."online_evaluators" to "service_role"; + +grant insert on table "public"."online_evaluators" to "service_role"; + +grant references on table "public"."online_evaluators" to "service_role"; + +grant select on table "public"."online_evaluators" to "service_role"; + +grant trigger on table "public"."online_evaluators" to "service_role"; + +grant truncate on table "public"."online_evaluators" to "service_role"; + +grant update on table "public"."online_evaluators" to "service_role"; + + diff --git a/valhalla/jawn/src/lib/db/database.types.ts b/valhalla/jawn/src/lib/db/database.types.ts index 23f7eaf2b5..3bb12db728 100644 --- a/valhalla/jawn/src/lib/db/database.types.ts +++ b/valhalla/jawn/src/lib/db/database.types.ts @@ -1521,6 +1521,45 @@ export type Database = { }, ] } + online_evaluators: { + Row: { + config: Json | null + created_at: string + evaluator: string + id: number + organization: string + } + Insert: { + config?: Json | null + created_at?: string + evaluator: string + id?: number + organization: string + } + Update: { + config?: Json | null + created_at?: string + evaluator?: string + id?: number + organization?: string + } + Relationships: [ + { + foreignKeyName: "public_online_evaluators_evaluator_fkey" + columns: ["evaluator"] + isOneToOne: false + referencedRelation: "evaluator" + referencedColumns: ["id"] + }, + { + foreignKeyName: "public_online_evaluators_organization_fkey" + columns: ["organization"] + isOneToOne: false + referencedRelation: "organization" + referencedColumns: ["id"] + }, + ] + } org_rate_limit_tracker: { Row: { created_at: string | null @@ -2963,7 +3002,6 @@ export type Database = { owner_id: string | null path_tokens: string[] | null updated_at: string | null - user_metadata: Json | null version: string | null } Insert: { @@ -2977,7 +3015,6 @@ export type Database = { owner_id?: string | null path_tokens?: string[] | null updated_at?: string | null - user_metadata?: Json | null version?: string | null } Update: { @@ -2991,7 +3028,6 @@ export type Database = { owner_id?: string | null path_tokens?: string[] | null updated_at?: string | null - user_metadata?: Json | null version?: string | null } Relationships: [ @@ -3013,7 +3049,6 @@ export type Database = { key: string owner_id: string | null upload_signature: string - user_metadata: Json | null version: string } Insert: { @@ -3024,7 +3059,6 @@ export type Database = { key: string owner_id?: string | null upload_signature: string - user_metadata?: Json | null version: string } Update: { @@ -3035,7 +3069,6 @@ export type Database = { key?: string owner_id?: string | null upload_signature?: string - user_metadata?: Json | null version?: string } Relationships: [ @@ -3132,7 +3165,7 @@ export type Database = { Args: { name: string } - Returns: unknown + Returns: string[] } get_size_by_bucket: { Args: Record @@ -3172,10 +3205,6 @@ export type Database = { updated_at: string }[] } - operation: { - Args: Record - Returns: string - } search: { Args: { prefix: string diff --git a/valhalla/jawn/src/lib/handlers/HandlerContext.ts b/valhalla/jawn/src/lib/handlers/HandlerContext.ts index 8af084b4d5..b6ff167d9c 100644 --- a/valhalla/jawn/src/lib/handlers/HandlerContext.ts +++ b/valhalla/jawn/src/lib/handlers/HandlerContext.ts @@ -76,6 +76,7 @@ export type ProcessedLog = { heliconeTemplate?: TemplateWithInputs; assets?: Map; properties?: Record; + scores?: Record; }; response: { model?: string; diff --git a/valhalla/jawn/src/lib/handlers/LoggingHandler.ts b/valhalla/jawn/src/lib/handlers/LoggingHandler.ts index 2abfaa8ee4..d91ecbc21f 100644 --- a/valhalla/jawn/src/lib/handlers/LoggingHandler.ts +++ b/valhalla/jawn/src/lib/handlers/LoggingHandler.ts @@ -36,6 +36,11 @@ export type BatchPayload = { requestResponseVersionedCH: RequestResponseRMT[]; searchRecords: Database["public"]["Tables"]["request_response_search"]["Insert"][]; experimentCellValues: ExperimentCellValue[]; + scores: { + organizationId: string; + requestId: string; + scores: Record; + }[]; }; export class LoggingHandler extends AbstractLogHandler { @@ -62,6 +67,7 @@ export class LoggingHandler extends AbstractLogHandler { requestResponseVersionedCH: [], searchRecords: [], experimentCellValues: [], + scores: [], }; } @@ -87,6 +93,11 @@ export class LoggingHandler extends AbstractLogHandler { this.batchPayload.responses.push(responseMapped); this.batchPayload.assets.push(...assetsMapped); this.batchPayload.searchRecords.push(...searchRecordsMapped); + this.batchPayload.scores.push({ + organizationId: context.orgParams?.id ?? "", + requestId: requestMapped.id ?? "", + scores: context.processedLog.request.scores ?? {}, + }); if (s3RecordMapped) { this.batchPayload.s3Records.push(s3RecordMapped); @@ -445,7 +456,11 @@ export class LoggingHandler extends AbstractLogHandler { assets: context.processedLog.assets ? Array.from(context.processedLog.assets.keys()) : [], - scores: {}, + scores: Object.fromEntries( + Object.entries(context.processedLog.request.scores ?? {}).map( + ([key, value]) => [key, +(value ?? 0)] + ) + ), request_body: this.extractRequestBodyMessage(context.processedLog.request.body) ?? "", response_body: diff --git a/valhalla/jawn/src/lib/handlers/OnlineEvalHandler.ts b/valhalla/jawn/src/lib/handlers/OnlineEvalHandler.ts new file mode 100644 index 0000000000..eaf3927006 --- /dev/null +++ b/valhalla/jawn/src/lib/handlers/OnlineEvalHandler.ts @@ -0,0 +1,102 @@ +import { TemplateWithInputs } from "@helicone/prompts/dist/objectParser"; +import { err, ok, PromiseGenericResult } from "../shared/result"; +import { AbstractLogHandler } from "./AbstractLogHandler"; +import { HandlerContext } from "./HandlerContext"; +import { sanitizeObject } from "../../utils/sanitize"; +import { OnlineEvalStore } from "../stores/OnlineEvalStore"; +import { LLMAsAJudge } from "../clients/LLMAsAJudge/LLMAsAJudge"; +import { OPENAI_KEY } from "../clients/constant"; +import { getEvaluatorScoreName } from "../../managers/evaluator/EvaluatorManager"; + +export class OnlineEvalHandler extends AbstractLogHandler { + public async handle(context: HandlerContext): PromiseGenericResult { + const orgId = context.orgParams?.id; + if (!orgId) { + return err(`Org ID not found in context`); + } + + const onlineEvalStore = new OnlineEvalStore(orgId); + const onlineEvals = await onlineEvalStore.getOnlineEvalsByOrgId(orgId); + + if (onlineEvals.error) { + return err(onlineEvals.error); + } + + for (const onlineEval of onlineEvals.data ?? []) { + const sampleRate = Number( + (onlineEval.config as any)?.["sampleRate"] ?? 100 + ); + + if ( + isNaN(sampleRate) || + sampleRate < 0 || + sampleRate > 100 || + Math.random() * 100 > sampleRate + ) { + context.processedLog.request.scores = + context.processedLog.request.scores ?? {}; + context.processedLog.request.scores[ + getEvaluatorScoreName(onlineEval.evaluator_name) + ] = undefined; + continue; + } + + const propertyFilters = ((onlineEval.config as any)?.[ + "propertyFilters" + ] ?? []) as { + key: string; + value: string; + }[]; + + const shouldWebhookProperties = propertyFilters.every( + (propertyFilter) => + context.processedLog.request.properties?.[propertyFilter.key] === + propertyFilter.value + ); + + if (!shouldWebhookProperties) { + context.processedLog.request.scores = + context.processedLog.request.scores ?? {}; + context.processedLog.request.scores[ + getEvaluatorScoreName(onlineEval.evaluator_name) + ] = undefined; + continue; + } + + const inputRecord = { + inputs: context.processedLog.request.heliconeTemplate?.inputs ?? {}, + autoInputs: + context.processedLog.request.heliconeTemplate?.autoInputs ?? {}, + } as { + inputs: Record; + autoInputs?: Record; + }; + + const llmAsAJudge = new LLMAsAJudge({ + openAIApiKey: OPENAI_KEY!, + scoringType: onlineEval.evaluator_scoring_type as + | "LLM-CHOICE" + | "LLM-BOOLEAN" + | "LLM-RANGE", + llmTemplate: onlineEval.evaluator_llm_template, + inputRecord, + output: JSON.stringify(context.processedLog.response.body), + evaluatorName: onlineEval.evaluator_name, + }); + + try { + const result = await llmAsAJudge.evaluate(); + + const scoreName = getEvaluatorScoreName(onlineEval.evaluator_name); + + context.processedLog.request.scores = + context.processedLog.request.scores ?? {}; + context.processedLog.request.scores[scoreName] = result.score ?? 0; + } catch (e) { + console.error(e); + } + } + + return await super.handle(context); + } +} diff --git a/valhalla/jawn/src/lib/stores/LogStore.ts b/valhalla/jawn/src/lib/stores/LogStore.ts index 38c61b1fb3..d8cb09a846 100644 --- a/valhalla/jawn/src/lib/stores/LogStore.ts +++ b/valhalla/jawn/src/lib/stores/LogStore.ts @@ -98,16 +98,6 @@ const onConflictRequestResponseSearch = `ON CONFLICT (request_id, organization_i request_body_vector = EXCLUDED.request_body_vector, response_body_vector = EXCLUDED.response_body_vector`; -const experimentCellValueColumns = new pgp.helpers.ColumnSet( - ["column_id", "row_index", "value", "status"], - { table: "experiment_cell" } -); - -const onConflictExperimentCellValue = - " ON CONFLICT (column_id, row_index) DO UPDATE SET " + - "value = EXCLUDED.value, " + - "status = EXCLUDED.status"; - export class LogStore { constructor() {} @@ -176,6 +166,17 @@ export class LogStore { } } + if (payload.scores && payload.scores.length > 0) { + for (const score of payload.scores) { + await this.processScore({ + score: score.scores, + requestId: score.requestId, + organizationId: score.organizationId, + t, + }); + } + } + try { const searchRecords = this.filterDuplicateSearchRecords( payload.searchRecords @@ -382,6 +383,96 @@ export class LogStore { return ok("Prompt processed successfully"); } + private mapScores(scores: Record): { + score_attribute_key: string; + score_attribute_type: string; + score_attribute_value: number; + }[] { + return Object.entries(scores).map(([key, value]) => { + if (typeof value === "boolean") { + // Convert booleans to integers (1 for true, 0 for false) + return { + score_attribute_key: key, + score_attribute_type: "boolean", + score_attribute_value: value ? 1 : 0, + }; + } else if (typeof value === "number") { + // Check if the number is an integer + if (Number.isInteger(value)) { + return { + score_attribute_key: key, + score_attribute_type: "number", + score_attribute_value: value, + }; + } else { + // Throw an error if the value is a float + throw new Error( + `Score value for key '${key}' must be an integer. Received: ${value}` + ); + } + } else { + // Throw an error if the value is neither boolean nor number + throw new Error( + `Invalid score value for key '${key}': ${value}. Expected an integer or boolean.` + ); + } + }); + } + + async processScore({ + score, + requestId, + t, + organizationId, + }: { + score: Record; + requestId: string; + t: pgPromise.ITask<{}>; + organizationId: string; + }) { + const mappedScores = this.mapScores(score); + + // Convert arrays for the upsert + const scoreKeys = mappedScores.map((s) => + s.score_attribute_type === "boolean" + ? `${s.score_attribute_key}-hcone-bool` + : s.score_attribute_key + ); + const scoreTypes = mappedScores.map((s) => s.score_attribute_type); + const scoreValues = mappedScores.map((s) => s.score_attribute_value); + const organizationIds = Array(scoreKeys.length).fill(organizationId); + + // First upsert the score attributes and get their IDs + const upsertedAttributes = await t.many( + ` + WITH upserted_attributes AS ( + INSERT INTO score_attribute (score_key, value_type, organization) + SELECT unnest($1::text[]), unnest($2::text[]), unnest($3::uuid[]) + ON CONFLICT (score_key, organization) DO UPDATE SET + score_key = EXCLUDED.score_key, + value_type = EXCLUDED.value_type + RETURNING id, score_key + ) + SELECT id, score_key + FROM upserted_attributes + `, + [scoreKeys, scoreTypes, organizationIds] + ); + + // Then insert the score values + const attributeIds = upsertedAttributes.map((attr) => attr.id); + await t.none( + ` + INSERT INTO score_value (score_attribute, request_id, int_value) + SELECT unnest($1::uuid[]), $2, unnest($3::bigint[]) + ON CONFLICT (score_attribute, request_id) DO NOTHING + `, + [attributeIds, requestId, scoreValues] + ); + + return ok("Scores processed successfully"); + } + filterDuplicateRequests( entries: Database["public"]["Tables"]["request"]["Insert"][] ) { diff --git a/valhalla/jawn/src/lib/stores/OnlineEvalStore.ts b/valhalla/jawn/src/lib/stores/OnlineEvalStore.ts new file mode 100644 index 0000000000..53499b2db3 --- /dev/null +++ b/valhalla/jawn/src/lib/stores/OnlineEvalStore.ts @@ -0,0 +1,39 @@ +import { RequestResponseRMT, clickhouseDb } from "../db/ClickhouseWrapper"; +import { dbExecute } from "../shared/db/dbExecute"; +import { + err, + resultMap, + ok, + Result, + PromiseGenericResult, +} from "../shared/result"; +import { BaseStore } from "./baseStore"; + +export type OnlineEvaluator = { + id: string; + evaluator_scoring_type: string; + evaluator_name: string; + evaluator_llm_template: string; + config: any; +}; + +export class OnlineEvalStore extends BaseStore { + constructor(organizationId: string) { + super(organizationId); + } + + public async getOnlineEvalsByOrgId( + orgId: string + ): PromiseGenericResult { + const { data, error } = await dbExecute( + `SELECT id, evaluator.scoring_type evaluator_scoring_type, evaluator.name evaluator_name, evaluator.llm_template evaluator_llm_template, config FROM online_evaluators JOIN evaluator ON online_evaluators.evaluator = evaluator.id WHERE organization = $1`, + [orgId] + ); + + if (error) { + return err(error); + } + + return ok(data ?? []); + } +} diff --git a/valhalla/jawn/src/managers/LogManager.ts b/valhalla/jawn/src/managers/LogManager.ts index 867597f033..c202ca5c97 100644 --- a/valhalla/jawn/src/managers/LogManager.ts +++ b/valhalla/jawn/src/managers/LogManager.ts @@ -15,12 +15,11 @@ import { VersionedRequestStore } from "../lib/stores/request/VersionedRequestSto import { KafkaProducer } from "../lib/clients/KafkaProducer"; import { WebhookHandler } from "../lib/handlers/WebhookHandler"; import { WebhookStore } from "../lib/stores/WebhookStore"; -import { FeatureFlagStore } from "../lib/stores/FeatureFlagStore"; import { supabaseServer } from "../lib/db/supabase"; import { dataDogClient } from "../lib/clients/DataDogClient"; import { LytixHandler } from "../lib/handlers/LytixHandler"; -import { ExperimentHandler } from "../lib/handlers/ExperimentHandler"; import { SegmentLogHandler } from "../lib/handlers/SegmentLogHandler"; +import { OnlineEvalHandler } from "../lib/handlers/OnlineEvalHandler"; export class LogManager { public async processLogEntry(logMessage: Message): Promise { @@ -55,6 +54,7 @@ export class LogManager { const requestHandler = new RequestBodyHandler(); const responseBodyHandler = new ResponseBodyHandler(); const promptHandler = new PromptHandler(); + const onlineEvalHandler = new OnlineEvalHandler(); const loggingHandler = new LoggingHandler( new LogStore(), new VersionedRequestStore(""), @@ -64,8 +64,6 @@ export class LogManager { const posthogHandler = new PostHogHandler(); const lytixHandler = new LytixHandler(); - const experimentHandler = new ExperimentHandler(); - const webhookHandler = new WebhookHandler( new WebhookStore(supabaseServer.client) ); @@ -77,10 +75,10 @@ export class LogManager { .setNext(requestHandler) .setNext(responseBodyHandler) .setNext(promptHandler) + .setNext(onlineEvalHandler) .setNext(loggingHandler) .setNext(posthogHandler) .setNext(lytixHandler) - .setNext(experimentHandler) .setNext(webhookHandler) .setNext(segmentHandler); diff --git a/valhalla/jawn/src/managers/evaluator/EvaluatorManager.ts b/valhalla/jawn/src/managers/evaluator/EvaluatorManager.ts index a54f97d775..afba542707 100644 --- a/valhalla/jawn/src/managers/evaluator/EvaluatorManager.ts +++ b/valhalla/jawn/src/managers/evaluator/EvaluatorManager.ts @@ -45,7 +45,7 @@ export function placeAssetIdValues( return traverseAndTransform(heliconeTemplate); } -function getEvaluatorScoreName(evaluatorName: string) { +export function getEvaluatorScoreName(evaluatorName: string) { return evaluatorName .toLowerCase() .replace(" ", "_") diff --git a/web/supabase/database.types.ts b/web/supabase/database.types.ts index 23f7eaf2b5..3bb12db728 100644 --- a/web/supabase/database.types.ts +++ b/web/supabase/database.types.ts @@ -1521,6 +1521,45 @@ export type Database = { }, ] } + online_evaluators: { + Row: { + config: Json | null + created_at: string + evaluator: string + id: number + organization: string + } + Insert: { + config?: Json | null + created_at?: string + evaluator: string + id?: number + organization: string + } + Update: { + config?: Json | null + created_at?: string + evaluator?: string + id?: number + organization?: string + } + Relationships: [ + { + foreignKeyName: "public_online_evaluators_evaluator_fkey" + columns: ["evaluator"] + isOneToOne: false + referencedRelation: "evaluator" + referencedColumns: ["id"] + }, + { + foreignKeyName: "public_online_evaluators_organization_fkey" + columns: ["organization"] + isOneToOne: false + referencedRelation: "organization" + referencedColumns: ["id"] + }, + ] + } org_rate_limit_tracker: { Row: { created_at: string | null @@ -2963,7 +3002,6 @@ export type Database = { owner_id: string | null path_tokens: string[] | null updated_at: string | null - user_metadata: Json | null version: string | null } Insert: { @@ -2977,7 +3015,6 @@ export type Database = { owner_id?: string | null path_tokens?: string[] | null updated_at?: string | null - user_metadata?: Json | null version?: string | null } Update: { @@ -2991,7 +3028,6 @@ export type Database = { owner_id?: string | null path_tokens?: string[] | null updated_at?: string | null - user_metadata?: Json | null version?: string | null } Relationships: [ @@ -3013,7 +3049,6 @@ export type Database = { key: string owner_id: string | null upload_signature: string - user_metadata: Json | null version: string } Insert: { @@ -3024,7 +3059,6 @@ export type Database = { key: string owner_id?: string | null upload_signature: string - user_metadata?: Json | null version: string } Update: { @@ -3035,7 +3069,6 @@ export type Database = { key?: string owner_id?: string | null upload_signature?: string - user_metadata?: Json | null version?: string } Relationships: [ @@ -3132,7 +3165,7 @@ export type Database = { Args: { name: string } - Returns: unknown + Returns: string[] } get_size_by_bucket: { Args: Record @@ -3172,10 +3205,6 @@ export type Database = { updated_at: string }[] } - operation: { - Args: Record - Returns: string - } search: { Args: { prefix: string diff --git a/worker/supabase/database.types.ts b/worker/supabase/database.types.ts index 23f7eaf2b5..3bb12db728 100644 --- a/worker/supabase/database.types.ts +++ b/worker/supabase/database.types.ts @@ -1521,6 +1521,45 @@ export type Database = { }, ] } + online_evaluators: { + Row: { + config: Json | null + created_at: string + evaluator: string + id: number + organization: string + } + Insert: { + config?: Json | null + created_at?: string + evaluator: string + id?: number + organization: string + } + Update: { + config?: Json | null + created_at?: string + evaluator?: string + id?: number + organization?: string + } + Relationships: [ + { + foreignKeyName: "public_online_evaluators_evaluator_fkey" + columns: ["evaluator"] + isOneToOne: false + referencedRelation: "evaluator" + referencedColumns: ["id"] + }, + { + foreignKeyName: "public_online_evaluators_organization_fkey" + columns: ["organization"] + isOneToOne: false + referencedRelation: "organization" + referencedColumns: ["id"] + }, + ] + } org_rate_limit_tracker: { Row: { created_at: string | null @@ -2963,7 +3002,6 @@ export type Database = { owner_id: string | null path_tokens: string[] | null updated_at: string | null - user_metadata: Json | null version: string | null } Insert: { @@ -2977,7 +3015,6 @@ export type Database = { owner_id?: string | null path_tokens?: string[] | null updated_at?: string | null - user_metadata?: Json | null version?: string | null } Update: { @@ -2991,7 +3028,6 @@ export type Database = { owner_id?: string | null path_tokens?: string[] | null updated_at?: string | null - user_metadata?: Json | null version?: string | null } Relationships: [ @@ -3013,7 +3049,6 @@ export type Database = { key: string owner_id: string | null upload_signature: string - user_metadata: Json | null version: string } Insert: { @@ -3024,7 +3059,6 @@ export type Database = { key: string owner_id?: string | null upload_signature: string - user_metadata?: Json | null version: string } Update: { @@ -3035,7 +3069,6 @@ export type Database = { key?: string owner_id?: string | null upload_signature?: string - user_metadata?: Json | null version?: string } Relationships: [ @@ -3132,7 +3165,7 @@ export type Database = { Args: { name: string } - Returns: unknown + Returns: string[] } get_size_by_bucket: { Args: Record @@ -3172,10 +3205,6 @@ export type Database = { updated_at: string }[] } - operation: { - Args: Record - Returns: string - } search: { Args: { prefix: string