diff --git a/server/app/models/connector.rb b/server/app/models/connector.rb index a6f230be..ea15d106 100644 --- a/server/app/models/connector.rb +++ b/server/app/models/connector.rb @@ -99,6 +99,8 @@ class Connector < ApplicationRecord scope :web, -> { where(connector_sub_category: WEB_SUB_CATEGORIES) } scope :ai_ml_service, -> { where(connector_sub_category: AI_ML_SERVICE_SUB_CATEGORIES) } scope :vector, -> { where(connector_sub_category: VECTOR_SUB_CATEGORIES) } + scope :in_host, -> { where(in_host: true) } + scope :external, -> { where(in_host: false) } def connector_definition @connector_definition ||= connector_client.new.meta_data.with_indifferent_access diff --git a/server/app/models/hosted_data_store.rb b/server/app/models/hosted_data_store.rb new file mode 100644 index 00000000..79352915 --- /dev/null +++ b/server/app/models/hosted_data_store.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class HostedDataStore < ApplicationRecord + belongs_to :workspace + belongs_to :source_connector, class_name: "Connector", optional: true + belongs_to :destination_connector, class_name: "Connector", optional: true + has_many :hosted_data_store_tables, dependent: :destroy + + enum :database_type, { vector_db: 0, raw_sql: 1 } + enum :state, { disabled: 0, enabled: 1 } + + validates :name, presence: true + validates :database_type, presence: true + validates :description, presence: true + validates :state, presence: true +end diff --git a/server/app/models/hosted_data_store_table.rb b/server/app/models/hosted_data_store_table.rb new file mode 100644 index 00000000..bd534712 --- /dev/null +++ b/server/app/models/hosted_data_store_table.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class HostedDataStoreTable < ApplicationRecord + belongs_to :hosted_data_store + belongs_to :source_connector, class_name: "Connector", optional: true + belongs_to :destination_connector, class_name: "Connector", optional: true + enum :sync_enabled, { disabled: 0, enabled: 1 } + + validates :name, presence: true + validates :column_count, presence: true + validates :row_count, presence: true + validates :size, presence: true + validates :table_schema, presence: true +end diff --git a/server/app/models/workspace.rb b/server/app/models/workspace.rb index 77703250..83b7f7a7 100644 --- a/server/app/models/workspace.rb +++ b/server/app/models/workspace.rb @@ -30,7 +30,12 @@ class Workspace < ApplicationRecord has_many :workflows, class_name: "Agents::Workflow", dependent: :destroy has_many :workflow_runs, class_name: "Agents::WorkflowRun", dependent: :destroy has_many :workflow_logs, class_name: "Agents::WorkflowLog", dependent: :nullify +<<<<<<< HEAD has_many :workflow_integrations, dependent: :nullify +======= + has_many :workflow_integrations, class_name: "Agents::WorkflowIntegration", dependent: :nullify + has_many :hosted_data_stores, dependent: :nullify +>>>>>>> a574077d (chore(CE): Add Hosted Database & Hosted Table to DB (#1393)) belongs_to :organization has_many :sso_configurations, through: :organization diff --git a/server/db/data/20251029190846_add_hosted_datastore_permissions_to_roles.rb b/server/db/data/20251029190846_add_hosted_datastore_permissions_to_roles.rb new file mode 100644 index 00000000..5b0adede --- /dev/null +++ b/server/db/data/20251029190846_add_hosted_datastore_permissions_to_roles.rb @@ -0,0 +1,84 @@ +class AddHostedDatastorePermissionsToRoles < ActiveRecord::Migration[7.1] + # rubocop:disable Metrics/MethodLength + def change + admin_role = Role.find_by(role_name: "Admin") + member_role = Role.find_by(role_name: "Member") + viewer_role = Role.find_by(role_name: "Viewer") + + admin_role&.update!( + policies: { + permissions: { + connector_definition: { create: true, read: true, update: true, delete: true }, + connector: { create: true, read: true, update: true, delete: true }, + model: { create: true, read: true, update: true, delete: true }, + report: { create: true, read: true, update: true, delete: true }, + sync_record: { create: true, read: true, update: true, delete: true }, + sync_run: { create: true, read: true, update: true, delete: true }, + sync: { create: true, read: true, update: true, delete: true }, + user: { create: true, read: true, update: true, delete: true }, + workspace: { create: true, read: true, update: true, delete: true }, + data_app: { create: true, read: true, update: true, delete: true }, + audit_logs: { create: true, read: true, update: true, delete: true }, + alerts: { create: true, read: true, update: true, delete: true }, + billing: { create: true, read: true, update: true, delete: true }, + sso: { create: true, read: true, update: true, delete: true }, + eula: { create: true, read: true, update: true, delete: true }, + assistant: { create: true, read: true, update: true, delete: true }, + workflow: { create: true, read: true, update: true, delete: true }, + hosted_datastore: { create: true, read: true, update: true, delete: true } + } + } + ) + + member_role&.update!( + policies: { + permissions: { + connector_definition: { create: true, read: true, update: true, delete: true }, + connector: { create: true, read: true, update: true, delete: true }, + model: { create: true, read: true, update: true, delete: true }, + report: { create: true, read: true, update: true, delete: true }, + sync_record: { create: true, read: true, update: true, delete: true }, + sync_run: { create: true, read: true, update: true, delete: true }, + sync: { create: true, read: true, update: true, delete: true }, + user: { create: false, read: false, update: false, delete: false }, + workspace: { create: false, read: true, update: false, delete: false }, + data_app: { create: true, read: true, update: true, delete: true }, + audit_logs: { create: true, read: true, update: true, delete: true }, + alerts: { create: true, read: true, update: true, delete: true }, + billing: { create: false, read: false, update: false, delete: false }, + sso: { create: false, read: false, update: false, delete: false }, + eula: { create: false, read: true, update: false, delete: false }, + assistant: { create: false, read: false, update: false, delete: false }, + workflow: { create: true, read: true, update: true, delete: true }, + hosted_datastore: { create: false, read: true, update: false, delete: false } + } + } + ) + + viewer_role&.update!( + policies: { + permissions: { + connector_definition: { create: false, read: true, update: false, delete: false }, + connector: { create: false, read: true, update: false, delete: false }, + model: { create: false, read: true, update: false, delete: false }, + report: { create: false, read: true, update: false, delete: false }, + sync_record: { create: false, read: true, update: false, delete: false }, + sync_run: { create: false, read: true, update: false, delete: false }, + sync: { create: false, read: true, update: false, delete: false }, + user: { create: false, read: false, update: false, delete: false }, + workspace: { create: false, read: true, update: false, delete: false }, + data_app: { create: false, read: true, update: false, delete: false }, + audit_logs: { create: false, read: false, update: false, delete: false }, + alerts: { create: false, read: true, update: false, delete: false }, + billing: { create: false, read: false, update: false, delete: false }, + sso: { create: false, read: false, update: false, delete: false }, + eula: { create: false, read: true, update: false, delete: false }, + assistant: { create: false, read: false, update: false, delete: false }, + workflow: { create: false, read: true, update: false, delete: false }, + hosted_datastore: { create: false, read: true, update: false, delete: false } + } + } + ) + end + # rubocop:enable Metrics/MethodLength +end diff --git a/server/db/data_schema.rb b/server/db/data_schema.rb index ecbc9dc3..297206b6 100644 --- a/server/db/data_schema.rb +++ b/server/db/data_schema.rb @@ -1,3 +1,7 @@ +<<<<<<< HEAD # frozen_string_literal: true DataMigrate::Data.define(version: 20250724110926) +======= +DataMigrate::Data.define(version: 20251029190846) +>>>>>>> a574077d (chore(CE): Add Hosted Database & Hosted Table to DB (#1393)) diff --git a/server/db/migrate/20251017201600_create_hosted_data_store.rb b/server/db/migrate/20251017201600_create_hosted_data_store.rb new file mode 100644 index 00000000..10a5ced5 --- /dev/null +++ b/server/db/migrate/20251017201600_create_hosted_data_store.rb @@ -0,0 +1,26 @@ +class CreateHostedDataStore < ActiveRecord::Migration[7.1] + def up + create_table :hosted_data_stores do |t| + t.string :name + t.integer :workspace_id + t.integer :database_type + t.text :description + t.integer :state + t.integer :source_connector_id + t.integer :destination_connector_id + t.string :template_id, null: false + + t.timestamps + end + + # Add the foreign key only if the parent table exists + if table_exists?(:workspaces) + add_foreign_key :hosted_data_stores, :workspaces, validate: false + end + end + + def down + remove_foreign_key :hosted_data_stores, :workspaces rescue nil + drop_table :hosted_data_stores, if_exists: true + end +end diff --git a/server/db/migrate/20251017234511_create_hosted_data_store_table.rb b/server/db/migrate/20251017234511_create_hosted_data_store_table.rb new file mode 100644 index 00000000..00c29147 --- /dev/null +++ b/server/db/migrate/20251017234511_create_hosted_data_store_table.rb @@ -0,0 +1,29 @@ +class CreateHostedDataStoreTable < ActiveRecord::Migration[7.1] + def up + create_table :hosted_data_store_tables do |t| + t.integer :hosted_data_store_id + t.string :name + t.integer :column_count + t.integer :row_count + t.integer :size + t.integer :sync_enabled + t.integer :source_connector_id + t.integer :destination_connector_id + t.jsonb :table_schema, default: {} + + t.timestamps + end + + # Add the foreign key only if the parent table exists + if table_exists?(:hosted_data_stores) + add_foreign_key :hosted_data_store_tables, :hosted_data_stores, + on_delete: :cascade, + validate: false + end + end + + def down + remove_foreign_key :hosted_data_store_tables, :hosted_data_stores rescue nil + drop_table :hosted_data_store_tables, if_exists: true + end +end diff --git a/server/db/migrate/20251017235322_add_in_host_column_to_connector.rb b/server/db/migrate/20251017235322_add_in_host_column_to_connector.rb new file mode 100644 index 00000000..db18c8ae --- /dev/null +++ b/server/db/migrate/20251017235322_add_in_host_column_to_connector.rb @@ -0,0 +1,9 @@ +class AddInHostColumnToConnector < ActiveRecord::Migration[7.1] + def up + add_column :connectors, :in_host, :boolean, default: false + end + + def down + remove_column :connectors, :in_host + end +end diff --git a/server/db/schema.rb b/server/db/schema.rb index ad5a40d2..b268bdcb 100644 --- a/server/db/schema.rb +++ b/server/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2025_10_09_173752) do +ActiveRecord::Schema[7.1].define(version: 2025_10_17_235322) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -162,6 +162,7 @@ t.string "description" t.string "connector_category", default: "data", null: false t.string "connector_sub_category", default: "database", null: false + t.boolean "in_host", default: false end create_table "custom_visual_component_files", force: :cascade do |t| @@ -244,6 +245,33 @@ t.jsonb "additional_remarks" end + create_table "hosted_data_store_tables", force: :cascade do |t| + t.integer "hosted_data_store_id" + t.string "name" + t.integer "column_count" + t.integer "row_count" + t.integer "size" + t.integer "sync_enabled" + t.integer "source_connector_id" + t.integer "destination_connector_id" + t.jsonb "table_schema", default: {} + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + + create_table "hosted_data_stores", force: :cascade do |t| + t.string "name" + t.integer "workspace_id" + t.integer "database_type" + t.text "description" + t.integer "state" + t.integer "source_connector_id" + t.integer "destination_connector_id" + t.string "template_id", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "message_feedbacks", force: :cascade do |t| t.integer "workspace_id", null: false t.integer "data_app_id", null: false @@ -709,6 +737,8 @@ add_foreign_key "edges", "components", column: "target_component_id", validate: false add_foreign_key "edges", "workflows", validate: false add_foreign_key "edges", "workspaces", validate: false + add_foreign_key "hosted_data_store_tables", "hosted_data_stores", on_delete: :cascade, validate: false + add_foreign_key "hosted_data_stores", "workspaces", validate: false add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade diff --git a/server/spec/factories/hosted_data_store.rb b/server/spec/factories/hosted_data_store.rb new file mode 100644 index 00000000..87f74f01 --- /dev/null +++ b/server/spec/factories/hosted_data_store.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :hosted_data_store do + association :workspace + association :source_connector + association :destination_connector + name { "My Hosted Data Store" } + database_type { 0 } + description { "My Hosted Data Store Description" } + state { 0 } + template_id { "vector_store_hosted_connector" } + end +end diff --git a/server/spec/factories/hosted_data_store_tables.rb b/server/spec/factories/hosted_data_store_tables.rb new file mode 100644 index 00000000..9f20a6eb --- /dev/null +++ b/server/spec/factories/hosted_data_store_tables.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :hosted_data_store_table do + association :hosted_data_store + association :source_connector + association :destination_connector + name { "My Hosted Data Store Table" } + column_count { 10 } + row_count { 100 } + size { 1000 } + sync_enabled { 0 } + table_schema do + { + "columns" => [{ "name" => "id", "type" => "integer" }, { "name" => "name", "type" => "string" }], + "rows" => [{ "id" => 1, "name" => "John Doe" }] + } + end + end +end diff --git a/server/spec/factories/roles.rb b/server/spec/factories/roles.rb index 83f54fbd..6ec309d7 100644 --- a/server/spec/factories/roles.rb +++ b/server/spec/factories/roles.rb @@ -40,7 +40,8 @@ "sso" => { "create" => true, "read" => true, "update" => true, "delete" => true }, "eula" => { "create" => true, "read" => true, "update" => true, "delete" => true }, "assistant" => { "create" => true, "read" => true, "update" => true, "delete" => true }, - "workflow" => { "create" => true, "read" => true, "update" => true, "delete" => true } + "workflow" => { "create" => true, "read" => true, "update" => true, "delete" => true }, + "hosted_datastore" => { "create" => true, "read" => true, "update" => true, "delete" => true } } } end @@ -69,7 +70,8 @@ "sso" => { "create" => false, "read" => false, "update" => false, "delete" => false }, "eula" => { "create" => false, "read" => false, "update" => false, "delete" => false }, "assistant" => { "create" => false, "read" => false, "update" => false, "delete" => false }, - "workflow" => { "create" => true, "read" => true, "update" => true, "delete" => true } + "workflow" => { "create" => true, "read" => true, "update" => true, "delete" => true }, + "hosted_datastore" => { "create" => false, "read" => true, "update" => false, "delete" => false } } } end @@ -98,7 +100,8 @@ "sso" => { "create" => false, "read" => false, "update" => false, "delete" => false }, "eula" => { "create" => false, "read" => false, "update" => false, "delete" => false }, "assistant" => { "create" => false, "read" => false, "update" => false, "delete" => false }, - "workflow" => { "create" => false, "read" => true, "update" => false, "delete" => false } + "workflow" => { "create" => false, "read" => true, "update" => false, "delete" => false }, + "hosted_datastore" => { "create" => false, "read" => true, "update" => false, "delete" => false } } } end @@ -127,7 +130,8 @@ "sso" => { "create" => false, "read" => false, "update" => false, "delete" => false }, "eula" => { "create" => false, "read" => false, "update" => false, "delete" => false }, "assistant" => { "create" => false, "read" => false, "update" => false, "delete" => false }, - "workflow" => { "create" => false, "read" => true, "update" => false, "delete" => false } + "workflow" => { "create" => false, "read" => true, "update" => false, "delete" => false }, + "hosted_datastore" => { "create" => false, "read" => true, "update" => false, "delete" => false } } } end diff --git a/server/spec/models/connector_spec.rb b/server/spec/models/connector_spec.rb index 28f2a9c9..e237fd7a 100644 --- a/server/spec/models/connector_spec.rb +++ b/server/spec/models/connector_spec.rb @@ -687,4 +687,19 @@ end end end + + describe "scopes" do + context "when a connector is in host" do + let(:connector) { create(:connector, in_host: true) } + it "returns the connector" do + expect(Connector.in_host).to include(connector) + end + end + context "when a connector is external" do + let(:connector) { create(:connector, in_host: false) } + it "returns the connector" do + expect(Connector.external).to include(connector) + end + end + end end diff --git a/server/spec/models/hosted_data_store_spec.rb b/server/spec/models/hosted_data_store_spec.rb new file mode 100644 index 00000000..b87807db --- /dev/null +++ b/server/spec/models/hosted_data_store_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe HostedDataStore, type: :model do + describe "validations" do + it { should validate_presence_of(:name) } + it { should validate_presence_of(:database_type) } + it { should validate_presence_of(:description) } + it { should validate_presence_of(:state) } + end + + describe "enum for database_type" do + it { should define_enum_for(:database_type).with_values(%i[vector_db raw_sql]) } + end + + describe "enum for state" do + it { should define_enum_for(:state).with_values(%i[disabled enabled]) } + end + + describe "associations" do + it { should belong_to(:workspace) } + it { should belong_to(:source_connector).class_name("Connector").optional } + it { should belong_to(:destination_connector).class_name("Connector").optional } + it { should have_many(:hosted_data_store_tables).dependent(:destroy) } + end +end diff --git a/server/spec/models/hosted_data_store_table_spec.rb b/server/spec/models/hosted_data_store_table_spec.rb new file mode 100644 index 00000000..25c426ba --- /dev/null +++ b/server/spec/models/hosted_data_store_table_spec.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe HostedDataStoreTable, type: :model do + describe "validations" do + it { should validate_presence_of(:name) } + it { should validate_presence_of(:column_count) } + it { should validate_presence_of(:row_count) } + it { should validate_presence_of(:size) } + it { should validate_presence_of(:table_schema) } + end + + describe "enum for sync_enabled" do + it { should define_enum_for(:sync_enabled).with_values(%i[disabled enabled]) } + end + + describe "associations" do + it { should belong_to(:hosted_data_store) } + it { should belong_to(:source_connector).class_name("Connector").optional } + it { should belong_to(:destination_connector).class_name("Connector").optional } + end +end diff --git a/server/spec/models/workspace_spec.rb b/server/spec/models/workspace_spec.rb index a1846084..04d9c1db 100644 --- a/server/spec/models/workspace_spec.rb +++ b/server/spec/models/workspace_spec.rb @@ -39,6 +39,11 @@ it { should belong_to(:organization) } it { should have_many(:workflows).dependent(:destroy) } it { should have_many(:workflow_runs).dependent(:destroy) } +<<<<<<< HEAD +======= + it { should have_many(:workflow_integrations).dependent(:nullify) } + it { should have_many(:hosted_data_stores).dependent(:nullify) } +>>>>>>> a574077d (chore(CE): Add Hosted Database & Hosted Table to DB (#1393)) end context "before_validation callbacks" do