chore(CE): Add Hosted Database & Hosted Table to DB (#1393)

server/app/models/connector.rb

@@ -99,6 +99,8 @@ class Connector < ApplicationRecord
   scope :web, -> { where(connector_sub_category: WEB_SUB_CATEGORIES) }
   scope :ai_ml_service, -> { where(connector_sub_category: AI_ML_SERVICE_SUB_CATEGORIES) }
   scope :vector, -> { where(connector_sub_category: VECTOR_SUB_CATEGORIES) }
+  scope :in_host, -> { where(in_host: true) }
+  scope :external, -> { where(in_host: false) }
 
   def connector_definition
     @connector_definition ||= connector_client.new.meta_data.with_indifferent_access

server/app/models/hosted_data_store.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class HostedDataStore < ApplicationRecord
+  belongs_to :workspace
+  belongs_to :source_connector, class_name: "Connector", optional: true
+  belongs_to :destination_connector, class_name: "Connector", optional: true
+  has_many :hosted_data_store_tables, dependent: :destroy
+
+  enum :database_type, { vector_db: 0, raw_sql: 1 }
+  enum :state, { disabled: 0, enabled: 1 }
+
+  validates :name, presence: true
+  validates :database_type, presence: true
+  validates :description, presence: true
+  validates :state, presence: true
+end

server/app/models/hosted_data_store_table.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class HostedDataStoreTable < ApplicationRecord
+  belongs_to :hosted_data_store
+  belongs_to :source_connector, class_name: "Connector", optional: true
+  belongs_to :destination_connector, class_name: "Connector", optional: true
+  enum :sync_enabled, { disabled: 0, enabled: 1 }
+
+  validates :name, presence: true
+  validates :column_count, presence: true
+  validates :row_count, presence: true
+  validates :size, presence: true
+  validates :table_schema, presence: true
+end

server/app/models/workspace.rb

@@ -30,7 +30,12 @@ class Workspace < ApplicationRecord
   has_many :workflows, class_name: "Agents::Workflow", dependent: :destroy
   has_many :workflow_runs, class_name: "Agents::WorkflowRun", dependent: :destroy
   has_many :workflow_logs, class_name: "Agents::WorkflowLog", dependent: :nullify
+<<<<<<< HEAD
   has_many :workflow_integrations, dependent: :nullify
+=======
+  has_many :workflow_integrations, class_name: "Agents::WorkflowIntegration", dependent: :nullify
+  has_many :hosted_data_stores, dependent: :nullify
+>>>>>>> a574077d (chore(CE): Add Hosted Database & Hosted Table to DB (#1393))
 
   belongs_to :organization
   has_many :sso_configurations, through: :organization

server/db/data/20251029190846_add_hosted_datastore_permissions_to_roles.rb

@@ -0,0 +1,84 @@
+class AddHostedDatastorePermissionsToRoles < ActiveRecord::Migration[7.1]
+  # rubocop:disable Metrics/MethodLength
+  def change
+    admin_role = Role.find_by(role_name: "Admin")
+    member_role = Role.find_by(role_name: "Member")
+    viewer_role = Role.find_by(role_name: "Viewer")
+
+    admin_role&.update!(
+      policies: {
+        permissions: {
+          connector_definition: { create: true, read: true, update: true, delete: true },
+          connector: { create: true, read: true, update: true, delete: true },
+          model: { create: true, read: true, update: true, delete: true },
+          report: { create: true, read: true, update: true, delete: true },
+          sync_record: { create: true, read: true, update: true, delete: true },
+          sync_run: { create: true, read: true, update: true, delete: true },
+          sync: { create: true, read: true, update: true, delete: true },
+          user: { create: true, read: true, update: true, delete: true },
+          workspace: { create: true, read: true, update: true, delete: true },
+          data_app: { create: true, read: true, update: true, delete: true },
+          audit_logs: { create: true, read: true, update: true, delete: true },
+          alerts: { create: true, read: true, update: true, delete: true },
+          billing: { create: true, read: true, update: true, delete: true },
+          sso: { create: true, read: true, update: true, delete: true },
+          eula: { create: true, read: true, update: true, delete: true },
+          assistant: { create: true, read: true, update: true, delete: true },
+          workflow: { create: true, read: true, update: true, delete: true },
+          hosted_datastore: { create: true, read: true, update: true, delete: true }
+        }
+      }
+    )
+
+    member_role&.update!(
+      policies: {
+        permissions: {
+          connector_definition: { create: true, read: true, update: true, delete: true },
+          connector: { create: true, read: true, update: true, delete: true },
+          model: { create: true, read: true, update: true, delete: true },
+          report: { create: true, read: true, update: true, delete: true },
+          sync_record: { create: true, read: true, update: true, delete: true },
+          sync_run: { create: true, read: true, update: true, delete: true },
+          sync: { create: true, read: true, update: true, delete: true },
+          user: { create: false, read: false, update: false, delete: false },
+          workspace: { create: false, read: true, update: false, delete: false },
+          data_app: { create: true, read: true, update: true, delete: true },
+          audit_logs: { create: true, read: true, update: true, delete: true },
+          alerts: { create: true, read: true, update: true, delete: true },
+          billing: { create: false, read: false, update: false, delete: false },
+          sso: { create: false, read: false, update: false, delete: false },
+          eula: { create: false, read: true, update: false, delete: false },
+          assistant: { create: false, read: false, update: false, delete: false },
+          workflow: { create: true, read: true, update: true, delete: true },
+          hosted_datastore: { create: false, read: true, update: false, delete: false }
+        }
+      }
+    )
+
+    viewer_role&.update!(
+      policies: {
+        permissions: {
+          connector_definition: { create: false, read: true, update: false, delete: false },
+          connector: { create: false, read: true, update: false, delete: false },
+          model: { create: false, read: true, update: false, delete: false },
+          report: { create: false, read: true, update: false, delete: false },
+          sync_record: { create: false, read: true, update: false, delete: false },
+          sync_run: { create: false, read: true, update: false, delete: false },
+          sync: { create: false, read: true, update: false, delete: false },
+          user: { create: false, read: false, update: false, delete: false },
+          workspace: { create: false, read: true, update: false, delete: false },
+          data_app: { create: false, read: true, update: false, delete: false },
+          audit_logs: { create: false, read: false, update: false, delete: false },
+          alerts: { create: false, read: true, update: false, delete: false },
+          billing: { create: false, read: false, update: false, delete: false },
+          sso: { create: false, read: false, update: false, delete: false },
+          eula: { create: false, read: true, update: false, delete: false },
+          assistant: { create: false, read: false, update: false, delete: false },
+          workflow: { create: false, read: true, update: false, delete: false },
+          hosted_datastore: { create: false, read: true, update: false, delete: false }
+        }
+      }
+    )
+  end
+  # rubocop:enable Metrics/MethodLength
+end

server/db/data_schema.rb

@@ -1,3 +1,7 @@
+<<<<<<< HEAD
 # frozen_string_literal: true
 
 DataMigrate::Data.define(version: 20250724110926)
+=======
+DataMigrate::Data.define(version: 20251029190846)
+>>>>>>> a574077d (chore(CE): Add Hosted Database & Hosted Table to DB (#1393))

server/db/migrate/20251017201600_create_hosted_data_store.rb

@@ -0,0 +1,26 @@
+class CreateHostedDataStore < ActiveRecord::Migration[7.1]
+  def up
+    create_table :hosted_data_stores do |t|
+      t.string :name
+      t.integer :workspace_id
+      t.integer :database_type
+      t.text :description
+      t.integer :state
+      t.integer :source_connector_id
+      t.integer :destination_connector_id
+      t.string :template_id, null: false
+
+      t.timestamps
+    end
+
+    # Add the foreign key only if the parent table exists
+    if table_exists?(:workspaces)
+      add_foreign_key :hosted_data_stores, :workspaces, validate: false
+    end
+  end
+
+  def down
+    remove_foreign_key :hosted_data_stores, :workspaces rescue nil
+    drop_table :hosted_data_stores, if_exists: true
+  end
+end

server/db/migrate/20251017234511_create_hosted_data_store_table.rb

@@ -0,0 +1,29 @@
+class CreateHostedDataStoreTable < ActiveRecord::Migration[7.1]
+  def up
+    create_table :hosted_data_store_tables do |t|
+      t.integer :hosted_data_store_id
+      t.string :name
+      t.integer :column_count
+      t.integer :row_count
+      t.integer :size
+      t.integer :sync_enabled
+      t.integer :source_connector_id
+      t.integer :destination_connector_id
+      t.jsonb :table_schema, default: {}
+
+      t.timestamps
+    end
+
+    # Add the foreign key only if the parent table exists
+    if table_exists?(:hosted_data_stores)
+      add_foreign_key :hosted_data_store_tables, :hosted_data_stores,
+                      on_delete: :cascade,
+                      validate: false
+    end
+  end
+
+  def down
+    remove_foreign_key :hosted_data_store_tables, :hosted_data_stores rescue nil
+    drop_table :hosted_data_store_tables, if_exists: true
+  end
+end

server/db/migrate/20251017235322_add_in_host_column_to_connector.rb

@@ -0,0 +1,9 @@
+class AddInHostColumnToConnector < ActiveRecord::Migration[7.1]
+  def up
+    add_column :connectors, :in_host, :boolean, default: false
+  end
+
+  def down
+    remove_column :connectors, :in_host
+  end
+end

server/spec/factories/hosted_data_store.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :hosted_data_store do
+    association :workspace
+    association :source_connector
+    association :destination_connector
+    name { "My Hosted Data Store" }
+    database_type { 0 }
+    description { "My Hosted Data Store Description" }
+    state { 0 }
+    template_id { "vector_store_hosted_connector" }
+  end
+end

server/spec/factories/hosted_data_store_tables.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :hosted_data_store_table do
+    association :hosted_data_store
+    association :source_connector
+    association :destination_connector
+    name { "My Hosted Data Store Table" }
+    column_count { 10 }
+    row_count { 100 }
+    size { 1000 }
+    sync_enabled { 0 }
+    table_schema do
+      {
+        "columns" => [{ "name" => "id", "type" => "integer" }, { "name" => "name", "type" => "string" }],
+        "rows" => [{ "id" => 1, "name" => "John Doe" }]
+      }
+    end
+  end
+end

server/spec/factories/roles.rb

@@ -40,7 +40,8 @@
             "sso" => { "create" => true, "read" => true, "update" => true, "delete" => true },
             "eula" => { "create" => true, "read" => true, "update" => true, "delete" => true },
             "assistant" => { "create" => true, "read" => true, "update" => true, "delete" => true },
-            "workflow" => { "create" => true, "read" => true, "update" => true, "delete" => true }
+            "workflow" => { "create" => true, "read" => true, "update" => true, "delete" => true },
+            "hosted_datastore" => { "create" => true, "read" => true, "update" => true, "delete" => true }
           }
         }
       end
@@ -69,7 +70,8 @@
             "sso" => { "create" => false, "read" => false, "update" => false, "delete" => false },
             "eula" => { "create" => false, "read" => false, "update" => false, "delete" => false },
             "assistant" => { "create" => false, "read" => false, "update" => false, "delete" => false },
-            "workflow" => { "create" => true, "read" => true, "update" => true, "delete" => true }
+            "workflow" => { "create" => true, "read" => true, "update" => true, "delete" => true },
+            "hosted_datastore" => { "create" => false, "read" => true, "update" => false, "delete" => false }
           }
         }
       end
@@ -98,7 +100,8 @@
             "sso" => { "create" => false, "read" => false, "update" => false, "delete" => false },
             "eula" => { "create" => false, "read" => false, "update" => false, "delete" => false },
             "assistant" => { "create" => false, "read" => false, "update" => false, "delete" => false },
-            "workflow" => { "create" => false, "read" => true, "update" => false, "delete" => false }
+            "workflow" => { "create" => false, "read" => true, "update" => false, "delete" => false },
+            "hosted_datastore" => { "create" => false, "read" => true, "update" => false, "delete" => false }
           }
         }
       end
@@ -127,7 +130,8 @@
             "sso" => { "create" => false, "read" => false, "update" => false, "delete" => false },
             "eula" => { "create" => false, "read" => false, "update" => false, "delete" => false },
             "assistant" => { "create" => false, "read" => false, "update" => false, "delete" => false },
-            "workflow" => { "create" => false, "read" => true, "update" => false, "delete" => false }
+            "workflow" => { "create" => false, "read" => true, "update" => false, "delete" => false },
+            "hosted_datastore" => { "create" => false, "read" => true, "update" => false, "delete" => false }
           }
         }
       end

server/spec/models/connector_spec.rb

@@ -687,4 +687,19 @@
       end
     end
   end
+
+  describe "scopes" do
+    context "when a connector is in host" do
+      let(:connector) { create(:connector, in_host: true) }
+      it "returns the connector" do
+        expect(Connector.in_host).to include(connector)
+      end
+    end
+    context "when a connector is external" do
+      let(:connector) { create(:connector, in_host: false) }
+      it "returns the connector" do
+        expect(Connector.external).to include(connector)
+      end
+    end
+  end
 end

server/spec/models/hosted_data_store_spec.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe HostedDataStore, type: :model do
+  describe "validations" do
+    it { should validate_presence_of(:name) }
+    it { should validate_presence_of(:database_type) }
+    it { should validate_presence_of(:description) }
+    it { should validate_presence_of(:state) }
+  end
+
+  describe "enum for database_type" do
+    it { should define_enum_for(:database_type).with_values(%i[vector_db raw_sql]) }
+  end
+
+  describe "enum for state" do
+    it { should define_enum_for(:state).with_values(%i[disabled enabled]) }
+  end
+
+  describe "associations" do
+    it { should belong_to(:workspace) }
+    it { should belong_to(:source_connector).class_name("Connector").optional }
+    it { should belong_to(:destination_connector).class_name("Connector").optional }
+    it { should have_many(:hosted_data_store_tables).dependent(:destroy) }
+  end
+end