diff --git a/packages/browser-tests/cypress/commands.js b/packages/browser-tests/cypress/commands.js index 156208b2e..6b80eafbe 100644 --- a/packages/browser-tests/cypress/commands.js +++ b/packages/browser-tests/cypress/commands.js @@ -546,3 +546,35 @@ Cypress.Commands.add("createTabWithContent", (content, title) => { cy.get(".chrome-tab[active] .chrome-tab-rename").should("not.be.visible"); } }); + +Cypress.Commands.add("uploadParquetFile", (filenames) => { + const files = Array.isArray(filenames) ? filenames : [filenames]; + files.forEach((file) => { + cy.get('[data-hook="import-dropbox"] input[type="file"]') + .last() + .selectFile(`cypress/fixtures/${file}`, { + force: true, + }); + }); +}); + +Cypress.Commands.add("mockParquetUploadSuccess", (options = {}) => { + cy.intercept("POST", "**/api/v1/imports*", { + statusCode: 201, + body: { + data: [ + { + type: "import", + id: options.fileName || "trades-original.parquet", + }, + ], + }, + }).as("parquetUpload"); +}); + +Cypress.Commands.add("mockParquetUploadError", ({ status, body }) => { + cy.intercept("POST", "**/api/v1/imports*", { + statusCode: status, + body: body, + }).as("parquetUploadError"); +}); diff --git a/packages/browser-tests/cypress/fixtures/trades-original.parquet b/packages/browser-tests/cypress/fixtures/trades-original.parquet new file mode 100644 index 000000000..d1d1b45b5 Binary files /dev/null and b/packages/browser-tests/cypress/fixtures/trades-original.parquet differ diff --git a/packages/browser-tests/cypress/fixtures/trades-original2.parquet b/packages/browser-tests/cypress/fixtures/trades-original2.parquet new file mode 100644 index 000000000..d1d1b45b5 Binary files /dev/null and b/packages/browser-tests/cypress/fixtures/trades-original2.parquet differ diff --git a/packages/browser-tests/cypress/fixtures/trades-original3.parquet b/packages/browser-tests/cypress/fixtures/trades-original3.parquet new file mode 100644 index 000000000..d1d1b45b5 Binary files /dev/null and b/packages/browser-tests/cypress/fixtures/trades-original3.parquet differ diff --git a/packages/browser-tests/cypress/integration/console/import.spec.js b/packages/browser-tests/cypress/integration/console/import.spec.js index 87d51997b..c2c051242 100644 --- a/packages/browser-tests/cypress/integration/console/import.spec.js +++ b/packages/browser-tests/cypress/integration/console/import.spec.js @@ -1,80 +1,460 @@ /// -describe("questdb import", () => { - beforeEach(() => { - cy.loadConsoleWithAuth(); - }); +describe("import", () => { + describe("CSV Import", () => { + beforeEach(() => { + cy.loadConsoleWithAuth(); + }); - afterEach(() => { - cy.loadConsoleWithAuth(); - cy.typeQueryDirectly("drop all tables;"); - cy.clickRunIconInLine(1); - cy.getByDataHook("success-notification").should("be.visible"); - cy.clearEditor(); - }); + afterEach(() => { + cy.loadConsoleWithAuth(); + cy.typeQueryDirectly("drop all tables;"); + cy.clickRunIconInLine(1); + cy.getByDataHook("success-notification").should("be.visible"); + cy.clearEditor(); + }); + + it("display import panel", () => { + cy.getByDataHook("import-panel-button").click(); + cy.getByDataHook("import-csv-button").should("be.visible"); + cy.getByDataHook("import-parquet-button").should("be.visible"); + + cy.getByDataHook("import-csv-button").click(); + cy.getByDataHook("import-dropbox").should("be.visible"); + cy.getByDataHook("import-browse-from-disk").should("be.visible"); + + cy.get('input[type="file"]').selectFile("cypress/fixtures/test.csv", { + force: true, + }); + cy.getByDataHook("import-table-column-schema").should("be.visible"); + cy.getByDataHook("import-table-column-owner").should("not.exist"); + }); + + it("should import csv with a nanosecond timestamp", () => { + cy.getByDataHook("import-panel-button").click(); + cy.getByDataHook("import-csv-button").should("be.visible"); + cy.getByDataHook("import-parquet-button").should("be.visible"); + + cy.getByDataHook("import-csv-button").click(); + cy.getByDataHook("import-dropbox").should("be.visible"); + cy.getByDataHook("import-browse-from-disk").should("be.visible"); + + cy.get('input[type="file"]').selectFile("cypress/fixtures/nanos.csv", { + force: true, + }); + cy.getByDataHook("import-table-column-schema").should("be.visible"); + cy.getByDataHook("import-upload-button").should("be.enabled"); + cy.getByDataHook("import-upload-button").click(); + + cy.getByDataHook("import-file-status").should( + "contain", + "Imported 7 rows" + ); + cy.getByDataHook("schema-table-title").should("contain", "nanos.csv"); + + cy.getByDataHook("schema-table-title").dblclick(); + cy.getByDataHook("schema-folder-title").contains("Columns").dblclick(); + cy.get('[data-id="questdb:expanded:tables:nanos.csv:columns:timestamp"]') + .should("be.visible") + .should("contain", "timestamp") + .should("contain", "TIMESTAMP_NS"); + cy.getByDataHook("designated-timestamp-icon").should("not.exist"); + + cy.getByDataHook("table-schema-dialog-trigger") + .should("be.visible") + .should("contain", "4 cols"); + cy.getByDataHook("table-schema-dialog-trigger").click(); + + cy.getByDataHook("create-table-panel").should("be.visible"); + cy.getByDataHook("table-schema-dialog-column-0").should("be.visible"); + cy.get("input[name='schemaColumns.0.name']").should( + "have.value", + "timestamp" + ); + + cy.get("select[name='schemaColumns.0.type']") + .get("option[value='TIMESTAMP_NS']") + .should("be.selected"); + + cy.getByDataHook( + "table-schema-dialog-column-0-designated-button" + ).click(); + cy.getByDataHook("form-submit-button").click(); + + cy.getByDataHook("create-table-panel").should("not.be.visible"); + + cy.get('select[name="overwrite"]').select("true"); - it("display import panel", () => { - cy.getByDataHook("import-panel-button").click(); - cy.getByDataHook("import-dropbox").should("be.visible"); - cy.getByDataHook("import-browse-from-disk").should("be.visible"); + cy.getByDataHook("import-upload-button").should("be.enabled"); + cy.getByDataHook("import-upload-button").click(); - cy.get('input[type="file"]').selectFile("cypress/fixtures/test.csv", { - force: true, + cy.getByDataHook("import-file-status").should( + "contain", + "Imported 7 rows" + ); + cy.getByDataHook("designated-timestamp-icon").should("be.visible"); }); - cy.getByDataHook("import-table-column-schema").should("be.visible"); - cy.getByDataHook("import-table-column-owner").should("not.exist"); }); + describe("Parquet import", () => { + beforeEach(() => { + cy.loadConsoleWithAuth(); + }); + + describe("Basic Upload Operations", () => { + it("should display parquet import panel", () => { + cy.getByDataHook("import-panel-button").click(); + cy.getByDataHook("import-csv-button").should("be.visible"); + cy.getByDataHook("import-parquet-button").should("be.visible"); + + cy.getByDataHook("import-parquet-button").click(); + cy.getByDataHook("import-dropbox").should("be.visible"); + cy.getByDataHook("import-browse-from-disk").should("be.visible"); + }); + + it("should successfully upload a single parquet file", () => { + cy.mockParquetUploadSuccess(); + cy.getByDataHook("import-panel-button").click(); + cy.getByDataHook("import-parquet-button").click(); + + cy.uploadParquetFile("trades-original.parquet"); + cy.getByDataHook("import-parquet-file-name").should("be.visible"); + cy.getByDataHook("import-parquet-file-name").should( + "contain", + "trades-original.parquet" + ); + + cy.getByDataHook("import-parquet-upload-all").click(); + cy.wait("@parquetUpload"); + + cy.getByDataHook("import-parquet-status", { timeout: 3000 }).should( + "contain", + "Uploaded 1 file successfully" + ); + }); + + it("should handle multiple parquet file uploads", () => { + cy.mockParquetUploadSuccess(); + cy.getByDataHook("import-panel-button").click(); + cy.getByDataHook("import-parquet-button").click(); + + cy.uploadParquetFile("trades-original.parquet"); + cy.uploadParquetFile("trades-original2.parquet"); + cy.uploadParquetFile("trades-original3.parquet"); - it("should import csv with a nanosecond timestamp", () => { - cy.getByDataHook("import-panel-button").click(); - cy.getByDataHook("import-dropbox").should("be.visible"); - cy.getByDataHook("import-browse-from-disk").should("be.visible"); + cy.getByDataHook("import-parquet-file-name").should("have.length", 3); + cy.getByDataHook("import-parquet-upload-all").click(); + cy.wait("@parquetUpload"); + cy.getByDataHook("import-parquet-status", { timeout: 3000 }).should( + "contain", + "Uploaded 3 files successfully" + ); - cy.get('input[type="file"]').selectFile("cypress/fixtures/nanos.csv", { - force: true, + cy.getByDataHook("import-file-status").should(($statuses) => { + expect($statuses).to.have.length(3); + $statuses.each((_, status) => { + expect(status).to.contain.text("Imported"); + }); + }); + }); }); - cy.getByDataHook("import-table-column-schema").should("be.visible"); - cy.getByDataHook("import-upload-button").should("be.enabled"); - cy.getByDataHook("import-upload-button").click(); - cy.getByDataHook("import-file-status").should("contain", "Imported 7 rows"); - cy.getByDataHook("schema-table-title").should("contain", "nanos.csv"); + describe("Error Handling", () => { + it("should handle 409 conflict when file already exists", () => { + cy.mockParquetUploadError({ + status: 409, + body: { + errors: [ + { + meta: { name: "trades-original.parquet" }, + detail: "file already exists [file=trades-original]", + status: "409", + }, + ], + }, + }); - cy.getByDataHook("schema-table-title").dblclick(); - cy.getByDataHook("schema-folder-title").contains("Columns").dblclick(); - cy.get('[data-id="questdb:expanded:tables:nanos.csv:columns:timestamp"]') - .should("be.visible") - .should("contain", "timestamp") - .should("contain", "TIMESTAMP_NS"); - cy.getByDataHook("designated-timestamp-icon").should("not.exist"); + cy.getByDataHook("import-panel-button").click(); + cy.getByDataHook("import-parquet-button").click(); + cy.uploadParquetFile("trades-original.parquet"); + cy.getByDataHook("import-parquet-upload-all").click(); + cy.wait("@parquetUploadError"); - cy.getByDataHook("table-schema-dialog-trigger") - .should("be.visible") - .should("contain", "4 cols"); - cy.getByDataHook("table-schema-dialog-trigger").click(); + cy.getByDataHook("import-file-status").should("be.visible"); - cy.getByDataHook("create-table-panel").should("be.visible"); - cy.getByDataHook("table-schema-dialog-column-0").should("be.visible"); - cy.get("input[name='schemaColumns.0.name']").should( - "have.value", - "timestamp" - ); + cy.getByDataHook("import-file-status").within(() => { + cy.getByDataHook("import-file-status-expand").click({ force: true }); + }); - cy.get("select[name='schemaColumns.0.type']") - .get("option[value='TIMESTAMP_NS']") - .should("be.selected"); + cy.getByDataHook("import-file-status-details").should( + "contain", + "file already exists" + ); + cy.getByDataHook("import-parquet-retry-upload").should("be.visible"); + }); - cy.getByDataHook("table-schema-dialog-column-0-designated-button").click(); - cy.getByDataHook("form-submit-button").click(); + it("should show network error message", () => { + cy.intercept("POST", "**/api/v1/imports*", (req) => { + req.destroy(); // Prevent the request from reaching the server + }).as("networkError"); + + cy.getByDataHook("import-panel-button").click(); + cy.getByDataHook("import-parquet-button").click(); + cy.uploadParquetFile("trades-original.parquet"); + cy.getByDataHook("import-parquet-upload-all").click(); + cy.wait("@networkError"); + + cy.getByDataHook("import-file-status").should("be.visible"); + cy.getByDataHook("import-file-status").should("contain", "Cancelled"); + + cy.getByDataHook("import-parquet-status").should("be.visible"); + cy.getByDataHook("import-parquet-status").should( + "contain", + "Upload error" + ); + }); + + it("should handle server error gracefully", () => { + cy.mockParquetUploadError({ + status: 500, + body: { + errors: [ + { + meta: { name: "trades-original.parquet" }, + detail: "internal server error", + status: "500", + }, + ], + }, + }); + + cy.getByDataHook("import-panel-button").click(); + cy.getByDataHook("import-parquet-button").click(); + cy.uploadParquetFile("trades-original.parquet"); + cy.getByDataHook("import-parquet-upload-all").click(); + cy.wait("@parquetUploadError"); + + cy.getByDataHook("import-file-status").should("be.visible"); + + cy.getByDataHook("import-file-status").within(() => { + cy.getByDataHook("import-file-status-expand").click({ force: true }); + }); + + cy.getByDataHook("import-file-status-details").should( + "contain", + "internal server error" + ); + }); + }); - cy.getByDataHook("create-table-panel").should("not.be.visible"); + describe("File Operations", () => { + it("should allow renaming files before upload", () => { + cy.getByDataHook("import-panel-button").click(); + cy.getByDataHook("import-parquet-button").click(); + cy.uploadParquetFile("trades-original.parquet"); + cy.getByDataHook("import-parquet-file-name").should("be.visible"); - cy.get('select[name="overwrite"]').select("true"); + cy.getByDataHook("import-parquet-rename-file").first().click(); - cy.getByDataHook("import-upload-button").should("be.enabled"); - cy.getByDataHook("import-upload-button").click(); + cy.getByDataHook("import-parquet-rename-file-input").should( + "be.visible" + ); + cy.getByDataHook("import-parquet-rename-file-input") + .clear() + .type("btc_trades_2024"); + cy.getByDataHook("import-parquet-rename-file-submit").click(); - cy.getByDataHook("import-file-status").should("contain", "Imported 7 rows"); - cy.getByDataHook("designated-timestamp-icon").should("be.visible"); + cy.getByDataHook("import-parquet-rename-file") + .first() + .should("contain", "btc_trades_2024"); + }); + + it("should remove files from upload queue", () => { + cy.getByDataHook("import-panel-button").click(); + cy.getByDataHook("import-parquet-button").click(); + + cy.uploadParquetFile("trades-original.parquet"); + cy.uploadParquetFile("trades-original2.parquet"); + cy.getByDataHook("import-parquet-file-name").should("have.length", 2); + + cy.getByDataHook("import-parquet-remove-file").first().click(); + cy.getByDataHook("import-parquet-file-name").should("have.length", 1); + cy.getByDataHook("import-parquet-file-name").should( + "not.contain", + "trades-original.parquet" + ); + + cy.getByDataHook("import-parquet-remove-file").should("have.length", 1); + cy.getByDataHook("import-parquet-remove-file").click(); + cy.getByDataHook("import-dropbox").should("be.visible"); + }); + + it("should respect overwrite mode in API calls", () => { + cy.intercept("POST", "**/api/v1/imports?overwrite=false", { + statusCode: 201, + body: { data: [{ type: "import", id: "trades-original.parquet" }] }, + }).as("overwriteFalse"); + + cy.getByDataHook("import-panel-button").click(); + cy.getByDataHook("import-parquet-button").click(); + cy.uploadParquetFile("trades-original.parquet"); + cy.getByDataHook("import-parquet-upload-all").click(); + cy.wait("@overwriteFalse"); + + cy.getByDataHook("import-parquet-overwrite").click(); + + cy.intercept("POST", "**/api/v1/imports?overwrite=true", { + statusCode: 201, + body: { data: [{ type: "import", id: "trades-original2.parquet" }] }, + }).as("overwriteTrue"); + + cy.uploadParquetFile("trades-original2.parquet"); + cy.getByDataHook("import-parquet-upload-all").click(); + cy.wait("@overwriteTrue"); + }); + }); + + describe("Batch Upload with Retry Mechanism", () => { + it("should continue with remaining files after partial failure", () => { + let callCount = 0; + cy.intercept("POST", "**/api/v1/imports*", (req) => { + if (callCount === 0) { + callCount++; + req.reply({ + statusCode: 409, + body: { + errors: [ + { + meta: { name: "trades-original2.parquet" }, + detail: + "file already exists [file=trades-original2.parquet]", + status: "409", + }, + ], + }, + }); + } else { + req.reply({ + statusCode: 201, + body: { + data: [{ type: "import", id: "trades-original3.parquet" }], + }, + }); + } + }).as("batchUpload"); + + cy.getByDataHook("import-panel-button").click(); + cy.getByDataHook("import-parquet-button").click(); + + cy.uploadParquetFile("trades-original.parquet"); + cy.uploadParquetFile("trades-original2.parquet"); + cy.uploadParquetFile("trades-original3.parquet"); + + cy.getByDataHook("import-parquet-upload-all").click(); + cy.wait("@batchUpload"); + cy.wait("@batchUpload"); + + cy.getByDataHook("import-file-status").should("have.length", 3); + + cy.getByDataHook("import-file-status") + .eq(0) + .should("contain", "Imported"); + cy.getByDataHook("import-file-status") + .eq(1) + .should("contain", "Upload error"); + cy.getByDataHook("import-file-status") + .eq(2) + .should("contain", "Imported"); + }); + + it("should allow retrying failed single file upload", () => { + cy.mockParquetUploadError({ + status: 409, + body: { + errors: [ + { + meta: { name: "trades-original.parquet" }, + detail: "file already exists [file=trades-original.parquet]", + status: "409", + }, + ], + }, + }); + + cy.getByDataHook("import-panel-button").click(); + cy.getByDataHook("import-parquet-button").click(); + cy.uploadParquetFile("trades-original.parquet"); + cy.getByDataHook("import-parquet-upload-all").click(); + cy.wait("@parquetUploadError"); + + cy.getByDataHook("import-parquet-retry-upload").should("be.visible"); + + cy.mockParquetUploadSuccess(); + cy.getByDataHook("import-parquet-retry-upload").click(); + cy.wait("@parquetUpload"); + + cy.getByDataHook("import-file-status").should("contain", "Imported"); + }); + }); + + describe("UI States and Interactions", () => { + it("should show loading state during upload", () => { + cy.intercept("POST", "**/api/v1/imports*", (req) => { + req.reply({ + delay: 2000, + statusCode: 201, + body: { data: [{ type: "import", id: "trades-original.parquet" }] }, + }); + }).as("slowUpload"); + + cy.getByDataHook("import-panel-button").click(); + cy.getByDataHook("import-parquet-button").click(); + cy.uploadParquetFile("trades-original.parquet"); + cy.getByDataHook("import-parquet-upload-all").click(); + + cy.getByDataHook("import-parquet-upload-all").should("be.disabled"); + + cy.getByDataHook("import-parquet-status").should( + "contain", + "Uploading..." + ); + + cy.wait("@slowUpload"); + + cy.getByDataHook("import-parquet-upload-all").should("not.be.disabled"); + }); + + it("should allow viewing data after successful upload", () => { + cy.getByDataHook("import-panel-button").click(); + cy.getByDataHook("import-parquet-button").click(); + cy.uploadParquetFile("trades-original.parquet"); + + cy.getByDataHook("import-parquet-overwrite").click(); + cy.getByDataHook("import-parquet-upload-all").click(); + + cy.getByDataHook("import-parquet-view-data").first().click(); + cy.getByDataHook("success-notification").should( + "contain", + "SELECT * FROM read_parquet('trades-original.parquet')" + ); + + cy.getGrid().should("be.visible"); + cy.getColumnName(0).should("contain", "symbol"); + }); + + it("should handle empty response gracefully", () => { + cy.intercept("POST", "**/api/v1/imports*", { + statusCode: 201, + body: {}, + }).as("emptyResponse"); + + cy.getByDataHook("import-panel-button").click(); + cy.getByDataHook("import-parquet-button").click(); + cy.uploadParquetFile("trades-original.parquet"); + cy.getByDataHook("import-parquet-upload-all").click(); + cy.wait("@emptyResponse"); + + cy.getByDataHook("import-file-status").should("contain", "Imported"); + }); + }); }); }); diff --git a/packages/browser-tests/questdb b/packages/browser-tests/questdb index c113299ed..6b16453c4 160000 --- a/packages/browser-tests/questdb +++ b/packages/browser-tests/questdb @@ -1 +1 @@ -Subproject commit c113299ed19ad020bc57415ef00658a2a17af147 +Subproject commit 6b16453c412a964f2f1bc8ea72f278bd26bac807 diff --git a/packages/web-console/assets/csv-file.svg b/packages/web-console/assets/csv-file.svg new file mode 100644 index 000000000..7713717a7 --- /dev/null +++ b/packages/web-console/assets/csv-file.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/packages/web-console/assets/parquet-file.svg b/packages/web-console/assets/parquet-file.svg new file mode 100644 index 000000000..3c62a6676 --- /dev/null +++ b/packages/web-console/assets/parquet-file.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/packages/web-console/serve-dist.js b/packages/web-console/serve-dist.js index a1a5fe2c9..1e35140a4 100644 --- a/packages/web-console/serve-dist.js +++ b/packages/web-console/serve-dist.js @@ -22,7 +22,8 @@ const server = http.createServer((req, res) => { reqPathName.startsWith("/settings") || reqPathName.startsWith("/warnings") || reqPathName.startsWith("/chk") || - reqPathName.startsWith("/imp") + reqPathName.startsWith("/imp") || + reqPathName.startsWith("/api/") ) { // proxy /exec requests to localhost:9000 const options = { diff --git a/packages/web-console/src/components/Toast/index.tsx b/packages/web-console/src/components/Toast/index.tsx index 4468e7ad0..9af7c356b 100644 --- a/packages/web-console/src/components/Toast/index.tsx +++ b/packages/web-console/src/components/Toast/index.tsx @@ -9,7 +9,7 @@ import { } from "react-toastify" import { useNotificationCenter as RTNotificationCenter } from "react-toastify/addons/use-notification-center" import { NotificationCenterItem as RNotificationCenterItem } from "react-toastify/addons/use-notification-center/useNotificationCenter" -import { BadgeType } from "../../scenes/Import/ImportCSVFiles/types" +import { BadgeType } from "../../scenes/Import/FileStatus" import { CloseCircle, ErrorWarning, diff --git a/packages/web-console/src/consts/index.ts b/packages/web-console/src/consts/index.ts index 481421172..b06b19892 100644 --- a/packages/web-console/src/consts/index.ts +++ b/packages/web-console/src/consts/index.ts @@ -38,4 +38,6 @@ export const API = `https://${BASE}.questdb.io` // the console will understand export const API_VERSION = "2" +export const API_ROUTE_V1 = "/api/v1" + export const BUTTON_ICON_SIZE = "26px" diff --git a/packages/web-console/src/modules/ZeroState/start.tsx b/packages/web-console/src/modules/ZeroState/start.tsx index 3a3a48987..4939bdbb0 100644 --- a/packages/web-console/src/modules/ZeroState/start.tsx +++ b/packages/web-console/src/modules/ZeroState/start.tsx @@ -66,28 +66,32 @@ export const Start = () => { + onClick={() => { + dispatch(actions.console.setImportType("parquet")) dispatch(actions.console.setActiveBottomPanel("import")) - } + }} > File upload icon - Import CSV + Import Parquet dispatch(actions.console.setActiveSidebar("create"))} + onClick={() => { + dispatch(actions.console.setImportType("csv")) + dispatch(actions.console.setActiveBottomPanel("import")) + }} > Create table icon - Create table + Import CSV diff --git a/packages/web-console/src/scenes/Console/import.tsx b/packages/web-console/src/scenes/Console/import.tsx deleted file mode 100644 index 21cf4a0e9..000000000 --- a/packages/web-console/src/scenes/Console/import.tsx +++ /dev/null @@ -1,26 +0,0 @@ -import React from "react" -import { PaneContent, PaneWrapper } from "../../components" -import { ImportCSVFiles } from "../Import/ImportCSVFiles" -import { eventBus } from "../../modules/EventBus" -import { EventType } from "../../modules/EventBus/types" - -export const Import = () => ( - - - { - eventBus.publish(EventType.MSG_QUERY_SCHEMA) - }} - onViewData={(result) => { - if (result.status === "OK") { - eventBus.publish(EventType.MSG_QUERY_SCHEMA) - eventBus.publish(EventType.MSG_QUERY_FIND_N_EXEC, { - query: `"${result.location}"`, - options: { appendAt: "end" }, - }) - } - }} - /> - - -) diff --git a/packages/web-console/src/scenes/Console/index.tsx b/packages/web-console/src/scenes/Console/index.tsx index 31b499d86..8cf1182c9 100644 --- a/packages/web-console/src/scenes/Console/index.tsx +++ b/packages/web-console/src/scenes/Console/index.tsx @@ -13,11 +13,12 @@ import { actions, selectors } from "../../store" import { Tooltip } from "../../components" import { Sidebar } from "../../components/Sidebar" import { Navigation } from "../../components/Sidebar/navigation" -import { Database2, Grid, PieChart, Search, FileSearch } from "@styled-icons/remix-line" +import { Database2, Grid, PieChart, FileSearch } from "@styled-icons/remix-line" import { ResultViewMode } from "./types" import { BUTTON_ICON_SIZE } from "../../consts" import { PrimaryToggleButton } from "../../components" -import { Import } from "./import" +import { Import } from "../Import" +import { DropdownMenu } from "../../components/DropdownMenu" import { BottomPanel } from "../../store/Console/types" import { Allotment, AllotmentHandle } from "allotment" import { Import as ImportIcon } from "../../components/icons/import" @@ -78,6 +79,7 @@ const Console = () => { useLocalStorage() const result = useSelector(selectors.query.getResult) const activeBottomPanel = useSelector(selectors.console.getActiveBottomPanel) + const importType = useSelector(selectors.console.getImportType) const { consoleConfig } = useSettings() const { isSearchPanelOpen, setSearchPanelOpen, searchPanelRef } = useSearch() @@ -231,29 +233,64 @@ const Console = () => { {tooltipText} ))} - { - dispatch(actions.console.setActiveBottomPanel("import")) - }, - })} - selected={activeBottomPanel === "import"} - data-hook="import-panel-button" + {consoleConfig.readOnly ? ( + + + + } + > + + To use this feature, turn off read-only mode in the configuration file + + + ) : ( + + + + + + + } > - - - } - > - - {consoleConfig.readOnly - ? "To use this feature, turn off read-only mode in the configuration file" - : "Import files from CSV"} - - + Import data + + + + { + dispatch(actions.console.setImportType("parquet")) + dispatch(actions.console.setActiveBottomPanel("import")) + }} + > + Import Parquet + + { + dispatch(actions.console.setImportType("csv")) + dispatch(actions.console.setActiveBottomPanel("import")) + }} + > + Import CSV + + + + + )} {result && } @@ -262,7 +299,7 @@ const Console = () => { - + diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/dropbox.tsx b/packages/web-console/src/scenes/Import/Dropbox.tsx similarity index 73% rename from packages/web-console/src/scenes/Import/ImportCSVFiles/dropbox.tsx rename to packages/web-console/src/scenes/Import/Dropbox.tsx index a1f3596c2..8a12c20c1 100644 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/dropbox.tsx +++ b/packages/web-console/src/scenes/Import/Dropbox.tsx @@ -1,7 +1,6 @@ import React, { useRef, useState, useEffect } from "react" import styled from "styled-components" -import { Box } from "../../../components/Box" -import { ProcessedFile } from "./types" +import { Box } from "../../components/Box" const getFileDuplicates = ( inputFiles: FileList, @@ -13,39 +12,37 @@ const getFileDuplicates = ( return duplicates } -const Root = styled(Box).attrs({ flexDirection: "column" })<{ - isDragging: boolean -}>` +const Root = styled(Box).attrs({ flexDirection: "column" })<{ $isDragging: boolean }>` flex: 1; width: 100%; - padding: 4rem 0 0; gap: 2rem; - background: ${({ theme }) => theme.color.backgroundLighter}; - border: 3px dashed ${({ isDragging }) => (isDragging ? "#7f839b" : "#333543")}; + background: ${({ theme, $isDragging }) => $isDragging ? theme.color.selectionDarker : theme.color.backgroundLighter}; box-shadow: inset 0 0 10px 0 #1b1c23; transition: all 0.15s ease-in-out; ` type Props = { - files: ProcessedFile[] + existingFileNames: string[] onFilesDropped: (files: File[]) => void - dialogOpen: boolean + dialogOpen?: boolean + enablePaste?: boolean render: (props: { duplicates: File[] addToQueue: (inputFiles: FileList) => void + uploadInputRef: React.RefObject }) => React.ReactNode } -export const DropBox = ({ - files, +export const Dropbox = ({ + existingFileNames, onFilesDropped, - dialogOpen, + dialogOpen = false, + enablePaste = true, render, }: Props) => { const [isDragging, setIsDragging] = useState(false) const [duplicates, setDuplicates] = useState([]) const uploadInputRef = useRef(null) - const filenames = useRef(files.map((f) => f.table_name)) const handleDrag = (e: React.DragEvent) => { e.preventDefault() @@ -61,7 +58,7 @@ export const DropBox = ({ } const addToQueue = (inputFiles: FileList) => { - const duplicates = getFileDuplicates(inputFiles, filenames.current) + const duplicates = getFileDuplicates(inputFiles, existingFileNames) setDuplicates(duplicates) onFilesDropped( Array.from(inputFiles).filter((f) => !duplicates.includes(f)), @@ -78,22 +75,22 @@ export const DropBox = ({ } useEffect(() => { + if (!enablePaste) return + return () => { window.removeEventListener("paste", handlePaste) } - }, []) + }, [enablePaste]) useEffect(() => { + if (!enablePaste) return + if (dialogOpen) { window.removeEventListener("paste", handlePaste) } else { window.addEventListener("paste", handlePaste) } - }, [dialogOpen]) - - useEffect(() => { - filenames.current = files.map((f) => f.table_name) - }, [files]) + }, [dialogOpen, enablePaste]) return ( - {render({ duplicates, addToQueue })} + {render({ duplicates, addToQueue, uploadInputRef })} ) -} +} \ No newline at end of file diff --git a/packages/web-console/src/scenes/Import/DropboxUploadArea.tsx b/packages/web-console/src/scenes/Import/DropboxUploadArea.tsx new file mode 100644 index 000000000..d264e5c7a --- /dev/null +++ b/packages/web-console/src/scenes/Import/DropboxUploadArea.tsx @@ -0,0 +1,97 @@ +import React from "react" +import { Button, Heading, Text } from "@questdb/react-components" +import { Box } from "../../components/Box" +import { Upload2 } from "@styled-icons/remix-line" +import styled from "styled-components" + +const BrowseTextLink = styled.span` + text-decoration: underline; + cursor: pointer; + + &:hover { + text-decoration: none; + } +` + +type Props = { + title: string + accept: string + uploadInputRef: React.RefObject + addToQueue: (inputFiles: FileList) => void + duplicates: File[] + mode?: "initial" | "list" + children?: React.ReactNode +} + +export const DropboxUploadArea = ({ + title, + accept, + uploadInputRef, + addToQueue, + duplicates, + mode = "initial", + children +}: Props) => { + const handleFileChange = (e: React.ChangeEvent) => { + if (e.target.files) { + addToQueue(e.target.files) + e.target.value = "" + } + } + + return ( + <> + + + {mode === "initial" ? ( + + + + + {title} + + + + {duplicates.length > 0 && ( + + File{duplicates.length > 1 ? "s" : ""} already added to queue:{" "} + {duplicates.map((f) => f.name).join(", ")}. Change target table + name and try again. + + )} + {children} + + ) : ( + <> + + You can drag and drop more files or{" "} + uploadInputRef.current?.click()}> + browse from disk + + + {duplicates.length > 0 && ( + + File{duplicates.length > 1 ? "s" : ""} already added to queue:{" "} + {duplicates.map((f) => f.name).join(", ")}. Change import name + and try again. + + )} + + )} + + ) +} \ No newline at end of file diff --git a/packages/web-console/src/scenes/Import/FileStatus.tsx b/packages/web-console/src/scenes/Import/FileStatus.tsx new file mode 100644 index 000000000..33c589537 --- /dev/null +++ b/packages/web-console/src/scenes/Import/FileStatus.tsx @@ -0,0 +1,196 @@ +import React, { useState } from "react" +import { FileCheckStatus as FileStatusType, CSVUploadResult } from "../../utils" +import { Badge } from "@questdb/react-components" +import { Box } from "../../components/Box" +import styled from "styled-components" +import { ChevronDown } from "@styled-icons/boxicons-solid" +import { Error as ErrorIcon } from "@styled-icons/boxicons-regular" +import { CheckboxCircle } from "@styled-icons/remix-fill" +import { Text } from "../../components/Text" +import { ColorShape } from "../../types/styled" +import { ProcessedCSV } from "./ImportCSVFiles/types" +import { ProcessedParquet } from "./ImportParquet/types" + +export enum BadgeType { + SUCCESS = "success", + INFO = "info", + WARNING = "warning", + ERROR = "error", +} + +const CheckboxCircleIcon = styled(CheckboxCircle)` + color: ${({ theme }) => theme.color.green}; +` + +const ChevronIcon = styled(ChevronDown)<{ $expanded?: boolean; $color?: keyof ColorShape }>` + transform: rotate(${({ $expanded }) => $expanded ? "180deg" : "0deg"}); + cursor: pointer; + position: relative; + z-index: 1; + color: ${({ $color, theme }) => theme.color[$color ?? "gray2"]}; +` + +const ExclamationCircleIcon = styled(ErrorIcon)` + color: ${({ theme }) => theme.color.red}; +` + +const StyledBadge = styled(Badge)` + display: flex; + align-items: flex-start; + flex-direction: column; + gap: 0.5rem; + min-height: 3rem; + height: unset; +` + +const StyledBox = styled(Box)` + gap: 0.5rem; + white-space: nowrap; + height: 3rem; +` + +const FileTextBox = styled(Box)` + padding: 0.5rem 0; + text-align: left; + width: 350px; +` + +const mapStatusToLabel = ( + file: ProcessedCSV | ProcessedParquet, +): + | { + label: string + type: BadgeType + icon?: React.ReactNode + } + | undefined => { + // For Parquet files + if ('file_name' in file) { + if (file.isUploading) { + return { + label: "Uploading...", + type: BadgeType.WARNING, + } + } + if (file.uploaded) { + return { + label: "Imported", + type: BadgeType.SUCCESS, + icon: , + } + } + if (file.error) { + return { + label: "Upload error", + type: BadgeType.ERROR, + icon: , + } + } + if (file.cancelled) { + return { + label: "Cancelled", + type: BadgeType.ERROR, + icon: , + } + } + return { + label: "Ready to upload", + type: BadgeType.SUCCESS, + icon: , + } + } + // For CSV files + else if (!file.isUploading && file.uploaded && file.uploadResult) { + let label = "Imported" + const csvResult = file.uploadResult + label += ` ${csvResult.rowsImported.toLocaleString()} row${ + csvResult.rowsImported > 1 || + csvResult.rowsImported === 0 + ? "s" + : "" + }` + return { + label, + type: BadgeType.SUCCESS, + icon: , + } + } + + if (file.error) { + return { + label: "Upload error", + type: BadgeType.ERROR, + icon: , + } + } + + if (file.isUploading) { + return { + label: `Uploading: ${(file.uploadProgress || 0).toFixed(2)}%`, + type: BadgeType.WARNING, + } + } + + if ('status' in file) { + switch (file.status) { + case FileStatusType.EXISTS: + return { + label: "Table already exists", + type: BadgeType.WARNING, + } + case FileStatusType.RESERVED_NAME: + return { + label: "Reserved table name", + type: BadgeType.ERROR, + } + case FileStatusType.DOES_NOT_EXIST: + return { + label: "Ready to upload", + type: BadgeType.SUCCESS, + } + } + } +} + +const mapStatusToColor = (type: BadgeType): keyof ColorShape => { + switch (type) { + case BadgeType.SUCCESS: + return "green" + case BadgeType.ERROR: + return "red" + case BadgeType.WARNING: + return "orange" + case BadgeType.INFO: + return "cyan" + default: + return "gray2" + } +} + +export const FileStatus = ({ file }: { file: ProcessedCSV | ProcessedParquet }) => { + const [expanded, setExpanded] = useState(false) + const mappedStatus = mapStatusToLabel(file) + const statusDetails = file.error + + if (!mappedStatus) { + return null + } + + return ( + + + + {mappedStatus.icon} {mappedStatus.label} + {statusDetails && setExpanded(!expanded)} $color={mapStatusToColor(mappedStatus.type)} />} + + {expanded && statusDetails && ( + + + {statusDetails} + + + )} + + + ) +} diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/CSVUploadList.tsx b/packages/web-console/src/scenes/Import/ImportCSVFiles/CSVUploadList.tsx new file mode 100644 index 000000000..6d6dba6f5 --- /dev/null +++ b/packages/web-console/src/scenes/Import/ImportCSVFiles/CSVUploadList.tsx @@ -0,0 +1,316 @@ +import React, { useState, useEffect } from "react" +import styled from "styled-components" +import { Table, Button, Select } from "@questdb/react-components" +import { Column } from "@questdb/react-components/dist/components/Table" +import { Box } from "../../../components/Box" +import { Text } from "../../../components/Text" +import { Grid, Information } from "@styled-icons/remix-line" +import { ProcessedCSV } from "./types" +import { CSVUploadResult, UploadModeSettings } from "../../../utils" +import { RenameTableDialog } from "./rename-table-dialog" +import { FileStatus } from "../FileStatus" +import { UploadActions } from "./upload-actions" +import { UploadResultDialog } from "./upload-result-dialog" +import { shortenText } from "../../../utils" +import { bytesWithSuffix } from "../../../utils/bytesWithSuffix" +import { PopperHover, Tooltip } from "../../../components" +import { Dialog as TableSchemaDialog } from "../../../components/TableSchemaDialog/dialog" + +const StyledTable = styled(Table)` + width: 100%; + table-layout: fixed; + border-collapse: separate; + border-spacing: 0 2rem; + + th { + padding: 0 1.5rem; + } + + td { + padding: 1.5rem; + } + + tbody td { + background: #242531; + + &:first-child { + border-top-left-radius: ${({ theme }) => theme.borderRadius}; + border-bottom-left-radius: ${({ theme }) => theme.borderRadius}; + } + + &:last-child { + border-top-right-radius: ${({ theme }) => theme.borderRadius}; + border-bottom-right-radius: ${({ theme }) => theme.borderRadius}; + } + } +` + +const FileTextBox = styled(Box)` + padding: 0 1.1rem; +` + +interface Props { + files: ProcessedCSV[] + ownedByList: string[] + onFileRemove: (id: string) => void + onFileUpload: (id: string) => void + onFilePropertyChange: (id: string, file: Partial) => void + onViewData: (query: string) => void + onDialogToggle: (open: boolean) => void +} + +export const CSVUploadList = ({ + files, + ownedByList, + onFileRemove, + onFileUpload, + onFilePropertyChange, + onViewData, + onDialogToggle, +}: Props) => { + const [renameDialogOpen, setRenameDialogOpen] = useState() + const [schemaDialogOpen, setSchemaDialogOpen] = useState() + + useEffect(() => { + onDialogToggle(renameDialogOpen !== undefined || schemaDialogOpen !== undefined) + }, [renameDialogOpen, schemaDialogOpen, onDialogToggle]) + + if (files.length === 0) { + return null + } + + const columns: Column[] = [ + { + header: "CSV File", + align: "flex-start", + width: "400px", + render: ({ data }) => { + const needsTooltip = data.fileObject.name.length > 20 + const fileInfo = ( + + + {shortenText(data.fileObject.name, 20)} + + + {bytesWithSuffix(data.fileObject.size)} + + + ) + + return ( + + + CSV file icon + + {needsTooltip ? ( + + {data.fileObject.name} + + ) : ( + fileInfo + )} + + + {!data.isUploading && data.uploadResult && ( + <> + + + + )} + + + + {data.uploadResult && data.uploadResult.rowsRejected > 0 && ( + + + {data.uploadResult.rowsRejected.toLocaleString()} row + {data.uploadResult.rowsRejected > 1 ? "s" : ""} rejected + + + )} + + ) + }, + }, + { + header: "Table name", + align: "flex-end", + width: "180px", + render: ({ data }) => ( + setRenameDialogOpen(f?.id)} + onNameChange={(name) => { + onFilePropertyChange(data.id, { table_name: name }) + }} + file={data} + /> + ), + }, + ] + + if (ownedByList.length > 0) { + columns.push({ + header: ( + + Table owner + + + } + > + Required for external (non-database) users. + + ), + align: "center", + width: "150px", + render: ({ data }) => ( + ) => + onFilePropertyChange(data.id, { + settings: { + ...data.settings, + overwrite: e.target.value === "true", + }, + }) + } + options={[ + { + label: "Append", + value: "false", + }, + { + label: "Overwrite", + value: "true", + }, + ]} + /> + ), + }) + columns.push({ + align: "flex-end", + width: "300px", + render: ({ data }) => ( + onFileUpload(data.id)} + onRemove={() => onFileRemove(data.id)} + onSettingsChange={(settings: UploadModeSettings) => { + onFilePropertyChange(data.id, { settings }) + }} + /> + ), + }) + + return ( + + ) +} \ No newline at end of file diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/file-status.tsx b/packages/web-console/src/scenes/Import/ImportCSVFiles/file-status.tsx deleted file mode 100644 index ec9116dbe..000000000 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/file-status.tsx +++ /dev/null @@ -1,86 +0,0 @@ -import React from "react" -import { FileCheckStatus as FileStatusType } from "../../../utils" -import { Badge } from "@questdb/react-components" -import { BadgeType, ProcessedFile } from "./types" -import { Box } from "../../../components/Box" -import styled from "styled-components" -import { CheckboxCircle } from "@styled-icons/remix-fill" - -const CheckboxCircleIcon = styled(CheckboxCircle)` - color: ${({ theme }) => theme.color.green}; -` - -const StyledBox = styled(Box)` - gap: 0.5rem; - white-space: nowrap; -` - -const mapStatusToLabel = ( - file: ProcessedFile, -): - | { - label: string - type: BadgeType - icon?: React.ReactNode - } - | undefined => { - if (!file.isUploading && file.uploaded && file.uploadResult) { - return { - label: `Imported ${file.uploadResult.rowsImported.toLocaleString()} row${ - file.uploadResult.rowsImported > 1 || - file.uploadResult.rowsImported === 0 - ? "s" - : "" - }`, - type: BadgeType.SUCCESS, - icon: , - } - } - - if (file.error) { - return { - label: "Upload error", - type: BadgeType.ERROR, - } - } - - if (file.isUploading) { - return { - label: `Uploading: ${file.uploadProgress.toFixed(2)}%`, - type: BadgeType.WARNING, - } - } - - switch (file.status) { - case FileStatusType.EXISTS: - return { - label: "Table already exists", - type: BadgeType.WARNING, - } - case FileStatusType.RESERVED_NAME: - return { - label: "Reserved table name", - type: BadgeType.ERROR, - } - case FileStatusType.DOES_NOT_EXIST: - return { - label: "Ready to upload", - type: BadgeType.SUCCESS, - } - } -} - -export const FileStatus = ({ file }: { file: ProcessedFile }) => { - const mappedStatus = mapStatusToLabel(file) - return mappedStatus ? ( - - - - {mappedStatus.icon} {mappedStatus.label} - - - - ) : ( - <> - ) -} diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/files-to-upload.tsx b/packages/web-console/src/scenes/Import/ImportCSVFiles/files-to-upload.tsx index 120cb1654..60f71e2e8 100644 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/files-to-upload.tsx +++ b/packages/web-console/src/scenes/Import/ImportCSVFiles/files-to-upload.tsx @@ -1,54 +1,17 @@ -import React, { useEffect, useRef } from "react" +import React from "react" import styled from "styled-components" -import { Heading, Table, Select, Button } from "@questdb/react-components" -import { Column, Props as TableProps } from "@questdb/react-components/dist/components/Table" -import { PopperHover, Text, Tooltip } from "../../../components" +import { Heading } from "@questdb/react-components" +import { Text } from "../../../components" import { Box } from "../../../components/Box" -import { bytesWithSuffix } from "../../../utils/bytesWithSuffix" -import { FileStatus } from "./file-status" -import { Grid, Information } from "@styled-icons/remix-line" -import { FiletypeCsv } from "@styled-icons/bootstrap/FiletypeCsv" -import { ProcessedFile } from "./types" -import { UploadActions } from "./upload-actions" -import { RenameTableDialog } from "./rename-table-dialog" -import { Dialog as TableSchemaDialog } from "../../../components/TableSchemaDialog/dialog" -import { UploadResultDialog } from "./upload-result-dialog" -import { shortenText, UploadResult } from "../../../utils" -import { DropBox } from "./dropbox" +import { ProcessedCSV } from "./types" +import { Dropbox } from "../Dropbox" +import { DropboxUploadArea } from "../DropboxUploadArea" +import { CSVUploadList } from "./CSVUploadList" const Root = styled(Box).attrs({ flexDirection: "column", gap: "2rem" })` padding: 2rem; ` -const StyledTable = styled(Table)` - width: 100%; - table-layout: fixed; - border-collapse: separate; - border-spacing: 0 2rem; - - th { - padding: 0 1.5rem; - } - - td { - padding: 1.5rem; - } - - tbody td { - background: #242531; - - &:first-child { - border-top-left-radius: ${({ theme }) => theme.borderRadius}; - border-bottom-left-radius: ${({ theme }) => theme.borderRadius}; - } - - &:last-child { - border-top-right-radius: ${({ theme }) => theme.borderRadius}; - border-bottom-right-radius: ${({ theme }) => theme.borderRadius}; - } - } -` - const EmptyState = styled(Box).attrs({ justifyContent: "center" })` width: 100%; background: #242531; @@ -56,27 +19,14 @@ const EmptyState = styled(Box).attrs({ justifyContent: "center" })` padding: 1rem; ` -const FileTextBox = styled(Box)` - padding: 0 1.1rem; -` - -const BrowseTextLink = styled.span` - text-decoration: underline; - cursor: pointer; - - &:hover { - text-decoration: none; - } -` - type Props = { - files: ProcessedFile[] + files: ProcessedCSV[] onDialogToggle: (open: boolean) => void - onFileRemove: (id: string) => void - onFileUpload: (id: string) => void - onFilePropertyChange: (id: string, file: Partial) => void + onFileRemove: (filename: string) => void + onFileUpload: (filename: string) => void + onFilePropertyChange: (id: string, file: Partial) => void onFilesDropped: (files: File[]) => void - onViewData: (result: UploadResult) => void + onViewData: (query: string) => void dialogOpen: boolean ownedByList: string[] } @@ -92,324 +42,42 @@ export const FilesToUpload = ({ dialogOpen, ownedByList, }: Props) => { - const uploadInputRef = useRef(null) - const [renameDialogOpen, setRenameDialogOpen] = React.useState< - string | undefined - >() + const existingFileNames = files.map((f) => f.table_name) - const [schemaDialogOpen, setSchemaDialogOpen] = React.useState< - string | undefined - >() - - useEffect(() => { - onDialogToggle( - renameDialogOpen !== undefined || schemaDialogOpen !== undefined, - ) - }, [renameDialogOpen, schemaDialogOpen]) - - const columns: Column[] = [] - columns.push( - { - header: "File", - align: "flex-start", - ...(files.length > 0 && { width: "400px" }), - render: ({ data }) => { - const file = ( - - - {shortenText(data.fileObject.name, 20)} - - - - {bytesWithSuffix(data.fileObject.size)} - - - ) - return ( - - - - {data.fileObject.name.length > 20 && ( - - {data.fileObject.name} - - )} - {data.fileObject.name.length <= 20 && file} - - - {!data.isUploading && - data.uploadResult !== undefined && ( - - - - - )} - - {(data.uploadResult && - data.uploadResult.rowsRejected > 0) || - (data.error && ( - - {data.uploadResult && - data.uploadResult.rowsRejected > 0 && ( - - {data.uploadResult.rowsRejected.toLocaleString()}{" "} - row - {data.uploadResult.rowsRejected > 1 - ? "s" - : ""}{" "} - rejected - - )} - {data.error && ( - - {data.error} - - )} - - ))} - - - ) - }, - }, - { - header: "Table name", - align: "flex-end", - width: "180px", - render: ({ data }) => { - return ( - setRenameDialogOpen(f?.id)} - onNameChange={(name) => { - onFilePropertyChange(data.id, { - table_name: name, - }) - }} - file={data} - /> - ) - }, - }, - ) - - if (ownedByList && ownedByList.length > 0) { - columns.push( - { - header: ( - - Table owner - - - } - > - - Required for external (non-database) users. - - - ), - align: "center", - width: "150px", - render: ({ data }) => ( - ) => - onFilePropertyChange(data.id, { - settings: { - ...data.settings, - overwrite: e.target.value === "true", - }, - }) - } - options={[ - { - label: "Append", - value: "false", - }, - { - label: "Overwrite", - value: "true", - }, - ]} - /> - ), - }, - { - align: "flex-end", - width: "300px", - render: ({ data }) => ( - { - onFilePropertyChange(data.id, { - settings, - }) - }} - /> - ), - }, - ) - return ( - ( + enablePaste={true} + render={({ duplicates, addToQueue, uploadInputRef }) => ( - Upload queue - { - if (e.target.files === null) return - addToQueue(e.target.files) - }} - multiple={true} - ref={uploadInputRef} - style={{ display: "none" }} - value="" - /> - - You can drag and drop more files or{" "} - { - uploadInputRef.current?.click() - }} - > - browse from disk - - - {duplicates.length > 0 && ( - - File{duplicates.length > 1 ? "s" : ""} already added to queue:{" "} - {duplicates.map((f) => f.name).join(", ")}. Change target table - name and try again. - - )} - >> - columns={columns} - rows={files} + Import queue + - {files.length === 0 && ( + + {files.length === 0 ? ( No files in queue + ) : ( + onFilePropertyChange(id, file)} + onViewData={onViewData} + onDialogToggle={onDialogToggle} + /> )} )} diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/index.tsx b/packages/web-console/src/scenes/Import/ImportCSVFiles/index.tsx index cf0353049..dd6a6ae94 100644 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/index.tsx +++ b/packages/web-console/src/scenes/Import/ImportCSVFiles/index.tsx @@ -2,11 +2,11 @@ import React, { useEffect, useRef, useState } from "react" import styled from "styled-components" import { Box } from "../../../components/Box" import { FilesToUpload } from "./files-to-upload" -import { ProcessedFile } from "./types" +import { ProcessedCSV } from "./types" import { SchemaColumn } from "components/TableSchemaDialog/types" import { useContext } from "react" import { QuestContext } from "../../../providers" -import { pick, UploadResult, FileCheckStatus, Parameter } from "../../../utils" +import { pick, FileCheckStatus, Parameter, FileUploadResult } from "../../../utils" import * as QuestDB from "../../../utils/questdb" import { useSelector } from "react-redux" import { selectors } from "../../../store" @@ -28,8 +28,8 @@ import { ssoAuthState } from "../../../modules/OAuth2/ssoAuthState" type State = "upload" | "list" type Props = { - onViewData: (result: UploadResult) => void - onUpload: (result: UploadResult) => void + onViewData: (query: string) => void + onUpload: (result: FileUploadResult) => void } const Root = styled(Box).attrs({ gap: "4rem", flexDirection: "column" })` @@ -38,7 +38,7 @@ const Root = styled(Box).attrs({ gap: "4rem", flexDirection: "column" })` export const ImportCSVFiles = ({ onViewData, onUpload }: Props) => { const { quest } = useContext(QuestContext) - const [filesDropped, setFilesDropped] = useState([]) + const [filesDropped, setFilesDropped] = useState([]) const [dialogOpen, setDialogOpen] = useState(false) const tables = useSelector(selectors.query.getTables) const rootRef = useRef(null) @@ -89,7 +89,7 @@ export const ImportCSVFiles = ({ onViewData, onUpload }: Props) => { getOwnedByList() }, []) - const setFileProperties = (id: string, file: Partial) => { + const setFileProperties = (id: string, file: Partial) => { setFilesDropped((files) => files.map((f) => { if (f.id === id) { @@ -103,11 +103,11 @@ export const ImportCSVFiles = ({ onViewData, onUpload }: Props) => { ) } - const setIsUploading = (file: ProcessedFile, isUploading: boolean) => { + const setIsUploading = (file: ProcessedCSV, isUploading: boolean) => { setFileProperties(file.id, { isUploading }) } - const getFileConfigs = async (files: File[]): Promise => { + const getFileConfigs = async (files: File[]): Promise => { return await Promise.all( files.map(async (file) => { const result = await quest.checkCSVFile(file.name) @@ -168,6 +168,7 @@ export const ImportCSVFiles = ({ onViewData, onUpload }: Props) => { return { id: uuid(), + type: "csv", fileObject: file, table_name: file.name, table_owner: ownedByList[0], @@ -220,6 +221,12 @@ export const ImportCSVFiles = ({ onViewData, onUpload }: Props) => { } }, [isVisible]) + useEffect(() => { + if (filesDropped.length === 0) { + setState("upload") + } + }, [filesDropped]) + return ( {state === "upload" && ( @@ -239,8 +246,7 @@ export const ImportCSVFiles = ({ onViewData, onUpload }: Props) => { onDialogToggle={setDialogOpen} ownedByList={ownedByList} onFileUpload={async (id) => { - const file = filesDropped.find((f) => f.id === id) as ProcessedFile - + const file = filesDropped.find((f) => f.id === id) as ProcessedCSV if (file.isUploading) { return } @@ -290,18 +296,18 @@ export const ImportCSVFiles = ({ onViewData, onUpload }: Props) => { }) setIsUploading(file, false) onUpload(response) - } catch (err) { + } catch (err: any) { setIsUploading(file, false) setFileProperties(file.id, { uploaded: false, uploadResult: undefined, uploadProgress: 0, - error: "Upload error", + error: err.statusText || "Upload error", }) } }} onFileRemove={(id) => { - const file = filesDropped.find((f) => f.id === id) as ProcessedFile + const file = filesDropped.find((f) => f.id === id) as ProcessedCSV setFilesDropped( filesDropped.filter( (f) => f.fileObject.name !== file.fileObject.name, @@ -312,15 +318,20 @@ export const ImportCSVFiles = ({ onViewData, onUpload }: Props) => { const processedFiles = await Promise.all( filesDropped.map(async (file) => { if (file.id === id) { - // Only check for file existence if table name is changed - const result = partialFile.table_name - ? await quest.checkCSVFile(partialFile.table_name) - : await Promise.resolve({ status: file.status }) - return { - ...file, - ...partialFile, - status: result.status, - error: partialFile.table_name ? undefined : file.error, // reset prior error if table name is changed + if (partialFile.table_name) { + // Only check for file existence if table name is changed + const result = await quest.checkCSVFile(partialFile.table_name) + return { + ...file, + ...partialFile, + status: result.status, + error: undefined, // reset prior error if table name is changed + } + } else { + return { + ...file, + ...partialFile, + } } } else { return file diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/rename-table-dialog.tsx b/packages/web-console/src/scenes/Import/ImportCSVFiles/rename-table-dialog.tsx index b0ba4fb2c..dfd05d4bf 100644 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/rename-table-dialog.tsx +++ b/packages/web-console/src/scenes/Import/ImportCSVFiles/rename-table-dialog.tsx @@ -1,5 +1,5 @@ import React from "react" -import { ProcessedFile } from "./types" +import { ProcessedCSV } from "./types" import { Dialog, ForwardRef, Button, Overlay } from "@questdb/react-components" import { Edit } from "@styled-icons/remix-line" import { Undo } from "@styled-icons/boxicons-regular" @@ -27,9 +27,9 @@ const StyledDescription = styled(Dialog.Description)` type Props = { open: boolean - onOpenChange: (file?: ProcessedFile) => void + onOpenChange: (file?: ProcessedCSV) => void onNameChange: (name: string) => void - file: ProcessedFile + file: ProcessedCSV } const schema = Joi.object({ diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/types.ts b/packages/web-console/src/scenes/Import/ImportCSVFiles/types.ts index dd68e06af..e45799064 100644 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/types.ts +++ b/packages/web-console/src/scenes/Import/ImportCSVFiles/types.ts @@ -1,10 +1,14 @@ -import { UploadResult, UploadModeSettings } from "utils" +import { CSVUploadResult, UploadModeSettings } from "utils" import { SchemaColumn } from "../../../components/TableSchemaDialog/types" -export type ProcessedFile = { +export type ProcessedCSV = { id: string fileObject: File status: string + isUploading: boolean + uploaded: boolean + uploadProgress: number + error?: string table_name: string table_owner: string settings: UploadModeSettings @@ -13,18 +17,6 @@ export type ProcessedFile = { timestamp: string ttlValue: number ttlUnit: string - isUploading: boolean - uploaded: boolean - uploadResult?: UploadResult - uploadProgress: number - error?: string + uploadResult?: CSVUploadResult exists: boolean } - -// TODO: Refactor @questdb/react-components/Badge to ditch enum as prop value -export enum BadgeType { - SUCCESS = "success", - INFO = "info", - WARNING = "warning", - ERROR = "error", -} diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/upload-actions.tsx b/packages/web-console/src/scenes/Import/ImportCSVFiles/upload-actions.tsx index 350217e30..315e82713 100644 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/upload-actions.tsx +++ b/packages/web-console/src/scenes/Import/ImportCSVFiles/upload-actions.tsx @@ -1,5 +1,5 @@ import React, { useState } from "react" -import { ProcessedFile } from "./types" +import { ProcessedCSV } from "./types" import { Button } from "@questdb/react-components" import { PopperHover, Tooltip } from "../../../components" import { Box } from "../../../components/Box" @@ -8,7 +8,7 @@ import { UploadSettingsDialog } from "./upload-settings-dialog" import { UploadModeSettings } from "../../../utils" type Props = { - file: ProcessedFile + file: ProcessedCSV onUpload: (filename: string) => void onRemove: (filename: string) => void onSettingsChange: (settings: UploadModeSettings) => void diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/upload-result-dialog.tsx b/packages/web-console/src/scenes/Import/ImportCSVFiles/upload-result-dialog.tsx index 08cd58383..a57defd94 100644 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/upload-result-dialog.tsx +++ b/packages/web-console/src/scenes/Import/ImportCSVFiles/upload-result-dialog.tsx @@ -1,5 +1,5 @@ import React from "react" -import { ProcessedFile } from "./types" +import { ProcessedCSV } from "./types" import { Button, Table } from "@questdb/react-components" import type { Props as TableProps } from "@questdb/react-components/dist/components/Table" import { Search } from "@styled-icons/remix-line" @@ -66,7 +66,7 @@ const NotificationCircle = styled.span` ` type Props = { - file: ProcessedFile + file: ProcessedCSV } export const UploadResultDialog = ({ file }: Props) => { diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/upload-settings-dialog.tsx b/packages/web-console/src/scenes/Import/ImportCSVFiles/upload-settings-dialog.tsx index 64ecef426..3464b8f05 100644 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/upload-settings-dialog.tsx +++ b/packages/web-console/src/scenes/Import/ImportCSVFiles/upload-settings-dialog.tsx @@ -1,6 +1,6 @@ import React from "react" import styled from "styled-components" -import { ProcessedFile } from "./types" +import { ProcessedCSV } from "./types" import { Button, Select, Switch, Input } from "@questdb/react-components" import { Box } from "../../../components/Box" import { Text } from "../../../components/Text" @@ -44,7 +44,7 @@ const InputWrapper = styled.div` type Props = { open: boolean onOpenChange: (value: boolean) => void - file: ProcessedFile + file: ProcessedCSV onSubmit: (settings: UploadModeSettings) => void } diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/upload.tsx b/packages/web-console/src/scenes/Import/ImportCSVFiles/upload.tsx index f11ccb1b4..89acea0f2 100644 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/upload.tsx +++ b/packages/web-console/src/scenes/Import/ImportCSVFiles/upload.tsx @@ -1,17 +1,12 @@ -import React, { useContext, useEffect, useRef, useState } from "react" +import React, { useContext, useEffect, useState } from "react" import styled from "styled-components" -import { ProcessedFile } from "./types" -import { DropBox } from "./dropbox" -import { Search2 } from "@styled-icons/remix-line" -import { Box } from "../../../components/Box" +import { ProcessedCSV } from "./types" +import { Dropbox } from "../Dropbox" +import { DropboxUploadArea } from "../DropboxUploadArea" import { Text } from "@questdb/react-components" -import { Button, Heading } from "@questdb/react-components" import { Parameter } from "../../../utils" import { QuestContext } from "../../../providers" -const Actions = styled(Box).attrs({ flexDirection: "column", gap: "2rem" })` - margin: auto; -` const Info = styled.div` margin-top: 1rem; @@ -31,7 +26,7 @@ const InfoText = styled(Text)` ` type Props = { - files: ProcessedFile[] + files: ProcessedCSV[] onFilesDropped: (files: File[]) => void dialogOpen: boolean } @@ -49,7 +44,6 @@ const CopySQLLink = () => ( export const Upload = ({ files, onFilesDropped, dialogOpen }: Props) => { const { quest } = useContext(QuestContext) const [copyEnabled, setCopyEnabled] = useState(false) - const uploadInputRef = useRef(null) const enableCopyIfParamExists = async () => { try { @@ -70,51 +64,19 @@ export const Upload = ({ files, onFilesDropped, dialogOpen }: Props) => { }, []) return ( - f.table_name)} onFilesDropped={onFilesDropped} dialogOpen={dialogOpen} - render={({ duplicates, addToQueue }) => ( - - - File upload icon - - Drag CSV files here or paste from clipboard - - { - if (e.target.files === null) return - addToQueue(e.target.files) - }} - multiple={true} - ref={uploadInputRef} - style={{ display: "none" }} - value="" - /> - - {duplicates.length > 0 && ( - - File{duplicates.length > 1 ? "s" : ""} already added to queue:{" "} - {duplicates.map((f) => f.name).join(", ")}. Change target table - name and try again. - - )} + render={({ duplicates, addToQueue, uploadInputRef }) => ( + {copyEnabled ? ( @@ -130,8 +92,7 @@ export const Upload = ({ files, onFilesDropped, dialogOpen }: Props) => { )} - - + )} /> ) diff --git a/packages/web-console/src/scenes/Import/ImportParquet/ParquetFileList.tsx b/packages/web-console/src/scenes/Import/ImportParquet/ParquetFileList.tsx new file mode 100644 index 000000000..9378e3748 --- /dev/null +++ b/packages/web-console/src/scenes/Import/ImportParquet/ParquetFileList.tsx @@ -0,0 +1,197 @@ +import React, { useCallback, useRef, useState } from "react" +import styled from "styled-components" +import { Table, Button } from "@questdb/react-components" +import { Column } from "@questdb/react-components/dist/components/Table" +import { Box } from "../../../components/Box" +import { Text } from "../../../components/Text" +import { Close } from "@styled-icons/remix-line" +import { Eye } from "@styled-icons/remix-line" +import { ProcessedParquet } from "./types" +import { bytesWithSuffix } from "../../../utils/bytesWithSuffix" +import { PopperHover, Tooltip } from "../../../components" +import { RenameFileDialog } from "./rename-file-dialog" +import { FileStatus } from "../FileStatus" +import { shortenText } from "../../../utils" + +const StyledTable = styled(Table)` + width: 100%; + table-layout: fixed; + border-collapse: separate; + border-spacing: 0 2rem; + + th { + padding: 0 1.5rem; + } + + td { + padding: 1.5rem; + } + + tbody td { + background: #242531; + + &:first-child { + border-top-left-radius: ${({ theme }) => theme.borderRadius}; + border-bottom-left-radius: ${({ theme }) => theme.borderRadius}; + } + + &:last-child { + border-top-right-radius: ${({ theme }) => theme.borderRadius}; + border-bottom-right-radius: ${({ theme }) => theme.borderRadius}; + } + } +` + +const FileTextBox = styled(Box)` + padding: 0 1.1rem; +` + +const FileSize = styled(Text)` + font-size: 13px; + line-height: 2; +` + +const HiddenInput = styled.input` + display: none; +` + +interface Props { + files: ProcessedParquet[] + onRemoveFile: (id: string) => void + onFileNameChange: (id: string, name: string) => void + onAddMoreFiles: (files: File[]) => void + onViewData: (query: string) => void + onSingleFileUpload: (id: string) => void + isUploading: boolean +} + +export const ParquetFileList = ({ + files, + onRemoveFile, + onFileNameChange, + onAddMoreFiles, + onViewData, + onSingleFileUpload, + isUploading, +}: Props) => { + const fileInputRef = useRef(null) + const [renameDialogOpen, setRenameDialogOpen] = useState() + + const handleFileInputChange = useCallback((e: React.ChangeEvent) => { + if (e.target.files) { + onAddMoreFiles(Array.from(e.target.files)) + e.target.value = "" + } + }, [onAddMoreFiles]) + + const columns: Column[] = [ + { + header: "Parquet File", + align: "flex-start", + width: "400px", + render: ({ data }) => { + const fileInfo = ( + + + {shortenText(data.fileObject.name, 40)} + + + {bytesWithSuffix(data.fileObject.size)} + + + ) + + return ( + + + Parquet icon + + {fileInfo} + + + {data.uploaded && ( + + )} + + + + + ) + }, + }, + { + header: "Import Path", + align: "flex-end", + width: "400px", + render: ({ data }) => ( + setRenameDialogOpen(f?.id)} + onNameChange={(name) => { + onFileNameChange(data.id, name) + }} + file={data} + /> + ), + }, + { + header: "", + align: "flex-end", + width: "300px", + render: ({ data }) => ( + + {data.error && ( + + )} + onRemoveFile(data.id)} + > + + + } + > + Remove file from queue + + + ), + }, + ] + + return ( + <> + + + + ) +} \ No newline at end of file diff --git a/packages/web-console/src/scenes/Import/ImportParquet/index.tsx b/packages/web-console/src/scenes/Import/ImportParquet/index.tsx new file mode 100644 index 000000000..8e0b462b6 --- /dev/null +++ b/packages/web-console/src/scenes/Import/ImportParquet/index.tsx @@ -0,0 +1,350 @@ +import React, { useCallback, useState } from "react" +import styled from "styled-components" +import { Heading, Button, Switch, Loader } from "@questdb/react-components" +import { Box } from "../../../components/Box" +import { Text } from "../../../components/Text" +import { Dropbox } from "../Dropbox" +import { DropboxUploadArea } from "../DropboxUploadArea" +import { ParquetFileList } from "./ParquetFileList" +import { ParquetUploadError, ProcessedParquet, UploadError } from "./types" +import { useContext } from "react" +import { QuestContext } from "../../../providers" +import { CheckmarkOutline, CloseOutline } from "@styled-icons/evaicons-outline" +import { theme } from "../../../theme" +import { uuid } from "../ImportCSVFiles/utils" + +const Root = styled(Box).attrs({ gap: "3rem", flexDirection: "column" })` + flex: 1; +` + +const ControlPanel = styled(Box).attrs({ gap: "2rem", justifyContent: "space-between" })` + align-self: flex-end; +` + +const UploadButton = styled(Button)` + min-width: 150px; +` + +const CheckmarkIcon = styled(CheckmarkOutline)` + color: ${({ theme }) => theme.color.green}; + flex-shrink: 0; +` + +const CloseIcon = styled(CloseOutline)` + color: ${({ theme }) => theme.color.red}; + flex-shrink: 0; +` + +type State = "upload" | "list" + +type Props = { + onViewData: (query: string) => void +} + +export const ImportParquet = ({ onViewData }: Props) => { + const { quest } = useContext(QuestContext) + const [files, setFiles] = useState([]) + const [state, setState] = useState("upload") + const [overwrite, setOverwrite] = useState(false) + const [isUploading, setIsUploading] = useState(false) + const [status, setStatus] = useState<{ type: "error" | "success" | "warning", message: string } | undefined>(undefined) + + const handleFilesDropped = useCallback((droppedFiles: File[]) => { + const newFiles: ProcessedParquet[] = droppedFiles.map((file) => ({ + id: uuid(), + fileObject: file, + file_name: file.name, + })) + + setFiles((prevFiles) => [...prevFiles, ...newFiles]) + setState("list") + }, []) + + const handleRemoveFile = useCallback((id: string) => { + setFiles((prevFiles) => { + const newFiles = prevFiles.filter((f) => f.id !== id) + if (newFiles.length === 0) { + setState("upload") + } + return newFiles + }) + }, []) + + const handleFileNameChange = useCallback((id: string, name: string) => { + setFiles((prevFiles) => + prevFiles.map((f) => (f.id === id ? { ...f, file_name: name } : f)) + ) + }, []) + + const handleUploadAll = useCallback(async () => { + if (files.length === 0 || isUploading) return + + setIsUploading(true) + setStatus({ type: "warning", message: `Uploading...` }) + setFiles(prevFiles => + prevFiles.map(f => ({ ...f, isUploading: true, cancelled: false, error: undefined, uploaded: false })) + ) + + let remainingFiles = files.map((file, index) => ({ + file: file.fileObject, + name: file.file_name, + originalIndex: index + })) + + const failedFiles: number[] = [] + let successCount = 0 + let processedCount = 0 + + while (remainingFiles.length > 0) { + try { + const filesToUpload = remainingFiles.map(f => ({ + file: f.file, + name: f.name + })) + + await quest.uploadParquetFiles( + filesToUpload, + overwrite + ) + + const successfulIndices = remainingFiles.map(f => f.originalIndex) + successCount += remainingFiles.length + processedCount += remainingFiles.length + + setFiles(prevFiles => + prevFiles.map((f, i) => { + if (successfulIndices.includes(i)) { + return { ...f, isUploading: false, uploaded: true, error: undefined } + } + return f + }) + ) + break + + } catch (error) { + const uploadError = error as UploadError + + if (!uploadError.response) { + setStatus({ type: "error", message: `Upload error: ${uploadError.statusText || 'Upload failed'}` }) + setFiles(prevFiles => + prevFiles.map(f => ({ ...f, isUploading: false, uploaded: false })) + ) + break + } + + try { + const errorData = JSON.parse(uploadError.response) as ParquetUploadError + if (errorData.errors && Array.isArray(errorData.errors)) { + const uploadError = errorData.errors[0] + const errorFileName = uploadError.meta.name + + const failedFileIndex = remainingFiles.findIndex(f => f.name === errorFileName) + + if (failedFileIndex !== -1) { + const failedFile = remainingFiles[failedFileIndex] + failedFiles.push(failedFile.originalIndex) + + const successfulFiles = remainingFiles.slice(0, failedFileIndex) + successCount += successfulFiles.length + processedCount += failedFileIndex + 1 + + setStatus({ + type: "warning", + message: `Uploading...${processedCount > 0 ? ` (Processed ${processedCount}/${files.length})` : ''}` + }) + + setFiles(prevFiles => + prevFiles.map((f, i) => { + if (i === failedFile.originalIndex) { + return { ...f, error: uploadError.detail, isUploading: false, uploaded: false } + } + if (successfulFiles.some(sf => sf.originalIndex === i)) { + return { ...f, isUploading: false, uploaded: true } + } + return f + }) + ) + + remainingFiles = remainingFiles.slice(failedFileIndex + 1) + + if (remainingFiles.length > 0) { + continue + } + } else { + setStatus({ type: "error", message: "Failed to identify the problematic file" }) + break + } + } else { + setStatus({ type: "error", message: `Server error: ${uploadError.statusText || 'Unknown error'}` }) + break + } + } catch (parseError) { + setStatus({ type: "error", message: `Failed to parse error response: ${uploadError.statusText || 'Unknown error'}` }) + break + } + } + } + + setFiles(prevFiles => + prevFiles.map((f, i) => { + if (!failedFiles.includes(i) && !f.uploaded && !f.error) { + return { ...f, isUploading: false, cancelled: true } + } + return { ...f, isUploading: false } + }) + ) + + if (successCount === files.length) { + setStatus({ type: "success", message: `Uploaded ${files.length} file${files.length > 1 ? "s" : ""} successfully` }) + } else if (failedFiles.length > 0) { + setStatus({ type: "error", message: `Failed after uploading ${successCount}/${files.length} files` }) + } + + setIsUploading(false) + }, [files, overwrite, quest, isUploading]) + + const handleSingleFileUpload = useCallback(async (id: string) => { + const file = files.find(f => f.id === id) + if (!file || isUploading) return + + setIsUploading(true) + setFiles(prevFiles => + prevFiles.map(f => f.id === id + ? { ...f, isUploading: true, uploaded: false, error: undefined } + : f + ) + ) + + try { + await quest.uploadParquetFiles( + [{ file: file.fileObject, name: file.file_name }], + overwrite + ) + + setFiles(prevFiles => + prevFiles.map(f => f.id === id + ? { ...f, isUploading: false, uploaded: true, error: undefined } + : f + ) + ) + } catch (error) { + const uploadError = error as UploadError + + if (!uploadError.response) { + setFiles(prevFiles => + prevFiles.map(f => f.id === id + ? { ...f, isUploading: false, uploaded: false, error: uploadError.statusText || 'Network error: Unable to connect to server' } + : f + ) + ) + } else { + try { + const errorData = JSON.parse(uploadError.response) as ParquetUploadError + const errorMessage = errorData.errors?.[0]?.detail || 'Upload failed' + setFiles(prevFiles => + prevFiles.map(f => f.id === id + ? { ...f, isUploading: false, uploaded: false, error: errorMessage } + : f + ) + ) + } catch (parseError) { + setFiles(prevFiles => + prevFiles.map(f => f.id === id + ? { ...f, isUploading: false, uploaded: false, error: uploadError.statusText || 'Failed to parse the error response from the server' } + : f + ) + ) + } + } + } finally { + setIsUploading(false) + } + }, [files, overwrite, quest, isUploading]) + + return ( + + {state === "upload" && ( + f.file_name)} + onFilesDropped={handleFilesDropped} + render={({ duplicates, addToQueue, uploadInputRef }) => ( + + )} + /> + )} + + {state === "list" && ( + f.file_name)} + onFilesDropped={handleFilesDropped} + render={({ duplicates, addToQueue, uploadInputRef }) => ( + + + Import queue + + + + + {status && ( + + {status.type === "success" && } + {status.type === "error" && } + {status.type === "warning" && } + + {status.message} + + + )} + + setOverwrite(checked)} + /> + + Overwrite existing files + + + + + + {isUploading ? "Uploading..." : "Import all files"} + + + + + + )} + /> + )} + + ) +} \ No newline at end of file diff --git a/packages/web-console/src/scenes/Import/ImportParquet/rename-file-dialog.tsx b/packages/web-console/src/scenes/Import/ImportParquet/rename-file-dialog.tsx new file mode 100644 index 000000000..9f1ad829d --- /dev/null +++ b/packages/web-console/src/scenes/Import/ImportParquet/rename-file-dialog.tsx @@ -0,0 +1,121 @@ +import React from "react" +import { ProcessedParquet } from "./types" +import { Dialog, ForwardRef, Button, Overlay } from "@questdb/react-components" +import { Edit } from "@styled-icons/remix-line" +import { Undo } from "@styled-icons/boxicons-regular" +import { Text } from "../../../components/Text" +import { Form } from "../../../components/Form" +import { Box } from "../../../components/Box" +import Joi from "joi" +import styled from "styled-components" +import { shortenText } from "../../../utils" + +const StyledDescription = styled(Dialog.Description)` + display: grid; + gap: 2rem; +` + +type Props = { + open: boolean + onOpenChange: (file?: ProcessedParquet) => void + onNameChange: (name: string) => void + file: ProcessedParquet +} + +const schema = Joi.object({ + name: Joi.string() + .required() + .messages({ + "string.empty": "Please enter a name", + }), +}) + +export const RenameFileDialog = ({ + open, + onOpenChange, + onNameChange, + file, +}: Props) => { + const name = file.file_name + return ( + + + + + + + + + + + + + onOpenChange(undefined)} + onInteractOutside={() => onOpenChange(undefined)} + > + + name="rename-file" + defaultValues={{ name }} + onSubmit={(values) => { + onNameChange(values.name) + onOpenChange(undefined) + }} + validationSchema={schema} + > + + + + Change import path + + + + + + + + + This path is a relative path to the sql.copy.input.root directory. +
+
+ Example: subdir/test.parquet + {' '}will import the data into {"{"}sql.copy.input.root{"}"}/subdir/test.parquet + + + + + + + + + + + } + variant="success" + data-hook="import-parquet-rename-file-submit" + > + Change + + + + + + + + + ) +} \ No newline at end of file diff --git a/packages/web-console/src/scenes/Import/ImportParquet/types.ts b/packages/web-console/src/scenes/Import/ImportParquet/types.ts new file mode 100644 index 000000000..6cd0bdb0c --- /dev/null +++ b/packages/web-console/src/scenes/Import/ImportParquet/types.ts @@ -0,0 +1,25 @@ +export type ProcessedParquet = { + id: string + fileObject: File + file_name: string + isUploading?: boolean + uploaded?: boolean + error?: string + cancelled?: boolean +} + +export type ParquetUploadError = { + errors: { + meta: { + name: string + } + detail: string + status: string + }[] +} + +export type UploadError = { + status: number + statusText: string + response?: string +} \ No newline at end of file diff --git a/packages/web-console/src/scenes/Import/index.tsx b/packages/web-console/src/scenes/Import/index.tsx new file mode 100644 index 000000000..d5ab9a92c --- /dev/null +++ b/packages/web-console/src/scenes/Import/index.tsx @@ -0,0 +1,42 @@ +import React from "react" +import { PaneContent, PaneWrapper } from "../../components" +import { ImportCSVFiles } from "./ImportCSVFiles" +import { ImportParquet } from "./ImportParquet" +import { eventBus } from "../../modules/EventBus" +import { EventType } from "../../modules/EventBus/types" +import { ImportType } from "../../store/Console/types" + +interface Props { + type: ImportType +} + +export const Import = ({ type }: Props) => { + const handleViewData = (query: string) => { + if (query) { + eventBus.publish(EventType.MSG_QUERY_SCHEMA) + eventBus.publish(EventType.MSG_QUERY_FIND_N_EXEC, { + query, + options: { appendAt: "end" }, + }) + } + } + + const handleUpload = () => { + eventBus.publish(EventType.MSG_QUERY_SCHEMA) + } + + return ( + + + {type === "csv" ? ( + + ) : ( + + )} + + + ) +} \ No newline at end of file diff --git a/packages/web-console/src/store/Console/actions.ts b/packages/web-console/src/store/Console/actions.ts index 64f882e1b..14e2124a7 100644 --- a/packages/web-console/src/store/Console/actions.ts +++ b/packages/web-console/src/store/Console/actions.ts @@ -27,6 +27,7 @@ import { ImageToZoom, Sidebar, BottomPanel, + ImportType, } from "./types" const setActiveSidebar = (panel: Sidebar): ConsoleAction => ({ @@ -39,6 +40,11 @@ const setActiveBottomPanel = (panel: BottomPanel): ConsoleAction => ({ type: ConsoleAT.SET_ACTIVE_BOTTOM_PANEL, }) +const setImportType = (type: ImportType): ConsoleAction => ({ + payload: type, + type: ConsoleAT.SET_IMPORT_TYPE, +}) + const setImageToZoom = (image?: ImageToZoom): ConsoleAction => ({ payload: image, type: ConsoleAT.SET_IMAGE_TO_ZOOM, @@ -52,5 +58,6 @@ export default { toggleSideMenu, setActiveSidebar, setActiveBottomPanel, + setImportType, setImageToZoom, } diff --git a/packages/web-console/src/store/Console/reducers.ts b/packages/web-console/src/store/Console/reducers.ts index fcb626e3c..e23251a75 100644 --- a/packages/web-console/src/store/Console/reducers.ts +++ b/packages/web-console/src/store/Console/reducers.ts @@ -28,6 +28,7 @@ export const initialState: ConsoleStateShape = { sideMenuOpened: false, activeSidebar: undefined, activeBottomPanel: "zeroState", + importType: "csv", imageToZoom: undefined, } @@ -57,6 +58,13 @@ const _console = ( } } + case ConsoleAT.SET_IMPORT_TYPE: { + return { + ...state, + importType: action.payload, + } + } + case ConsoleAT.SET_IMAGE_TO_ZOOM: { return { ...state, diff --git a/packages/web-console/src/store/Console/selectors.ts b/packages/web-console/src/store/Console/selectors.ts index 7e972c46d..256056a1c 100644 --- a/packages/web-console/src/store/Console/selectors.ts +++ b/packages/web-console/src/store/Console/selectors.ts @@ -21,7 +21,7 @@ * limitations under the License. * ******************************************************************************/ -import { StoreShape, Sidebar, BottomPanel, ImageToZoom } from "types" +import { StoreShape, Sidebar, BottomPanel, ImageToZoom, ImportType } from "types" const getSideMenuOpened: (store: StoreShape) => boolean = (store) => store.console.sideMenuOpened @@ -32,6 +32,9 @@ const getActiveSidebar: (store: StoreShape) => Sidebar = (store) => const getActiveBottomPanel: (store: StoreShape) => BottomPanel = (store) => store.console.activeBottomPanel +const getImportType: (store: StoreShape) => ImportType = (store) => + store.console.importType + const getImageToZoom: (store: StoreShape) => ImageToZoom | undefined = ( store, ) => store.console.imageToZoom @@ -40,5 +43,6 @@ export default { getSideMenuOpened, getActiveSidebar, getActiveBottomPanel, + getImportType, getImageToZoom, } diff --git a/packages/web-console/src/store/Console/types.ts b/packages/web-console/src/store/Console/types.ts index e9c52f75b..218a69766 100644 --- a/packages/web-console/src/store/Console/types.ts +++ b/packages/web-console/src/store/Console/types.ts @@ -26,6 +26,8 @@ export type Sidebar = "news" | "create" | undefined export type BottomPanel = "result" | "zeroState" | "import" +export type ImportType = "csv" | "parquet" + export type ImageToZoom = { src: string alt: string @@ -37,6 +39,7 @@ export type ConsoleStateShape = Readonly<{ sideMenuOpened: boolean activeSidebar: Sidebar activeBottomPanel: BottomPanel + importType: ImportType imageToZoom: ImageToZoom | undefined }> @@ -44,6 +47,7 @@ export enum ConsoleAT { TOGGLE_SIDE_MENU = "CONSOLE/TOGGLE_SIDE_MENU", SET_ACTIVE_SIDEBAR = "CONSOLE/SET_ACTIVE_SIDEBAR", SET_ACTIVE_BOTTOM_PANEL = "CONSOLE/SET_ACTIVE_BOTTOM_PANEL", + SET_IMPORT_TYPE = "CONSOLE/SET_IMPORT_TYPE", SET_IMAGE_TO_ZOOM = "CONSOLE/SET_IMAGE_TO_ZOOM", } @@ -61,6 +65,11 @@ type setActiveBottomPanelAction = Readonly<{ type: ConsoleAT.SET_ACTIVE_BOTTOM_PANEL }> +type setImportTypeAction = Readonly<{ + payload: ImportType + type: ConsoleAT.SET_IMPORT_TYPE +}> + type setImageToZoomAction = Readonly<{ payload?: ImageToZoom type: ConsoleAT.SET_IMAGE_TO_ZOOM @@ -70,4 +79,5 @@ export type ConsoleAction = | ToggleSideMenuAction | setActiveSidebarAction | setActiveBottomPanelAction + | setImportTypeAction | setImageToZoomAction diff --git a/packages/web-console/src/utils/questdb/client.ts b/packages/web-console/src/utils/questdb/client.ts index 8724c9ad7..5b9fb91ba 100644 --- a/packages/web-console/src/utils/questdb/client.ts +++ b/packages/web-console/src/utils/questdb/client.ts @@ -3,7 +3,7 @@ import { TelemetryConfigShape } from "../../store/Telemetry/types" import { eventBus } from "../../modules/EventBus" import { EventType } from "../../modules/EventBus/types" import { AuthPayload } from "../../modules/OAuth2/types" -import { API_VERSION } from "../../consts" +import { API_ROUTE_V1, API_VERSION } from "../../consts" import { Type, ErrorResult, @@ -19,10 +19,11 @@ import { FileCheckResponse, UploadModeSettings, UploadOptions, - UploadResult, + CSVUploadResult, Value, Preferences, Permission, + ParquetUploadResult, } from "./types" import { ssoAuthState } from "../../modules/OAuth2/ssoAuthState"; @@ -372,7 +373,7 @@ export class Client { partitionBy, timestamp, onProgress, - }: UploadOptions): Promise { + }: UploadOptions): Promise { const formData = new FormData() if (schema) { formData.append("schema", JSON.stringify(schema)) @@ -530,6 +531,63 @@ export class Client { return Promise.reject(error) } } + + async uploadParquetFiles( + files: { file: File; name: string }[], + overwrite: boolean, + onProgress?: (progress: number) => void + ): Promise { + const formData = new FormData() + files.forEach(({ file, name }) => { + formData.append(name, file) + }) + + return new Promise((resolve, reject) => { + const request = new XMLHttpRequest() + request.open("POST", `${API_ROUTE_V1}/imports?overwrite=${overwrite}`) + request.withCredentials = true + + Object.keys(this.commonHeaders).forEach((key) => { + request.setRequestHeader(key, this.commonHeaders[key]) + }) + + if (onProgress) { + request.upload.addEventListener("progress", (e) => { + const percent_completed = (e.loaded / e.total) * 100 + onProgress(percent_completed) + }) + } + + request.onload = () => { + if (request.status >= 200 && request.status < 300) { + try { + const response = JSON.parse(request.responseText) + resolve(response as ParquetUploadResult) + } catch (error) { + reject({ + status: request.status, + statusText: "Invalid JSON response", + }) + } + } else { + reject({ + status: request.status, + statusText: request.statusText, + response: request.responseText, + }) + } + } + + request.onerror = () => { + reject({ + status: request.status, + statusText: request.statusText || "Upload error", + }) + } + + request.send(formData) + }) + } } async function extractErrorMessage(response: Response) { diff --git a/packages/web-console/src/utils/questdb/types.ts b/packages/web-console/src/utils/questdb/types.ts index 6f8054a94..4f85656ac 100644 --- a/packages/web-console/src/utils/questdb/types.ts +++ b/packages/web-console/src/utils/questdb/types.ts @@ -298,7 +298,7 @@ export type UploadResultColumn = { errors: number } -export type UploadResult = { +export type CSVUploadResult = { columns: UploadResultColumn[] header: boolean location: string @@ -307,6 +307,12 @@ export type UploadResult = { status: string } +export type ParquetUploadResult = { + data: { type: string, id: string }[] +} + +export type FileUploadResult = CSVUploadResult | ParquetUploadResult + export type Parameter = { property_path: string env_var_name: string diff --git a/packages/web-console/webpack.config.js b/packages/web-console/webpack.config.js index 1923b8e9d..a4afa02be 100644 --- a/packages/web-console/webpack.config.js +++ b/packages/web-console/webpack.config.js @@ -86,7 +86,7 @@ module.exports = { context: [ config.contextPath + "/imp", config.contextPath + "/exp", config.contextPath + "/exec", config.contextPath + "/chk", config.contextPath + "/settings", config.contextPath + "/warnings", - config.contextPath + "/preferences" + config.contextPath + "/preferences", config.contextPath + "/api" ], target: config.backendUrl, },