6.x.x

.github/workflows/build.yml

@@ -37,9 +37,9 @@ jobs:
         env:
           AUTO_TEST: true
           LOGIN_TEST: true
-          TEST_DATABASE: "e2e"
-          ENGINE_CONNECTION_STRING: "https://sdkse2etest.eastus.kusto.windows.net"
-          DM_CONNECTION_STRING: "https://ingest-sdkse2etest.eastus.kusto.windows.net"
+          TEST_DATABASE: ${{ secrets.TEST_DATABASE }}
+          ENGINE_CONNECTION_STRING: ${{ secrets.ENGINE_CONNECTION_STRING }}
+          DM_CONNECTION_STRING: ${{ secrets.DM_CONNECTION_STRING }}
           AZURE_CLIENT_ID: ${{ secrets.APP_ID }}
           AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
       - name: Upload coverage to Codecov

.gitignore

@@ -11,6 +11,7 @@
 */__*.js
 # User-specific files (MonoDevelop/Xamarin Studio)
 *.userprefs
+.nx
 
 # Build results
 [Dd]ebug/

CHANGELOG.md

@@ -5,6 +5,26 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [6.0.4] - 2025-01-23
+
+### Fixed
+
+-   Fixed handling of tags by @noamcohen97
+-   Updated dependencies
+
+## [6.0.3] - 2025-01-23
+
+### Fixed
+
+-   Fixed error when doing streaming ingestion with a blob url
+-   Updated azure storage deps
+-   Server timeout was with accuracy of tenth of a second.
+-   Updated default endpoint urls
+
+### Security
+
+-   Updated dependencies to fix security vulnerabilities
+
 ## [6.0.2] - 2024-04-11
 
 ### Changed

lerna.json

@@ -1,4 +1,4 @@
 {
-  "version": "6.0.2",
+  "version": "6.0.4",
   "useNx": true
 }

packages/azure-kusto-data/example.js

@@ -6,11 +6,11 @@ const KustoConnectionStringBuilder = require("azure-kusto-data").KustoConnection
 const ClientRequestProperties = require("azure-kusto-data").ClientRequestProperties;
 const { v4: uuidv4 } = require("uuid");
 
-const clusterConectionString = "https://<cluster>.<region>.kusto.windows.net";
+const clusterConnectionString = "https://<cluster>.<region>.kusto.windows.net";
 const database = "<databaseName>";
 const table = "<tableName>";
 
-const kcs = KustoConnectionStringBuilder.withAadDeviceAuthentication(clusterConectionString);
+const kcs = KustoConnectionStringBuilder.withAadDeviceAuthentication(clusterConnectionString);
 const kustoClient = new KustoClient(kcs); // After using the client, you should use `close()` to free up resources
 start();
 

packages/azure-kusto-data/package.json

@@ -1,6 +1,6 @@
 {
     "name": "azure-kusto-data",
-    "version": "6.0.2",
+    "version": "6.0.4",
     "description": "Azure Data Explorer Query SDK",
     "module": "dist-esm/src/index.js",
     "types": "./types/src/index.d.ts",
@@ -58,7 +58,7 @@
     "dependencies": {
         "@azure/identity": "^4.0.1",
         "@types/uuid": "^8.3.4",
-        "axios": "^1.6.0",
+        "axios": "^1.8.4",
         "follow-redirects": "^1.15.1",
         "https-browserify": "^1.0.0",
         "stream-http": "^3.2.0",

packages/azure-kusto-data/src/clientDetails.ts

@@ -118,4 +118,5 @@ export interface KustoHeaders {
     "x-ms-app": string | null;
     "x-ms-user": string | null;
     "x-ms-client-request-id": string | null;
+    "x-ms-version": string | null;
 }

packages/azure-kusto-data/src/clientRequestProperties.ts

@@ -3,6 +3,8 @@
 
 import { KustoHeaders } from "./clientDetails";
 
+const SDK_API_VERSION = "2024-12-12"; // TODO: Update when new API version is released
+
 export class ClientRequestProperties {
     private _options: { [option: string]: any };
     private _parameters: { [option: string]: any };
@@ -121,6 +123,7 @@ export class ClientRequestProperties {
         if (this.application) {
             headers["x-ms-app"] = this.application;
         }
+        headers["x-ms-version"] = SDK_API_VERSION;
         return headers;
     }
 }

packages/azure-kusto-data/src/cloudSettings.ts

@@ -12,6 +12,8 @@ export type CloudInfo = {
     FirstPartyAuthorityUrl: string;
 };
 
+const AXIOS_ERR_NETWORK = axios?.AxiosError?.ERR_NETWORK ?? "ERR_NETWORK";
+
 /**
  * This class holds data for all cloud instances, and returns the specific data instance by parsing the dns suffix from a URL
  */
@@ -42,7 +44,7 @@ class CloudSettings {
         }
 
         try {
-            const response = await axios.get<{ AzureAD: CloudInfo | undefined }>(kustoUri + this.METADATA_ENDPOINT, {
+            const response = await axios.get<{ AzureAD: CloudInfo | undefined }>(this.getAuthMetadataEndpointFromClusterUri(kustoUri), {
                 headers: {
                     "Cache-Control": "no-cache",
                     // Disable caching - it's being cached in memory (Service returns max-age).
@@ -64,7 +66,7 @@ class CloudSettings {
         } catch (ex) {
             if (axios.isAxiosError(ex)) {
                 // Axios library has a bug in browser, not propagating the status code, see: https://github.com/axios/axios/issues/5330
-                if ((ex.response?.status === 404 && isNode) || (ex.code === axios.AxiosError.ERR_NETWORK && !isNode)) {
+                if ((isNode && ex.response?.status === 404) || (!isNode && (!ex.code || ex.code === AXIOS_ERR_NETWORK))) {
                     // For now as long not all proxies implement the metadata endpoint, if no endpoint exists return public cloud data
                     this.cloudCache[kustoUri] = this.defaultCloudInfo;
                 } else {
@@ -84,6 +86,12 @@ class CloudSettings {
         return urlString;
     }
 
+    getAuthMetadataEndpointFromClusterUri(kustoUri: string): string {
+        const url = new URL(kustoUri);
+        // Returns endpoint URL in the form of https://<cluster>:port/v1/rest/auth/metadata
+        return `${url.protocol}//${url.host}${this.METADATA_ENDPOINT}`;
+    }
+
     static getAuthorityUri(cloudInfo: CloudInfo, authorityId?: string): string {
         return cloudInfo.LoginEndpoint + "/" + (authorityId || "organizations");
     }

packages/azure-kusto-data/test/cloudSettingsTest.ts

@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import assert from "assert";
+
+import { CloudSettings } from "../src/cloudSettings";
+
+describe("CloudSettings.getAuthMetadataEndpointFromClusterUri", () => {
+    it("valid input", () => {
+        assert.strictEqual(
+            CloudSettings.getAuthMetadataEndpointFromClusterUri("https://statusreturner.azurewebsites.net/"),
+            "https://statusreturner.azurewebsites.net/v1/rest/auth/metadata"
+        );
+
+        // With path
+        assert.strictEqual(
+            CloudSettings.getAuthMetadataEndpointFromClusterUri("https://statusreturner.azurewebsites.net/test/test2/test"),
+            "https://statusreturner.azurewebsites.net/v1/rest/auth/metadata"
+        );
+
+        // With non-default port
+        assert.strictEqual(
+            CloudSettings.getAuthMetadataEndpointFromClusterUri("https://statusreturner.azurewebsites.net:5050/"),
+            "https://statusreturner.azurewebsites.net:5050/v1/rest/auth/metadata"
+        );
+
+        // With leading slash
+        assert.strictEqual(
+            CloudSettings.getAuthMetadataEndpointFromClusterUri("https://statusreturner.azurewebsites.net//////"),
+            "https://statusreturner.azurewebsites.net/v1/rest/auth/metadata"
+        );
+    });
+});

packages/azure-kusto-ingest/package.json

@@ -1,6 +1,6 @@
 {
     "name": "azure-kusto-ingest",
-    "version": "6.0.2",
+    "version": "6.0.4",
     "description": "Azure Data Explorer Ingestion SDK",
     "module": "dist-esm/src/index.js",
     "types": "./types/src/index.d.ts",
@@ -64,7 +64,7 @@
         "@types/tmp": "^0.2.3",
         "@types/uuid": "^8.3.4",
         "@types/uuid-validate": "0.0.1",
-        "azure-kusto-data": "^6.0.2",
+        "azure-kusto-data": "^6.0.4",
         "browserify-zlib": "0.2.0",
         "buffer": "^6.0.3",
         "is-ip": "^3.1.0",

packages/azure-kusto-ingest/src/descriptors.ts

@@ -31,6 +31,15 @@ export abstract class AbstractDescriptor {
     constructor(public sourceId: string | null = null, public size: number | null = null) {
         this.sourceId = getSourceId(sourceId);
     }
+
+    _calculateSize(fileSize: number, modifier: number = 1): void {
+        if (this.size == null || this.size <= 0) {
+            if (fileSize <= 0) {
+                throw Error("Empty file.");
+            }
+            this.size = fileSize * modifier;
+        }
+    }
 }
 
 export class StreamDescriptor extends AbstractDescriptor {

packages/azure-kusto-ingest/src/fileDescriptor.browser.ts

@@ -22,22 +22,27 @@ export class FileDescriptor extends AbstractDescriptor implements FileDescriptor
     ) {
         super(sourceId);
         this.compressionType = compressionType;
-        this.size = size || file.size;
+        this.size = size;
+
         this.zipped = compressionType !== CompressionType.None || this.extension === ".gz" || this.extension === ".zip";
         this.shouldNotCompress = !shouldCompressFileByExtension(this.extension);
     }
 
     async prepare(ingestionProperties?: IngestionPropertiesInput): Promise<Blob> {
         const shouldNotCompressByFormat = !shouldCompressFileByFormat(ingestionProperties);
-        if (!this.zipped && !this.shouldNotCompress && !shouldNotCompressByFormat) {
-            try {
-                const gzipped = pako.gzip(await this.file.arrayBuffer());
-                return new Blob([gzipped]);
-            } catch (e) {
-                // Ignore - return the file itself
-            }
+        if (this.zipped || this.shouldNotCompress || shouldNotCompressByFormat) {
+            const estimatedCompressionModifier = 11;
+            this._calculateSize(this.file.size, estimatedCompressionModifier);
+            return this.file;
         }
 
+        const gzipped = pako.gzip(await this.file.arrayBuffer());
+        try {
+            return new Blob([gzipped]);
+        } catch (e) {
+            // Ignore - return the file itself
+        }
+        this._calculateSize(this.file.size);
         return this.file;
     }
 

packages/azure-kusto-ingest/src/fileDescriptor.ts

@@ -17,6 +17,7 @@ export class FileDescriptor extends AbstractDescriptor implements FileDescriptor
     compressionType: CompressionType;
     shouldNotCompress: boolean;
     cleanupTmp?: () => Promise<void>;
+    private getSize: (file: string | Buffer | URL) => Promise<fs.Stats>;
 
     constructor(
         /**
@@ -36,6 +37,8 @@ export class FileDescriptor extends AbstractDescriptor implements FileDescriptor
 
         this.zipped = compressionType !== CompressionType.None || this.extension === ".gz" || this.extension === ".zip";
         this.shouldNotCompress = !shouldCompressFileByExtension(this.extension);
+
+        this.getSize = promisify(fs.stat);
     }
 
     async _gzip(): Promise<string> {
@@ -65,22 +68,15 @@ export class FileDescriptor extends AbstractDescriptor implements FileDescriptor
         const shouldNotCompressByFormat = !shouldCompressFileByFormat(ingestionProperties);
         if (this.zipped || this.shouldNotCompress || shouldNotCompressByFormat) {
             const estimatedCompressionModifier = 11;
-            await this._calculateSize(estimatedCompressionModifier);
+            this._calculateSize((await this.getSize(this.file as string)).size, estimatedCompressionModifier);
             return this.file as string;
         }
 
         const path = await this._gzip();
-        await this._calculateSize();
+        this._calculateSize((await this.getSize(this.file as string)).size);
         return path;
     }
 
-    private async _calculateSize(modifier: number = 1): Promise<void> {
-        if (this.size == null || this.size <= 0) {
-            const asyncStat = promisify(fs.stat);
-            this.size = (await asyncStat(this.file as string)).size * modifier;
-        }
-    }
-
     async cleanup(): Promise<void> {
         if (this.cleanupTmp) {
             await this.cleanupTmp();

packages/azure-kusto-ingest/src/ingestClientBase.ts

@@ -6,7 +6,7 @@ import { Client as KustoClient, KustoConnectionStringBuilder } from "azure-kusto
 import ResourceManager from "./resourceManager";
 
 import IngestionBlobInfo from "./ingestionBlobInfo";
-import { ContainerClient } from "@azure/storage-blob";
+import { BlobUploadCommonResponse, ContainerClient } from "@azure/storage-blob";
 
 import { QueueClient } from "@azure/storage-queue";
 
@@ -136,22 +136,32 @@ export abstract class KustoIngestClientBase extends AbstractKustoClient {
         for (let i = 0; i < retryCount; i++) {
             const containerClient = new ContainerClient(containers[i].uri);
             const blockBlobClient = containerClient.getBlockBlobClient(blobName);
+            let uploadResponse: BlobUploadCommonResponse;
             try {
                 if (typeof descriptor == "string") {
-                    await blockBlobClient.uploadFile(descriptor);
+                    uploadResponse = await blockBlobClient.uploadFile(descriptor);
                 } else if (descriptor instanceof StreamDescriptor) {
                     if (descriptor.stream instanceof Buffer) {
-                        await blockBlobClient.uploadData(descriptor.stream as Buffer);
+                        uploadResponse = await blockBlobClient.uploadData(descriptor.stream as Buffer);
                     } else {
-                        await blockBlobClient.uploadStream(descriptor.stream as Readable);
+                        uploadResponse = await blockBlobClient.uploadStream(descriptor.stream as Readable);
                     }
                 } else {
-                    await blockBlobClient.uploadData(descriptor);
+                    uploadResponse = await blockBlobClient.uploadData(descriptor);
                 }
+
+                if (uploadResponse?.errorCode != null) {
+                    throw new Error(`Failed to upload to blob: ${uploadResponse.errorCode}`);
+                }
+
                 this.resourceManager.reportResourceUsageResult(containerClient.accountName, true);
                 return blockBlobClient.url;
             } catch (ex) {
                 this.resourceManager.reportResourceUsageResult(containerClient.accountName, false);
+
+                if (i === retryCount - 1) {
+                    throw ex;
+                }
             }
         }
 

packages/azure-kusto-ingest/src/ingestionBlobInfo.ts

@@ -49,17 +49,17 @@ export class IngestionBlobInfo {
 
         const tags: string[] = [];
         if (ingestionProperties.additionalTags) {
-            tags.concat(ingestionProperties.additionalTags);
+            tags.push(...ingestionProperties.additionalTags);
         }
         if (ingestionProperties.dropByTags) {
-            tags.concat(ingestionProperties.dropByTags.map((t) => "drop-by:" + t));
+            tags.push(...ingestionProperties.dropByTags.map((t) => "drop-by:" + t));
         }
         if (ingestionProperties.ingestByTags) {
-            tags.concat(ingestionProperties.ingestByTags.map((t) => "ingest-by:" + t));
+            tags.push(...ingestionProperties.ingestByTags.map((t) => "ingest-by:" + t));
         }
 
-        if (tags && tags.length > 0) {
-            additionalProperties.tags = tags;
+        if (tags.length > 0) {
+            additionalProperties.tags = JSON.stringify(tags);
         }
 
         if (ingestionProperties.ingestIfNotExists) {

packages/azure-kusto-ingest/src/ingestionProperties.ts

@@ -182,7 +182,7 @@ export interface IngestionPropertiesFields {
      */
     ingestionMappingType?: IngestionMappingKind;
     ingestionMappingKind?: IngestionMappingKind;
-    additionalTags?: string;
+    additionalTags?: string[];
     ingestIfNotExists?: string;
     ingestByTags?: string[];
     dropByTags?: string[];

packages/azure-kusto-ingest/src/streamingIngestClient.browser.ts

@@ -28,6 +28,7 @@ class KustoStreamingIngestClient extends KustoStreamingIngestClientBase {
             descriptor.stream,
             props.format,
             props.ingestionMappingReference ?? null,
+            undefined,
             clientRequestId
         );
     }

packages/azure-kusto-ingest/src/streamingIngestClient.ts

@@ -41,6 +41,7 @@ class KustoStreamingIngestClient extends KustoStreamingIngestClientBase {
             compressedStream,
             props.format,
             props.ingestionMappingReference ?? null,
+            undefined,
             clientRequestId
         );
     }