Skip to content

Commit da6d8e5

Browse files
fantavlikfantavlik
committed
Add logic to pick a single search head when search head cluster is detected.
1 parent ff89188 commit da6d8e5

File tree

3 files changed

+91
-27
lines changed

3 files changed

+91
-27
lines changed

‎out/notebooks/controller.ts‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import {
44
cancelSearchJob,
55
createSearchJob,
66
getClient,
7+
getSearchHeadClusterMemberClient,
78
getSearchJob,
89
getSearchJobResults,
910
wait,
@@ -18,6 +19,7 @@ export class SplunkController {
1819
protected supportedLanguages: string[];
1920

2021
protected _controller: vscode.NotebookController;
22+
protected _service; // Splunk Javascript SDK Client
2123
private _executionOrder = 0;
2224
private _interrupted = false;
2325
private _tokens = {};
@@ -55,6 +57,8 @@ export class SplunkController {
5557
this.notebookType,
5658
this.label
5759
);
60+
// Attempt to connect to individual search head if part of a search head cluster
61+
this.refreshService();
5862

5963
this._controller.supportedLanguages = this.supportedLanguages;
6064
this._controller.supportsExecutionOrder = true;
@@ -70,6 +74,20 @@ export class SplunkController {
7074
this._execute([cell], notebookDocument, this._controller);
7175
}
7276

77+
async refreshService() {
78+
const config = vscode.workspace.getConfiguration();
79+
const restUrl = config.get<string>('splunk.commands.splunkRestUrl');
80+
const token = config.get<string>('splunk.commands.token');
81+
// Create a new SDK client if one hasn't been created or token / url settings have been changed
82+
if (!this._service || (this._service._originalURL !== restUrl) || (this._service.sessionKey !== token)) {
83+
this._service = getClient();
84+
// Check to see if the splunk deployment is part of a search head cluster, choose a single search head
85+
// to target if so to ensure that adhoc jobs are immediately available (without replication)
86+
const newService = await getSearchHeadClusterMemberClient(this._service);
87+
this._service = newService;
88+
}
89+
}
90+
7391
protected _execute(
7492
cells: vscode.NotebookCell[],
7593
_notebook: vscode.NotebookDocument,
@@ -145,9 +163,8 @@ export class SplunkController {
145163

146164
let query = cell.document.getText().trim().replace(/^\s+|\s+$/g, '');
147165

148-
const service = getClient()
149-
150-
let jobs = service.jobs();
166+
await this.refreshService();
167+
let jobs = this._service.jobs();
151168

152169
const tokenRegex = /\$([a-zA-Z0-9_.|]*?)\$/g;
153170

‎out/notebooks/spl2/controller.ts‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
import * as vscode from 'vscode';
22

3-
import {
4-
dispatchSpl2Module,
5-
getClient,
6-
} from '../splunk';
3+
import { dispatchSpl2Module } from '../splunk';
74
import { SplunkController } from '../controller';
85
import { splunkMessagesToOutputItems } from '../utils/messages';
96
import { getAppSubNamespace } from './serializer';
@@ -30,7 +27,6 @@ export class Spl2Controller extends SplunkController {
3027
const execution = super._startExecution(cell);
3128

3229
const spl2Module = cell.document.getText().trim();
33-
const service = getClient();
3430
let fullNamespace: string = cell?.metadata?.splunk?.namespace || '';
3531
// Get apps.<app>[.optional.sub.namespaces] from fullNamespace
3632
const [app, subNamespace] = getAppSubNamespace(fullNamespace);
@@ -39,8 +35,9 @@ export class Spl2Controller extends SplunkController {
3935

4036
let job;
4137
try {
38+
await this.refreshService();
4239
job = await dispatchSpl2Module(
43-
service,
40+
this._service,
4441
spl2Module,
4542
app,
4643
subNamespace,

‎out/notebooks/splunk.ts‎

Lines changed: 68 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,38 +9,33 @@ export function getClient() {
99
const restUrl = config.get<string>('splunk.commands.splunkRestUrl');
1010
const token = config.get<string>('splunk.commands.token');
1111

12-
let url = new URL(restUrl);
12+
const url = new URL(restUrl);
1313
const scheme = url.protocol.replace(':', '');
1414
const port = url.port || (scheme === 'https' ? '443' : '80');
1515
const host = url.hostname;
1616

17-
let service = new splunk.Service({
17+
const service = new splunk.Service({
1818
scheme: scheme,
1919
host: host,
2020
port: port,
2121
sessionKey: token,
22-
version: '8',
2322
authorization: 'Bearer',
2423
});
24+
service._originalURL = restUrl;
2525

2626
return service;
2727
}
2828

2929
export function splunkLogin(service) {
30-
3130
return new Promise(function(resolve, reject) {
32-
3331
service.login(function(err, wasSuccessful) {
3432
if (err !== null || !wasSuccessful) {
3533
reject(err);
3634
} else {
3735
resolve(null);
3836
}
3937
});
40-
4138
});
42-
43-
4439
}
4540

4641

@@ -53,7 +48,6 @@ export function createSearchJob(jobs, query, options) {
5348
resolve(data);
5449
}
5550
});
56-
5751
});
5852
}
5953

@@ -69,6 +63,61 @@ function makeHeaders(service: any): object {
6963
};
7064
}
7165

66+
/**
67+
* Check to see if the SDK client is part of a search head cluster, if so return a new
68+
* client pointing to an individual search head member, such that any search ids created
69+
* will be immediately available for results rather than waiting for artifact replication
70+
* across the search head cluster.
71+
* @param service Instance of the Javascript SDK Service
72+
*
73+
* @returns Promise<void>
74+
*/
75+
export function getSearchHeadClusterMemberClient(service: any): Promise<any> {
76+
const shcUrl = `${service.prefix}/services/shcluster/member/members?output_mode=json`;
77+
console.log(`Attempting to determine SHC info if present using ${shcUrl}`);
78+
return needle(
79+
"GET",
80+
shcUrl,
81+
{
82+
'headers': makeHeaders(service),
83+
'followAllRedirects': true,
84+
'timeout': 0,
85+
'strictSSL': false,
86+
'rejectUnauthorized' : false,
87+
})
88+
.then((response) => {
89+
console.log(`Response from shcUrl status code: ${response.statusCode}`);
90+
console.log(`Response from shcUrl body: \n'${JSON.stringify(response.body)}'`);
91+
const data = response.body;
92+
if (response.statusCode >= 400 ||
93+
!Object.prototype.isPrototypeOf(data)
94+
|| data.entry === undefined
95+
|| !Array.isArray(data.entry)
96+
|| data.entry.length === 0
97+
|| data.entry[0].content === undefined
98+
|| data.entry[0].content.mgmt_uri === undefined
99+
) {
100+
console.warn("Unsuccessful response from /services/shcluster/member/members endpoint encountered, reverting to original service client.")
101+
return service;
102+
}
103+
// This is in the expected successful response format
104+
vscode.window.showInformationMessage(`Discovered search head cluster members. Attempting to communicate directly with SH ${data.entry[0].content.mgmt_uri}`);
105+
const url = new URL(data.entry[0].content.mgmt_uri);
106+
const scheme = url.protocol.replace(':', '');
107+
const port = url.port || (scheme === 'https' ? '443' : '80');
108+
const host = url.hostname;
109+
const newService = new splunk.Service({
110+
scheme: scheme,
111+
host: host,
112+
port: port,
113+
sessionKey: service.sessionKey,
114+
authorization: 'Bearer',
115+
});
116+
newService._originalURL = service._originalURL;
117+
return newService;
118+
});
119+
}
120+
72121
/**
73122
* Update a module by calling the PUT /services/spl2/modules/<namespace>.<moduleName>
74123
* @param service Instance of the Javascript SDK Service
@@ -77,7 +126,7 @@ function makeHeaders(service: any): object {
77126
* @param module Full contents of the module to update with
78127
* @returns Promise<void> (or throw Error containing data.messages[])
79128
*/
80-
export function updateSpl2Module(service: any, moduleName: string, namespace: string, module: string) {
129+
export function updateSpl2Module(service: any, moduleName: string, namespace: string, module: string): Promise<void> {
81130
// The Splunk SDK for Javascript doesn't currently support the spl2/modules endpoints
82131
// nor does it support sending requests in JSON format (only receiving responses), so
83132
// for now use the underlying needle library that the SDK uses for requests/responses
@@ -131,9 +180,9 @@ export function updateSpl2Module(service: any, moduleName: string, namespace: st
131180
* @param namespace Namespace _within_ the apps.<app> to run, this will be used directly in the body of the request
132181
* @param earliest Earliest time to be included in the body of the request
133182
* @param latest Latest time to be included in the body of the request
134-
* @returns A Promise containing the job id created (or throw an Error containing data.messages[])
183+
* @returns A Promise containing the job information including sid created (or throw an Error containing data.messages[])
135184
*/
136-
export function dispatchSpl2Module(service: any, spl2Module: string, app: string, namespace: string, earliest: string, latest: string) {
185+
export function dispatchSpl2Module(service: any, spl2Module: string, app: string, namespace: string, earliest: string, latest: string): Promise<any> {
137186
// For now we're using /services/<app> which doesn't respect relative namespaces,
138187
// so for now hardcode this to '' but if/when /servicesNS/<app>
139188
namespace = '';
@@ -194,6 +243,7 @@ export function dispatchSpl2Module(service: any, spl2Module: string, app: string
194243
.then((response) => {
195244
console.log(`Response status code: ${response.statusCode}`);
196245
console.log(`Response body: \n'${JSON.stringify(response.body)}'`);
246+
console.log(`Response headers: \n'${JSON.stringify(response.headers)}'`);
197247
const data = response.body;
198248
if (response.statusCode >= 400 || !Array.prototype.isPrototypeOf(data) || data.length < 1) {
199249
handleErrorPayloads(data, response.statusCode);
@@ -267,7 +317,7 @@ function handleErrorPayloads(data: any, statusCode: number) {
267317
});
268318
}
269319

270-
export function getSearchJobBySid(service, sid) {
320+
export function getSearchJobBySid(service, sid): Promise<any> {
271321
return new Promise(function(resolve, reject) {
272322
service.getJob(sid, function(err, data) {
273323
if (err != null) {
@@ -280,7 +330,7 @@ export function getSearchJobBySid(service, sid) {
280330
}
281331

282332

283-
export function getSearchJob(job) {
333+
export function getSearchJob(job): Promise<any> {
284334
return new Promise(function(resolve, reject) {
285335
job.fetch(function(err, job) {
286336
if (err !== null) {
@@ -293,7 +343,7 @@ export function getSearchJob(job) {
293343
});
294344
}
295345

296-
export function getJobSearchLog(job) {
346+
export function getJobSearchLog(job): Promise<any> {
297347
return new Promise(function(resolve, reject) {
298348
job.searchlog(function(err, log) {
299349
if (err !== null) {
@@ -306,7 +356,7 @@ export function getJobSearchLog(job) {
306356
});
307357
}
308358

309-
export function getSearchJobResults(job) {
359+
export function getSearchJobResults(job): Promise<any> {
310360
return new Promise(function(resolve, reject) {
311361
job.get("results", {"output_mode": "json_cols"},function(err, results) {
312362
if (err !== null) {
@@ -319,7 +369,7 @@ export function getSearchJobResults(job) {
319369
});
320370
}
321371

322-
export function cancelSearchJob(job) {
372+
export function cancelSearchJob(job): Promise<any> {
323373
return new Promise(function(resolve, reject) {
324374
job.cancel(function(err, results) {
325375
if (err !== null) {
@@ -332,7 +382,7 @@ export function cancelSearchJob(job) {
332382
});
333383
}
334384

335-
export function wait(ms = 1000) {
385+
export function wait(ms = 1000): Promise<void> {
336386
return new Promise(resolve => {
337387
setTimeout(resolve, ms);
338388
});

0 commit comments

Comments
 (0)