Skip to content

Commit 8cb8322

Browse files
frikkyfrikky
committed
Optimised it again to work better and better for custom_action to get down to <1 second no matter what
1 parent a1c9e4c commit 8cb8322

1 file changed

Lines changed: 80 additions & 29 deletions

File tree

‎pkg/api.go‎

Lines changed: 80 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import (
2020
"os/exec"
2121
"time"
2222
"strings"
23-
"reflect"
2423
"path/filepath"
2524

2625
uuid "github.com/satori/go.uuid"
@@ -418,16 +417,18 @@ func autoUploadSingulOutput(ctx context.Context, orgId string, curApikey string,
418417
foundArray = append(foundArray, innerMap)
419418
}
420419

421-
if debug {
422-
log.Printf("\n\n\nAUTO UPLOAD FUNCTION: EXIT ON PURPOSE TO LOOK FOR GMAIL -> '%s'. Data type: %#v\n\nTYPE: %#v\n\n\n", value.Label, parsedTranslation.Output, reflect.TypeOf(parsedTranslation.Output))
423-
}
420+
//if debug {
421+
// log.Printf("\n\n\nAUTO UPLOAD FUNCTION: EXIT ON PURPOSE TO LOOK FOR GMAIL -> '%s'. Data type: %#v\n\nTYPE: %#v\n\n\n", value.Label, parsedTranslation.Output, reflect.TypeOf(parsedTranslation.Output))
422+
//}
424423

425424
isListRequest := false
426425

427426
// .Output = translated
428427
// .RawResponse = original (raw)
429428
//if foundArray, ok := parsedTranslation.Output.([]interface{}); ok {
430-
log.Printf("[DEBUG] Found list/search output for label '%s'. Array Length: %d", value.Label, len(foundArray))
429+
if debug {
430+
log.Printf("[DEBUG] Found get/list/search output for label '%s' during upload. Array Length: %d", value.Label, len(foundArray))
431+
}
431432

432433
actualLabel := strings.ToLower(strings.Join(foundLabelSplit[1:], "_"))
433434
if len(actualLabel) == 0 {
@@ -447,16 +448,29 @@ func autoUploadSingulOutput(ctx context.Context, orgId string, curApikey string,
447448
}
448449
}
449450

451+
// Avoids isListRequest override
452+
if actualLabel == "ticket" || actualLabel == "user" || actualLabel == "asset" || actualLabel == "contact" || actualLabel == "alert" || actualLabel == "case" || actualLabel == "event" || actualLabel == "domain" || actualLabel == "ip" || actualLabel == "url" || actualLabel == "file" {
453+
actualLabel = fmt.Sprintf("%ss", actualLabel)
454+
}
455+
456+
if strings.HasPrefix(actualLabel, "get_") {
457+
actualLabel = strings.TrimPrefix(actualLabel, "get_")
458+
} else if strings.HasPrefix(actualLabel, "list_") {
459+
actualLabel = strings.TrimPrefix(actualLabel, "list_")
460+
} else if strings.HasPrefix(actualLabel, "search_") {
461+
actualLabel = strings.TrimPrefix(actualLabel, "search_")
462+
}
463+
450464
if actualLabel == "tickets" || actualLabel == "alerts" || actualLabel == "cases" || actualLabel == "messages" || actualLabel == "chats" || actualLabel == "incidents" {
451-
actualLabel = "shuffle-security incidents"
465+
actualLabel = "shuffle-security_incidents"
452466
}
453467

454468
if actualLabel == "assets" || actualLabel == "endpoints" {
455-
actualLabel = "shuffle-security assets"
469+
actualLabel = "shuffle-security_assets"
456470
}
457471

458472
if actualLabel == "users" {
459-
actualLabel = "shuffle-security users"
473+
actualLabel = "shuffle-security_users"
460474
}
461475

462476
allEntries := []shuffle.CacheKeyData{}
@@ -589,21 +603,23 @@ func autoUploadSingulOutput(ctx context.Context, orgId string, curApikey string,
589603

590604
shuffle.SetCache(ctx, cacheKey, []byte("true"), 60*60*24*3) // 3 days as we don't want to keep running the same one over and over.
591605

592-
//marshalledBody, err := json.Marshal(item)
593606
marshalledBody, err := json.Marshal(generatedItem)
594607
if err != nil {
595608
log.Printf("[ERROR] Failed marshalling item in schemaless output for label %s: %s", value.Label, err)
596609
continue
597610
}
598611

612+
//if debug {
613+
// log.Printf("[DEBUG] Uploading '%s' in category '%s' with data %s", foundIdentifier, actualLabel, string(marshalledBody))
614+
//}
615+
599616
datastoreEntry := shuffle.CacheKeyData{
600617
Key: foundIdentifier,
601618
Value: string(marshalledBody),
602619
Category: actualLabel,
603620
}
604621

605622
allEntries = append(allEntries, datastoreEntry)
606-
607623
if cnt > 100 {
608624
break
609625
}
@@ -1966,12 +1982,6 @@ func RunActionWrapper(ctx context.Context, user shuffle.User, value shuffle.Cate
19661982
missingFields = append(missingFields, field.Key)
19671983
}
19681984
}
1969-
1970-
/*
1971-
if debug {
1972-
log.Printf("[DEBUG] Not all required fields were handled (1). Missing: %#v. Should force use of all fields? Handled fields: %3v", missingFields, handledRequiredFields)
1973-
}
1974-
*/
19751985
}
19761986

19771987
// Send request to /api/v1/conversation with this data
@@ -2151,6 +2161,38 @@ func RunActionWrapper(ctx context.Context, user shuffle.User, value shuffle.Cate
21512161
}
21522162
}
21532163

2164+
// Arbitrary lookup of field value in other fields as it may have been
2165+
// replaced already
2166+
newMissingFields := []string{}
2167+
for _, missingField := range missingFields {
2168+
relevantValue := ""
2169+
for _, field := range value.Fields {
2170+
if field.Key == missingField {
2171+
relevantValue = field.Value
2172+
break
2173+
}
2174+
}
2175+
2176+
// Handle it properly
2177+
if len(relevantValue) >= 4 {
2178+
found := false
2179+
for _, param := range secondAction.Parameters {
2180+
if strings.Contains(param.Value, relevantValue) {
2181+
found = true
2182+
break
2183+
}
2184+
}
2185+
2186+
if found {
2187+
continue
2188+
}
2189+
}
2190+
2191+
newMissingFields = append(newMissingFields, missingField)
2192+
}
2193+
2194+
missingFields = newMissingFields
2195+
21542196
// AI fallback mechanism to handle missing fields
21552197
// This is in case some fields are not sent in properly
21562198
orgId := ""
@@ -2227,7 +2269,11 @@ func RunActionWrapper(ctx context.Context, user shuffle.User, value shuffle.Cate
22272269
//func RunSelfCorrectingRequest(originalFields []Valuereplace, action Action, status int, additionalInfo, fullUrl, outputBody, appname, inputdata string, attempt ...int) (Action, string, error) {
22282270

22292271
if err != nil {
2230-
log.Printf("[ERROR] Failed running self-correcting request for custom action: %s", err)
2272+
log.Printf("[ERROR] Failed running self-correcting request for custom_action: %s", err)
2273+
2274+
// :thinking: => This was added March 2026 to refix custom action mapping
2275+
newOutputAction.Name = "custom_action"
2276+
secondAction = newOutputAction
22312277
} else {
22322278
newOutputAction.Name = "custom_action"
22332279
secondAction = newOutputAction
@@ -2288,6 +2334,8 @@ func RunActionWrapper(ctx context.Context, user shuffle.User, value shuffle.Cate
22882334
}
22892335
}
22902336

2337+
// This may screw us over e.g. for xml, but autocorrect should solve it
2338+
// over time
22912339
for paramIndex, param := range secondAction.Parameters {
22922340
if param.Name == "headers" && len(param.Value) == 0 {
22932341
secondAction.Parameters[paramIndex].Value = "Content-Type: application/json"
@@ -2860,6 +2908,7 @@ func RunActionWrapper(ctx context.Context, user shuffle.User, value shuffle.Cate
28602908
outputmap := make(map[string]interface{})
28612909
schemalessOutput, err := schemaless.Translate(ctx, value.Label, marshalledBody, authConfig)
28622910
if err != nil {
2911+
log.Printf("[ERROR] Schemaless failure for label '%s' in org '%s'", value.Label, orgId)
28632912

28642913
if value.AppName == "HTTP" || value.App == "HTTP" {
28652914
parsedTranslation.Success = true
@@ -2877,6 +2926,10 @@ func RunActionWrapper(ctx context.Context, user shuffle.User, value shuffle.Cate
28772926
}
28782927
*/
28792928
} else {
2929+
if debug {
2930+
log.Printf("[DEBUG] In schemaless success pre upload.")
2931+
}
2932+
28802933
parsedTranslation.Success = true
28812934
//parsedTranslation.RawOutput = string(schemalessOutput)
28822935

@@ -2939,6 +2992,10 @@ func RunActionWrapper(ctx context.Context, user shuffle.User, value shuffle.Cate
29392992

29402993
// Handles the actual uploading itself
29412994
if len(foundLabelSplit) > 1 && (strings.HasPrefix(value.Label, "list_") || strings.HasPrefix(value.Label, "get_") || strings.HasPrefix(value.Label, "search_")) && len(curApikey) > 0 && len(curOrg) > 0 {
2995+
if debug {
2996+
log.Printf("[DEBUG] Should upload Singul result for label '%s'", value.Label)
2997+
}
2998+
29422999
autoUploadSingulOutput(ctx, curOrg, curApikey, curBackend, parsedTranslation, value, secondAction)
29433000
} else {
29443001
log.Printf("\n\nNot uploading. Label: %#v\n\n", value.Label)
@@ -2951,7 +3008,6 @@ func RunActionWrapper(ctx context.Context, user shuffle.User, value shuffle.Cate
29513008
//}
29523009

29533010
parsedTranslation.Retries = i + 1
2954-
29553011
if len(foundName) > 0 {
29563012
log.Printf("[DEBUG] Reverse lookup SUCCESS! Name: '%s', Labels: %v", foundName, foundLabels)
29573013
parsedTranslation.ActionName = foundName
@@ -3358,9 +3414,9 @@ func GetTranslatedHttpAction(app shuffle.WorkflowApp, action shuffle.WorkflowApp
33583414
originalActionName = fmt.Sprintf("get_%s", originalActionName)
33593415
}
33603416

3361-
if debug {
3362-
log.Printf("%s & %s", originalActionName, parsedSummary)
3363-
}
3417+
//if debug {
3418+
// log.Printf("[DEBUG] %s & %s", originalActionName, parsedSummary)
3419+
//}
33643420
//if strings.Contains(originalActionName, "messages_get") && strings.Contains(parsedSummary, "messages_get") {
33653421
//os.Exit(3)
33663422
//}
@@ -3548,8 +3604,6 @@ func GetTranslatedHttpAction(app shuffle.WorkflowApp, action shuffle.WorkflowApp
35483604
action.Parameters = append(action.Parameters, customActionParam)
35493605
}
35503606

3551-
3552-
log.Printf("ACTIONNAMEFOUND: %#v => %#v", originalActionName, originalActionNameFound)
35533607
if !originalActionNameFound {
35543608
log.Printf("[ERROR] Failed to map original action name '%s' to custom_action parameters. Returning original action.", originalActionName)
35553609
return action
@@ -4400,10 +4454,6 @@ func GetOrgspecificParameters(ctx context.Context, fields []shuffle.Valuereplace
44004454

44014455
file, err := shuffle.GetFileSingul(ctx, fileId)
44024456
if err != nil || file.Status != "active" {
4403-
//if debug {
4404-
// log.Printf("[WARNING] Parameter file %s%s NOT found or not active. Status: %#v. Err: %s", shuffle.GetSingulStandaloneFilepath(), fileId, file.Status, err)
4405-
//}
4406-
44074457
continue
44084458
}
44094459

@@ -4420,8 +4470,9 @@ func GetOrgspecificParameters(ctx context.Context, fields []shuffle.Valuereplace
44204470
continue
44214471
}
44224472

4423-
4424-
// FIXME: What happens here?
4473+
// Prepares for the replacement to work well
4474+
// E.g. /api/data/{id} => /api/data/1234
4475+
// Loops all input fields to make sure they get set
44254476
stringContent := string(content)
44264477
for _, field := range fields {
44274478
stringContent = strings.ReplaceAll(stringContent, fmt.Sprintf("{%s}", field.Key), field.Value)

0 commit comments

Comments
 (0)