diff --git a/README.md b/README.md index 87cab5bb..509003b4 100644 --- a/README.md +++ b/README.md @@ -254,3 +254,5 @@ go fmt "./..." echo "Adding changed files back to git" git diff --cached --name-only --diff-filter=ACM | grep -E "\.(go)$" | xargs git add ``` + +Update Readme \ No newline at end of file diff --git a/cmd/create.go b/cmd/create.go index 7a9baf54..1074f598 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -120,7 +120,7 @@ func NewCreateCommand(parentCmd *cobra.Command) func() { return err } - err = json.PrintJson(string(outputJson)) + err = json.PrintJsonToStdout(string(outputJson)) if err != nil { return err @@ -141,7 +141,7 @@ func NewCreateCommand(parentCmd *cobra.Command) func() { } } - return json.PrintJson(body) + return json.PrintJsonToStdout(body) } } diff --git a/cmd/delete-all.go b/cmd/delete-all.go index f7d72ffd..b9d9a6dc 100644 --- a/cmd/delete-all.go +++ b/cmd/delete-all.go @@ -95,7 +95,7 @@ func deleteAllInternal(ctx context.Context, args []string) error { } params := url.Values{} - params.Add("page[limit]", "25") + params.Add("page[limit]", "100") resp, err := httpclient.DoRequest(ctx, "GET", resourceURL, params.Encode(), nil) @@ -103,7 +103,19 @@ func deleteAllInternal(ctx context.Context, args []string) error { return err } - ids, totalCount, err := apihelper.GetResourceIdsFromHttpResponse(resp) + if resp.StatusCode >= 400 { + log.Warnf("Could not retrieve page of data, aborting") + break + } + + bodyTxt, err := io.ReadAll(resp.Body) + + if err != nil { + return err + } + + ids, totalCount, err := apihelper.GetResourceIdsFromHttpResponse(bodyTxt) + resp.Body.Close() allIds := make([][]id.IdableAttributes, 0) diff --git a/cmd/delete.go b/cmd/delete.go index d42aced9..9a9d5f77 100644 --- a/cmd/delete.go +++ b/cmd/delete.go @@ -95,7 +95,7 @@ func NewDeleteCommand(parentCmd *cobra.Command) func() { if err != nil { if body != "" { if !noBodyPrint { - json.PrintJson(body) + json.PrintJsonToStdout(body) } } return err @@ -104,7 +104,7 @@ func NewDeleteCommand(parentCmd *cobra.Command) func() { if noBodyPrint { return nil } else { - return json.PrintJson(body) + return json.PrintJsonToStdout(body) } } diff --git a/cmd/get-all.go b/cmd/get-all.go new file mode 100644 index 00000000..40f6ec68 --- /dev/null +++ b/cmd/get-all.go @@ -0,0 +1,576 @@ +package cmd + +import ( + "context" + gojson "encoding/json" + "fmt" + "github.com/elasticpath/epcc-cli/external/apihelper" + "github.com/elasticpath/epcc-cli/external/httpclient" + "github.com/elasticpath/epcc-cli/external/id" + "github.com/elasticpath/epcc-cli/external/json" + "github.com/elasticpath/epcc-cli/external/toposort" + "github.com/yukithm/json2csv" + "github.com/yukithm/json2csv/jsonpointer" + "os" + "sort" + "strings" + "sync" + + "github.com/elasticpath/epcc-cli/external/resources" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/thediveo/enumflag" + "io" + "net/url" + "reflect" + "strconv" +) + +type OutputFormat enumflag.Flag + +const ( + Jsonl OutputFormat = iota + Json + Csv + EpccCli + EpccCliRunbook +) + +var OutputFormatIds = map[OutputFormat][]string{ + Jsonl: {"jsonl"}, + Json: {"json"}, + Csv: {"csv"}, + EpccCli: {"epcc-cli"}, + EpccCliRunbook: {"epcc-cli-runbook"}, +} + +func NewGetAllCommand(parentCmd *cobra.Command) func() { + + var getAll = &cobra.Command{ + Use: "get-all", + Short: "Get all of a resource", + SilenceUsage: false, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return fmt.Errorf("please specify a resource, epcc get-all [RESOURCE], see epcc delete-all --help") + } else { + return fmt.Errorf("invalid resource [%s] specified, see all with epcc delete-all --help", args[0]) + } + }, + } + + for _, resource := range resources.GetPluralResources() { + if resource.GetCollectionInfo == nil { + continue + } + + resourceName := resource.PluralName + + var outputFile string + var outputFormat OutputFormat + + var getAllResourceCmd = &cobra.Command{ + Use: resourceName, + Short: GetGetAllShort(resource), + Hidden: false, + RunE: func(cmd *cobra.Command, args []string) error { + return getAllInternal(context.Background(), outputFormat, outputFile, append([]string{resourceName}, args...)) + }, + } + + getAllResourceCmd.Flags().StringVarP(&outputFile, "output-file", "", "", "The file to output results to") + + getAllResourceCmd.Flags().VarP( + enumflag.New(&outputFormat, "output-format", OutputFormatIds, enumflag.EnumCaseInsensitive), + "output-format", "", + "sets output format; can be 'jsonl', 'csv', 'epcc-cli'") + + getAll.AddCommand(getAllResourceCmd) + } + + parentCmd.AddCommand(getAll) + return func() {} + +} + +func writeJson(obj interface{}, writer io.Writer) error { + line, err := gojson.Marshal(&obj) + + if err != nil { + return fmt.Errorf("could not create JSON for %s, error: %v", line, err) + + } + + _, err = writer.Write(line) + + if err != nil { + return fmt.Errorf("could not save line %s, error: %v", line, err) + + } + + _, err = writer.Write([]byte{10}) + + if err != nil { + return fmt.Errorf("Could not save line %s, error: %v", line, err) + } + + return nil +} + +func getAllInternal(ctx context.Context, outputFormat OutputFormat, outputFile string, args []string) error { + // Find Resource + resource, ok := resources.GetResourceByName(args[0]) + + if !ok { + return fmt.Errorf("could not find resource %s", args[0]) + } + + if resource.GetCollectionInfo == nil { + return fmt.Errorf("resource %s doesn't support GET collection", args[0]) + } + + allParentEntityIds, err := getParentIds(ctx, resource) + + if err != nil { + return fmt.Errorf("could not retrieve parent ids for for resource %s, error: %w", resource.PluralName, err) + } + + if len(allParentEntityIds) == 1 { + log.Debugf("Resource %s is a top level resource need to scan only one path to delete all resources", resource.PluralName) + } else { + log.Debugf("Resource %s is not a top level resource, need to scan %d paths to delete all resources", resource.PluralName, len(allParentEntityIds)) + } + + var syncGroup = sync.WaitGroup{} + + syncGroup.Add(1) + + type idableAttributesWithType struct { + id.IdableAttributes + Type string `yaml:"type,omitempty" json:"type,omitempty"` + EpccCliType string `yaml:"epcc_cli_type,omitempty" json:"epcc_cli_type,omitempty"` + } + + type msg struct { + txt []byte + id []idableAttributesWithType + } + var sendChannel = make(chan msg, 0) + + var writer io.Writer + if outputFile == "" { + writer = os.Stdout + } else { + file, err := os.OpenFile(outputFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + if err != nil { + panic(err) + } + defer file.Close() + writer = file + } + + topoSortNeeded := false + + topoSortKeys := make([]string, 0) + for k, v := range resource.Attributes { + if (v.Type == fmt.Sprintf("RESOURCE_ID:%s", resource.PluralName)) || (v.Type == fmt.Sprintf("RESOURCE_ID:%s", resource.SingularName)) { + topoSortKeys = append(topoSortKeys, k) + topoSortNeeded = true + } + + } + + lines := map[string]string{} + graph := toposort.NewGraph() + + outputWriter := func() { + defer syncGroup.Done() + + csvLines := make([]interface{}, 0) + + if outputFormat == EpccCliRunbook && !topoSortNeeded { + // We need to prefix + _, err := writer.Write([]byte("- |\n")) + + if err != nil { + log.Errorf("Error writing command: %v", err) + } + } + + endMessages: + for msgs := 0; ; msgs++ { + select { + case result, ok := <-sendChannel: + if !ok { + log.Debugf("Channel closed, we are done.") + break endMessages + } + var obj interface{} + err = gojson.Unmarshal(result.txt, &obj) + + if err != nil { + log.Errorf("Couldn't unmarshal JSON response %s due to error: %v", result, err) + continue + } + + newObjs, err := json.RunJQWithArray(".data[]", obj) + + if err != nil { + log.Errorf("Couldn't process response %s due to error: %v", result, err) + continue + } + + for _, newObj := range newObjs { + + wrappedObj := map[string]interface{}{ + "data": newObj, + "meta": map[string]interface{}{ + "_epcc_cli_parent_resources": result.id, + }, + } + + if outputFormat == Jsonl { + err = writeJson(wrappedObj, writer) + + if err != nil { + log.Errorf("Error writing JSON line: %v", err) + continue + } + } else if outputFormat == Json || outputFormat == Csv { + csvLines = append(csvLines, wrappedObj) + } else if outputFormat == EpccCli || outputFormat == EpccCliRunbook { + sb := &strings.Builder{} + + sb.WriteString("epcc create -s --skip-alias-processing ") + sb.WriteString(resource.SingularName) + + sb.WriteString(" ") + sb.WriteString("--save-as-alias") + sb.WriteString(" ") + sb.WriteString("exported_source_id=") + + var myId = "" + if mp, ok := newObj.(map[string]interface{}); ok { + myId = fmt.Sprintf("%s", mp["id"]) + sb.WriteString(myId) + } else { + log.Errorf("Error casting newObj to map[string]interface{}") + sb.WriteString("\n") + continue + } + + if topoSortNeeded { + graph.AddNode(myId) + } + + for _, resId := range result.id { + sb.WriteString(" ") + sb.WriteString(resources.MustGetResourceByName(resId.EpccCliType).JsonApiType) + sb.WriteString("/") + sb.WriteString("exported_source_id=") + sb.WriteString(resId.Id) + + } + + kvs, err := json2csv.JSON2CSV(newObj) + if err != nil { + log.Errorf("Error generating Key/Value pairs: %v", err) + sb.WriteString("\n") + continue + } + + for _, kv := range kvs { + + keys := kv.Keys() + + sort.Strings(keys) + + nextKey: + for _, k := range keys { + v := kv[k] + + jp, err := jsonpointer.New(k) + + if err != nil { + log.Errorf("Couldn't generate JSON Pointer for %s: %v", k, err) + + continue + } + + jsonPointerKey := jp.DotNotation(true) + + if strings.HasPrefix(jsonPointerKey, "meta.") { + continue + } + + if strings.HasPrefix(jsonPointerKey, "links.") { + continue + } + + if jsonPointerKey == "id" { + continue + } + + if jsonPointerKey == "type" { + continue + } + + for _, e := range resource.ExcludedJsonPointersFromImport { + if len(e) == 0 { + continue + } + + if e[len(e)-1] == '.' { + if strings.HasPrefix(jsonPointerKey, e) { + continue nextKey + } + } else { + if jsonPointerKey == e { + continue nextKey + } + } + } + + sb.WriteString(" ") + sb.WriteString(jsonPointerKey) + sb.WriteString(" ") + + if s, ok := v.(string); ok { + + writeValueFromJson := true + + for _, k := range topoSortKeys { + if jsonPointerKey == k { + dependentId := fmt.Sprintf("%s", v) + graph.AddEdge(dependentId, myId) + writeValueFromJson = false + sb.WriteString(`"`) + sb.WriteString("exported_source_id=") + sb.WriteString(dependentId) + sb.WriteString(`"`) + } + } + + if writeValueFromJson { + // This is to prevent shell characters from interpreting things + sb.WriteString(`"`) + + quoteArgument := json.ValueNeedsQuotes(s) + + if quoteArgument { + // This is to force the EPCC CLI to interpret the value as a string + sb.WriteString("\\\"") + } + value := strings.ReplaceAll(s, `\`, `\\`) + value = strings.ReplaceAll(value, `$`, `\$`) + value = strings.ReplaceAll(value, `"`, `\"`) + sb.WriteString(value) + + if quoteArgument { + // This is to force the EPCC CLI to interpret the value as a string + sb.WriteString("\\\"") + } + // This is to prevent shell characters from interpreting things + sb.WriteString(`"`) + } + } else { + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in output + // TODO handle arrays in outputd + sb.WriteString(fmt.Sprintf("%v", v)) + } + + } + } + + sb.WriteString("\n") + if topoSortNeeded { + lines[myId] = sb.String() + } else { + if outputFormat == EpccCliRunbook { + // We need to prefix + _, err := writer.Write([]byte(" ")) + + if err != nil { + log.Errorf("Error writing command: %v", err) + } + } + + _, err = writer.Write([]byte(sb.String())) + + if err != nil { + log.Errorf("Error writing command: %v", err) + } + } + } + } + } + } + + if outputFormat == Json { + err = writeJson(csvLines, writer) + + if err != nil { + log.Errorf("Error writing JSON line: %v", err) + } + } else if outputFormat == Csv { + + // Create writer that saves to string + results, err := json2csv.JSON2CSV(csvLines) + + if err != nil { + log.Errorf("Error converting to CSV: %v", err) + return + } + + csvWriter := json2csv.NewCSVWriter(writer) + + csvWriter.HeaderStyle = json2csv.DotBracketStyle + csvWriter.Transpose = false + + if err := csvWriter.WriteCSV(results); err != nil { + log.Errorf("Error writing CSV: %v", err) + return + } + } else if (outputFormat == EpccCli || outputFormat == EpccCliRunbook) && topoSortNeeded { + stages, err := graph.ParallelizableStages() + + if err != nil { + log.Fatalf("Error sorting data: %v", err) + } + + for idx, stage := range stages { + writer.Write([]byte(fmt.Sprintf("# Stage %d\n", idx))) + if outputFormat == EpccCliRunbook { + writer.Write([]byte(fmt.Sprintf("- |\n"))) + } + + for _, id := range stage { + if outputFormat == EpccCliRunbook { + writer.Write([]byte(fmt.Sprintf(" "))) + } + + _, err = writer.Write([]byte(lines[id])) + + if err != nil { + log.Errorf("Error writing command: %v", err) + } + } + } + + } else if outputFormat == EpccCliRunbook && topoSortNeeded { + + } + + } + + go outputWriter() + + for _, parentEntityIds := range allParentEntityIds { + lastIds := make([][]id.IdableAttributes, 1) + for offset := 0; offset <= 10000; offset += 100 { + resourceURL, err := resources.GenerateUrlViaIdableAttributes(resource.GetCollectionInfo, parentEntityIds) + + if err != nil { + return err + } + + types, err := resources.GetSingularTypesOfVariablesNeeded(resource.GetCollectionInfo.Url) + + if err != nil { + return err + } + + params := url.Values{} + params.Add("page[limit]", "100") + params.Add("page[offset]", strconv.Itoa(offset)) + + resp, err := httpclient.DoRequest(ctx, "GET", resourceURL, params.Encode(), nil) + + if err != nil { + return err + } + + if resp.StatusCode >= 400 { + log.Warnf("Could not retrieve page of data, aborting") + + break + } + + bodyTxt, err := io.ReadAll(resp.Body) + + if err != nil { + + return err + } + + ids, totalCount, err := apihelper.GetResourceIdsFromHttpResponse(bodyTxt) + resp.Body.Close() + + allIds := make([][]id.IdableAttributes, 0) + for _, id := range ids { + allIds = append(allIds, append(parentEntityIds, id)) + } + + if reflect.DeepEqual(allIds, lastIds) { + log.Warnf("Data on the previous two pages did not change. Does this resource support pagination? Aborting export", resource.PluralName, len(allIds)) + + break + } else { + lastIds = allIds + } + + idsWithType := make([]idableAttributesWithType, len(types)) + + for i, t := range types { + idsWithType[i].IdableAttributes = parentEntityIds[i] + idsWithType[i].EpccCliType = t + idsWithType[i].Type = resources.MustGetResourceByName(t).JsonApiType + } + + sendChannel <- msg{ + bodyTxt, + idsWithType, + } + + if len(allIds) == 0 { + log.Infof("Total ids retrieved for %s in %s is %d, we are done", resource.PluralName, resourceURL, len(allIds)) + + break + } else { + if totalCount >= 0 { + log.Infof("Total number of %s in %s is %d", resource.PluralName, resourceURL, totalCount) + } else { + log.Infof("Total number %s in %s is unknown", resource.PluralName, resourceURL) + } + } + + } + } + + close(sendChannel) + + syncGroup.Wait() + + return nil +} diff --git a/cmd/get.go b/cmd/get.go index 73e50fcd..d0631694 100644 --- a/cmd/get.go +++ b/cmd/get.go @@ -189,7 +189,7 @@ func NewGetCommand(parentCmd *cobra.Command) func() { return err } - err = json.PrintJson(string(outputJson)) + err = json.PrintJsonToStdout(string(outputJson)) if err != nil { return err @@ -210,7 +210,7 @@ func NewGetCommand(parentCmd *cobra.Command) func() { } } - printError := json.PrintJson(body) + printError := json.PrintJsonToStdout(body) if retriesFailedError != nil { return retriesFailedError diff --git a/cmd/helper.go b/cmd/helper.go index 2c05dd44..f2292c97 100644 --- a/cmd/helper.go +++ b/cmd/helper.go @@ -243,6 +243,10 @@ func GetDeleteAllShort(resource resources.Resource) string { return fmt.Sprintf("Calls DELETE %s for every resource in GET %s", GetHelpResourceUrls(resource.DeleteEntityInfo.Url), GetHelpResourceUrls(resource.GetCollectionInfo.Url)) } +func GetGetAllShort(resource resources.Resource) string { + return fmt.Sprintf("Calls GET %s and iterates over all pages and parent resources (if applicable)", GetHelpResourceUrls(resource.GetCollectionInfo.Url)) +} + func GetGetLong(resourceName string, resourceUrl string, usageGetType string, completionVerb int, urlInfo *resources.CrudEntityInfo, resource resources.Resource) string { if DisableLongOutput { diff --git a/cmd/login.go b/cmd/login.go index 34eee5fc..352d314a 100644 --- a/cmd/login.go +++ b/cmd/login.go @@ -358,7 +358,7 @@ var loginCustomer = &cobra.Command{ authentication.SaveCustomerToken(*customerTokenResponse) - return json.PrintJson(body) + return json.PrintJsonToStdout(body) }, } @@ -580,7 +580,7 @@ var loginAccountManagement = &cobra.Command{ } jsonBody, _ := gojson.Marshal(selectedAccount) - return json.PrintJson(string(jsonBody)) + return json.PrintJsonToStdout(string(jsonBody)) }, } diff --git a/cmd/reset-store.go b/cmd/reset-store.go index cac506e4..1c2c48fe 100644 --- a/cmd/reset-store.go +++ b/cmd/reset-store.go @@ -207,7 +207,7 @@ func resetResourcesUndeletableResources(ctx context.Context, overrides *httpclie errors = append(errors, fmt.Errorf("error resetting %s: %v", resetCmd[0], err).Error()) } - err = json.PrintJson(body) + err = json.PrintJsonToStdout(body) if err != nil { errors = append(errors, fmt.Errorf("error resetting %s: %v", resetCmd[0], err).Error()) diff --git a/cmd/root.go b/cmd/root.go index 9ce3fba7..b29f9de7 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -110,6 +110,9 @@ func InitializeCmd() { log.Tracef("Building Delete All Commands") NewDeleteAllCommand(RootCmd) + log.Tracef("Building Get All Commands") + NewGetAllCommand(RootCmd) + Logs.AddCommand(LogsList, LogsShow, LogsClear) testJson.ResetFlags() diff --git a/cmd/runbooks.go b/cmd/runbooks.go index 2a57908d..34810b43 100644 --- a/cmd/runbooks.go +++ b/cmd/runbooks.go @@ -15,6 +15,7 @@ import ( "github.com/spf13/cobra" "golang.org/x/sync/semaphore" "gopkg.in/yaml.v3" + "os" "strconv" "strings" "sync/atomic" @@ -35,6 +36,71 @@ func initRunbookCommands() { runbookGlobalCmd.AddCommand(initRunbookRunCommands()) runbookGlobalCmd.AddCommand(initRunbookDevCommands()) runbookGlobalCmd.AddCommand(initRunbookValidateCommands()) + runbookGlobalCmd.AddCommand(initRunbookRunScriptCmd()) + +} + +func initRunbookRunScriptCmd() *cobra.Command { + + var runScript = &cobra.Command{ + Use: "exec-script", + Short: "Execute a script file specified on the command line", + Long: `This command uses a Yaml syntax and should be an array of strings. + +Each element is split on new lines and processed concurrently. + +- epcc create account name Account1 --auto-fill +- epcc create account name Account2 --auto-fill +- epcc create account-address name=Account1 --auto-fill +- epcc create account-address name=Account2 --auto-fill + +The above creates 2 accounts and 2 addresses, it takes 4 cycles to complete. + +- | + epcc create account name Account1 --auto-fill + epcc create account name Account2 --auto-fill +- | + epcc create account-address name=Account1 --auto-fill + epcc create account-address name=Account2 --auto-fill + + +The above creates the same 4 resources, it takes 2 cycles to complete, each batch can be run in parallel. + +epcc get-all can output data in the correct format to minimize execution time.`, + Args: cobra.ExactArgs(1), + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]cobra.Completion, cobra.ShellCompDirective) { + return []cobra.Completion{"yaml", "yml"}, cobra.ShellCompDirectiveFilterFileExt + }, + } + + execTimeoutInSeconds := runScript.PersistentFlags().Int64("execution-timeout", 900, "How long should the script take to execute before timing out") + maxConcurrency := runScript.PersistentFlags().Int("max-concurrency", 20, "Maximum number of commands that can run simultaneously") + + runScript.RunE = func(cmd *cobra.Command, args []string) error { + + fileContents, err := os.ReadFile(args[0]) + if err != nil { + return fmt.Errorf("Could not read file %s: %v", args[0], err) + } + + cmds := []string{} + + err = yaml.Unmarshal(fileContents, &cmds) + + action := runbooks.RunbookAction{ + Name: "exec-script", + Description: &runbooks.RunbookDescription{ + Long: "", + Short: "", + }, + RawCommands: cmds, + IgnoreErrors: false, + Variables: nil, + } + return processRunBookCommands("script", map[string]*string{}, &action, maxConcurrency, execTimeoutInSeconds) + } + + return runScript } var AbortRunbookExecution = atomic.Bool{} @@ -227,184 +293,189 @@ func initRunbookRunCommands() *cobra.Command { Long: runbookAction.Description.Long, Short: runbookAction.Description.Short, RunE: func(cmd *cobra.Command, args []string) error { - numSteps := len(runbookAction.RawCommands) + return processRunBookCommands(runbook.Name, runbookStringArguments, runbookAction, maxConcurrency, execTimeoutInSeconds) + }, + } + processRunbookVariablesOnCommand(runbookActionRunActionCommand, runbookStringArguments, runbookAction.Variables, true) - parentCtx := context.Background() + runbookRunRunbookCmd.AddCommand(runbookActionRunActionCommand) + } + } - ctx, cancelFunc := context.WithCancel(parentCtx) + return runbookRunCommand +} - concurrentRunSemaphore := semaphore.NewWeighted(int64(*maxConcurrency)) - factory := pool.NewPooledObjectFactorySimple( - func(ctx2 context.Context) (interface{}, error) { - return generateRunbookCmd(), nil - }) +func processRunBookCommands(runbookName string, runbookStringArguments map[string]*string, runbookAction *runbooks.RunbookAction, maxConcurrency *int, execTimeoutInSeconds *int64) error { + rawCmds := runbookAction.RawCommands - objectPool := pool.NewObjectPool(ctx, factory, &pool.ObjectPoolConfig{ - MaxTotal: *maxConcurrency, - MaxIdle: *maxConcurrency, - }) + numSteps := len(runbookAction.RawCommands) - rawCmds := runbookAction.RawCommands - for stepIdx := 0; stepIdx < len(rawCmds); stepIdx++ { + parentCtx := context.Background() - origIndex := &stepIdx - // Create a copy of loop variables - stepIdx := stepIdx - rawCmd := rawCmds[stepIdx] + ctx, cancelFunc := context.WithCancel(parentCtx) - templateName := fmt.Sprintf("Runbook: %s Action: %s Step: %d", runbook.Name, runbookAction.Name, stepIdx) - rawCmdLines, err := runbooks.RenderTemplates(templateName, rawCmd, runbookStringArguments, runbookAction.Variables) + concurrentRunSemaphore := semaphore.NewWeighted(int64(*maxConcurrency)) + factory := pool.NewPooledObjectFactorySimple( + func(ctx2 context.Context) (interface{}, error) { + return generateRunbookCmd(), nil + }) - if err != nil { - cancelFunc() - return err - } + objectPool := pool.NewObjectPool(ctx, factory, &pool.ObjectPoolConfig{ + MaxTotal: *maxConcurrency, + MaxIdle: *maxConcurrency, + }) - joinedString := strings.Join(rawCmdLines, "\n") - renderedCmd := []string{} + for stepIdx := 0; stepIdx < len(rawCmds); stepIdx++ { - err = yaml.Unmarshal([]byte(joinedString), &renderedCmd) + origIndex := &stepIdx + // Create a copy of loop variables + stepIdx := stepIdx + rawCmd := rawCmds[stepIdx] - if err == nil { - log.Tracef("Line %d is a Yaml array %s, inserting into stack", stepIdx, joinedString) - newCmds := make([]string, 0, len(rawCmds)+len(renderedCmd)-1) - newCmds = append(newCmds, rawCmds[0:stepIdx]...) - newCmds = append(newCmds, renderedCmd...) - newCmds = append(newCmds, rawCmds[stepIdx+1:]...) - rawCmds = newCmds - *origIndex-- - continue - } + templateName := fmt.Sprintf("Runbook: %s Action: %s Step: %d", runbookName, runbookAction.Name, stepIdx) + rawCmdLines, err := runbooks.RenderTemplates(templateName, rawCmd, runbookStringArguments, runbookAction.Variables) - log.Infof("Executing> %s", rawCmd) - resultChan := make(chan *commandResult, *maxConcurrency*2) - funcs := make([]func(), 0, len(rawCmdLines)) + if err != nil { + cancelFunc() + return err + } - for commandIdx, rawCmdLine := range rawCmdLines { + joinedString := strings.Join(rawCmdLines, "\n") + renderedCmd := []string{} - commandIdx := commandIdx - rawCmdLine := strings.Trim(rawCmdLine, " \n") + err = yaml.Unmarshal([]byte(joinedString), &renderedCmd) - if rawCmdLine == "" { - // Allow blank lines - continue - } + if err == nil { + log.Tracef("Line %d is a Yaml array %s, inserting into stack", stepIdx, joinedString) + newCmds := make([]string, 0, len(rawCmds)+len(renderedCmd)-1) + newCmds = append(newCmds, rawCmds[0:stepIdx]...) + newCmds = append(newCmds, renderedCmd...) + newCmds = append(newCmds, rawCmds[stepIdx+1:]...) + rawCmds = newCmds + *origIndex-- + continue + } - if !strings.HasPrefix(rawCmdLine, "epcc ") { - // Some commands like sleep don't have prefix - // This hack allows them to run - rawCmdLine = "epcc " + rawCmdLine - } - rawCmdArguments, err := shellwords.SplitPosix(strings.Trim(rawCmdLine, " \n")) + log.Infof("Executing> %s", rawCmd) + resultChan := make(chan *commandResult, *maxConcurrency*2) + funcs := make([]func(), 0, len(rawCmdLines)) - if err != nil { - cancelFunc() - return err - } + for commandIdx, rawCmdLine := range rawCmdLines { - funcs = append(funcs, func() { + commandIdx := commandIdx + rawCmdLine := strings.Trim(rawCmdLine, " \n") - log.Tracef("(Step %d/%d Command %d/%d) Building Commmand", stepIdx+1, numSteps, commandIdx+1, len(funcs)) + if rawCmdLine == "" { + // Allow blank lines + continue + } - stepCmdObject, err := objectPool.BorrowObject(ctx) - defer objectPool.ReturnObject(ctx, stepCmdObject) + if !strings.HasPrefix(rawCmdLine, "epcc ") { + // Some commands like sleep don't have prefix + // This hack allows them to run + rawCmdLine = "epcc " + rawCmdLine + } + rawCmdArguments, err := shellwords.SplitPosix(strings.Trim(rawCmdLine, " \n")) - if err == nil { - commandAndResetFunc := stepCmdObject.(*CommandAndReset) - commandAndResetFunc.reset() - stepCmd := commandAndResetFunc.cmd + if err != nil { + cancelFunc() + return err + } - tweakedArguments := misc.AddImplicitDoubleDash(rawCmdArguments) - stepCmd.SetArgs(tweakedArguments[1:]) + funcs = append(funcs, func() { - log.Tracef("(Step %d/%d Command %d/%d) Starting Command", stepIdx+1, numSteps, commandIdx+1, len(funcs)) + log.Tracef("(Step %d/%d Command %d/%d) Building Commmand", stepIdx+1, numSteps, commandIdx+1, len(funcs)) - stepCmd.ResetFlags() - err = stepCmd.ExecuteContext(ctx) - log.Tracef("(Step %d/%d Command %d/%d) Complete Command", stepIdx+1, numSteps, commandIdx+1, len(funcs)) - } + stepCmdObject, err := objectPool.BorrowObject(ctx) + defer objectPool.ReturnObject(ctx, stepCmdObject) - commandResult := &commandResult{ - stepIdx: stepIdx, - commandIdx: commandIdx, - commandLine: rawCmdLine, - error: err, - } + if err == nil { + commandAndResetFunc := stepCmdObject.(*CommandAndReset) + commandAndResetFunc.reset() + stepCmd := commandAndResetFunc.cmd - resultChan <- commandResult + tweakedArguments := misc.AddImplicitDoubleDash(rawCmdArguments) + stepCmd.SetArgs(tweakedArguments[1:]) - }) + log.Tracef("(Step %d/%d Command %d/%d) Starting Command", stepIdx+1, numSteps, commandIdx+1, len(funcs)) - } + stepCmd.ResetFlags() + err = stepCmd.ExecuteContext(ctx) + log.Tracef("(Step %d/%d Command %d/%d) Complete Command", stepIdx+1, numSteps, commandIdx+1, len(funcs)) + } - if len(funcs) > 1 { - log.Debugf("Running %d commands", len(funcs)) - } + commandResult := &commandResult{ + stepIdx: stepIdx, + commandIdx: commandIdx, + commandLine: rawCmdLine, + error: err, + } - // Start processing all the functions - go func() { - for idx, fn := range funcs { - idx := idx - if shutdown.ShutdownFlag.Load() { - log.Infof("Aborting runbook execution, after %d scheduled executions", idx) - cancelFunc() - break - } - - fn := fn - log.Tracef("Run %d is waiting on semaphore", idx) - if err := concurrentRunSemaphore.Acquire(ctx, 1); err == nil { - go func() { - log.Tracef("Run %d is starting", idx) - defer concurrentRunSemaphore.Release(1) - fn() - }() - } else { - log.Warnf("Run %d failed to get semaphore %v", idx, err) - } - } - }() - - errorCount := 0 - for i := 0; i < len(funcs); i++ { - select { - case result := <-resultChan: - if !shutdown.ShutdownFlag.Load() { - if result.error != nil { - log.Warnf("(Step %d/%d Command %d/%d) %v", result.stepIdx+1, numSteps, result.commandIdx+1, len(funcs), fmt.Errorf("error processing command [%s], %w", result.commandLine, result.error)) - errorCount++ - } else { - log.Debugf("(Step %d/%d Command %d/%d) finished successfully ", result.stepIdx+1, numSteps, result.commandIdx+1, len(funcs)) - } - } else { - log.Tracef("Shutdown flag enabled, completion result %v", result) - cancelFunc() - } - case <-time.After(time.Duration(*execTimeoutInSeconds) * time.Second): - return fmt.Errorf("timeout of %d seconds reached, only %d of %d commands finished of step %d/%d", *execTimeoutInSeconds, i+1, len(funcs), stepIdx+1, numSteps) + resultChan <- commandResult - } - } + }) - if len(funcs) > 1 { - log.Debugf("Running %d commands complete", len(funcs)) - } + } - if !runbookAction.IgnoreErrors && errorCount > 0 { - return fmt.Errorf("error occurred while processing script aborting") - } + if len(funcs) > 1 { + log.Debugf("Running %d commands", len(funcs)) + } + + // Start processing all the functions + go func() { + for idx, fn := range funcs { + idx := idx + if shutdown.ShutdownFlag.Load() { + log.Infof("Aborting runbook execution, after %d scheduled executions", idx) + cancelFunc() + break + } + + fn := fn + log.Tracef("Run %d is waiting on semaphore", idx) + if err := concurrentRunSemaphore.Acquire(ctx, 1); err == nil { + go func() { + log.Tracef("Run %d is starting", idx) + defer concurrentRunSemaphore.Release(1) + fn() + }() + } else { + log.Warnf("Run %d failed to get semaphore %v", idx, err) + } + } + }() + + errorCount := 0 + for i := 0; i < len(funcs); i++ { + select { + case result := <-resultChan: + if !shutdown.ShutdownFlag.Load() { + if result.error != nil { + log.Warnf("(Step %d/%d Command %d/%d) %v", result.stepIdx+1, numSteps, result.commandIdx+1, len(funcs), fmt.Errorf("error processing command [%s], %w", result.commandLine, result.error)) + errorCount++ + } else { + log.Debugf("(Step %d/%d Command %d/%d) finished successfully ", result.stepIdx+1, numSteps, result.commandIdx+1, len(funcs)) } - defer cancelFunc() - return nil - }, + } else { + log.Tracef("Shutdown flag enabled, completion result %v", result) + cancelFunc() + } + case <-time.After(time.Duration(*execTimeoutInSeconds) * time.Second): + return fmt.Errorf("timeout of %d seconds reached, only %d of %d commands finished of step %d/%d", *execTimeoutInSeconds, i+1, len(funcs), stepIdx+1, numSteps) + } - processRunbookVariablesOnCommand(runbookActionRunActionCommand, runbookStringArguments, runbookAction.Variables, true) + } - runbookRunRunbookCmd.AddCommand(runbookActionRunActionCommand) + if len(funcs) > 1 { + log.Debugf("Running %d commands complete", len(funcs)) } - } - return runbookRunCommand + if !runbookAction.IgnoreErrors && errorCount > 0 { + return fmt.Errorf("error occurred while processing script aborting") + } + } + defer cancelFunc() + return nil } func processRunbookVariablesOnCommand(runbookActionRunActionCommand *cobra.Command, runbookStringArguments map[string]*string, variables map[string]runbooks.Variable, enableRequiredVars bool) { diff --git a/cmd/test-json.go b/cmd/test-json.go index da423a10..eec267ea 100644 --- a/cmd/test-json.go +++ b/cmd/test-json.go @@ -15,7 +15,7 @@ var testJson = &cobra.Command{ res, err := json.ToJson(args, noWrapping, compliant, map[string]*resources.CrudEntityAttribute{}, true) if res != "" { - json.PrintJson(res) + json.PrintJsonToStdout(res) } return err diff --git a/cmd/update.go b/cmd/update.go index e6179837..12bac063 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -112,7 +112,7 @@ func NewUpdateCommand(parentCmd *cobra.Command) func() { return err } - err = json.PrintJson(string(outputJson)) + err = json.PrintJsonToStdout(string(outputJson)) if err != nil { return err @@ -133,7 +133,7 @@ func NewUpdateCommand(parentCmd *cobra.Command) func() { } } - return json.PrintJson(body) + return json.PrintJsonToStdout(body) } } diff --git a/external/aliases/aliases.go b/external/aliases/aliases.go index 3426bf99..614cb721 100644 --- a/external/aliases/aliases.go +++ b/external/aliases/aliases.go @@ -137,8 +137,9 @@ func getAliasesForSingleJsonApiType(jsonApiType string) map[string]*id.IdableAtt func ResolveAliasValuesOrReturnIdentity(jsonApiType string, alternateJsonApiTypes []string, aliasName string, attribute string) string { splitAlias := strings.Split(aliasName, "/") + // TODO you can get weird behaviour if you use a / in an alias + // fix in another bug (I think the Commerce Extension Runbook also needs a fix) if len(splitAlias) == 2 { - // alternateJsonApiTypes = append(alternateJsonApiTypes, splitAlias[0]) aliasName = splitAlias[1] } @@ -167,7 +168,6 @@ func ResolveAliasValuesOrReturnIdentity(jsonApiType string, alternateJsonApiType } log.Warnf("Alias was found for for %s, but the attribute is unknown, must be one of {id, slug, sku, code}, but got %s", aliasName, attribute) - } return aliasName } diff --git a/external/apihelper/get_all_ids.go b/external/apihelper/get_all_ids.go index 3fb4cb37..b1fdbc9a 100644 --- a/external/apihelper/get_all_ids.go +++ b/external/apihelper/get_all_ids.go @@ -7,6 +7,7 @@ import ( "github.com/elasticpath/epcc-cli/external/id" "github.com/elasticpath/epcc-cli/external/resources" log "github.com/sirupsen/logrus" + "io" "net/url" "reflect" ) @@ -79,7 +80,13 @@ func GetAllIds(ctx context.Context, resource *resources.Resource) ([][]id.Idable return myEntityIds, err } - ids, _, err := GetResourceIdsFromHttpResponse(resp) + bodyTxt, err := io.ReadAll(resp.Body) + + if err != nil { + return nil, err + } + + ids, _, err := GetResourceIdsFromHttpResponse(bodyTxt) if reflect.DeepEqual(ids, lastPageIds) { log.Debugf("Resource %s does not seem to support pagination as we got the exact same set of ids back as the last page... breaking. This might happen if exactly a paginated number of records is returned", resource.PluralName) diff --git a/external/apihelper/map_collection_response_to_ids.go b/external/apihelper/map_collection_response_to_ids.go index d8f07beb..3bde437e 100644 --- a/external/apihelper/map_collection_response_to_ids.go +++ b/external/apihelper/map_collection_response_to_ids.go @@ -4,21 +4,12 @@ import ( json2 "encoding/json" "fmt" "github.com/elasticpath/epcc-cli/external/id" - log "github.com/sirupsen/logrus" - "io" - "net/http" ) -func GetResourceIdsFromHttpResponse(resp *http.Response) ([]id.IdableAttributes, int, error) { - // Read the body - body, err := io.ReadAll(resp.Body) - - if err != nil { - log.Fatal(err) - } +func GetResourceIdsFromHttpResponse(bodyTxt []byte) ([]id.IdableAttributes, int, error) { var jsonStruct = map[string]interface{}{} - err = json2.Unmarshal(body, &jsonStruct) + err := json2.Unmarshal(bodyTxt, &jsonStruct) if err != nil { return nil, 0, fmt.Errorf("response for get was not JSON: %w", err) } diff --git a/external/id/idable_attributes.go b/external/id/idable_attributes.go index 57e0d127..5c48ca94 100644 --- a/external/id/idable_attributes.go +++ b/external/id/idable_attributes.go @@ -1,8 +1,8 @@ package id type IdableAttributes struct { - Id string `yaml:"id"` - Slug string `yaml:"slug,omitempty"` - Sku string `yaml:"sku,omitempty"` - Code string `yaml:"code,omitempty"` + Id string `yaml:"id" json:"id"` + Slug string `yaml:"slug,omitempty" json:"slug,omitempty"` + Sku string `yaml:"sku,omitempty" json:"sku,omitempty"` + Code string `yaml:"code,omitempty" json:"code,omitempty"` } diff --git a/external/json/encoder.go b/external/json/encoder.go index 962bdf93..31c13cf7 100644 --- a/external/json/encoder.go +++ b/external/json/encoder.go @@ -17,13 +17,14 @@ import ( // https://github.com/itchyny/gojq/blob/main/cli/color.go type encoder struct { - out io.Writer - w *bytes.Buffer - tab bool - indent int - depth int - buf [64]byte - keyStack []string + out io.Writer + w *bytes.Buffer + tab bool + monoOutput bool + indent int + depth int + buf [64]byte + keyStack []string } type colorInfo struct { @@ -31,8 +32,8 @@ type colorInfo struct { colorString string } -func setColor(buf *bytes.Buffer, color colorInfo) { - if !MonochromeOutput { +func (e *encoder) setColor(buf *bytes.Buffer, color colorInfo) { + if !e.monoOutput { buf.Write([]byte(color.colorString)) } } @@ -107,9 +108,9 @@ var ( objectColor = newColor("", "") // No color ) -func NewEncoder(tab bool, indent int) *encoder { +func NewEncoder(tab bool, indent int, monoOutput bool) *encoder { // reuse the buffer in multiple calls of marshal - return &encoder{w: new(bytes.Buffer), tab: tab, indent: indent} + return &encoder{w: new(bytes.Buffer), tab: tab, indent: indent, monoOutput: monoOutput} } func (e *encoder) Marshal(v interface{}, w io.Writer) error { @@ -182,7 +183,7 @@ func (e *encoder) encodeFloat64(f float64) { // ref: encodeState#string in encoding/json func (e *encoder) encodeString(s string, color *colorInfo) { if color != nil { - setColor(e.w, *color) + e.setColor(e.w, *color) } e.w.WriteByte('"') start := 0 @@ -236,7 +237,7 @@ func (e *encoder) encodeString(s string, color *colorInfo) { } e.w.WriteByte('"') if color != nil { - setColor(e.w, resetColor) + e.setColor(e.w, resetColor) } } @@ -378,9 +379,9 @@ func (e *encoder) writeByte(b byte, color *colorInfo) { if color == nil || color.colorString == "" { e.w.WriteByte(b) } else { - setColor(e.w, *color) + e.setColor(e.w, *color) e.w.WriteByte(b) - setColor(e.w, resetColor) + e.setColor(e.w, resetColor) } } @@ -388,8 +389,8 @@ func (e *encoder) write(bs []byte, color *colorInfo) { if color == nil || color.colorString == "" { e.w.Write(bs) } else { - setColor(e.w, *color) + e.setColor(e.w, *color) e.w.Write(bs) - setColor(e.w, resetColor) + e.setColor(e.w, resetColor) } } diff --git a/external/json/print_json.go b/external/json/print_json.go index 0b5a8f8f..a0dd7877 100644 --- a/external/json/print_json.go +++ b/external/json/print_json.go @@ -12,15 +12,33 @@ import ( var MonochromeOutput = false -func PrintJson(json string) error { +func PrintJsonToStdout(json string) error { defer os.Stdout.Sync() - return printJsonToWriter(json, os.Stdout) + return printJsonToWriter(json, shouldPrintMonochrome(), os.Stdout) +} + +func shouldPrintMonochrome() bool { + m := MonochromeOutput + // Adapted from gojq + if !m && os.Getenv("TERM") == "dumb" { + m = true + } else { + colorCapableTerminal := isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) + if !colorCapableTerminal { + m = true + } + } + return m +} + +func PrintJsonToWriter(json string, w io.Writer) error { + return printJsonToWriter(json, true, w) } func PrintJsonToStderr(json string) error { defer os.Stderr.Sync() - return printJsonToWriter(json, os.Stderr) + return printJsonToWriter(json, shouldPrintMonochrome(), os.Stderr) } func PrettyPrint(in string) string { @@ -32,28 +50,19 @@ func PrettyPrint(in string) string { return out.String() } -func printJsonToWriter(json string, w io.Writer) error { - // Adapted from gojq - if os.Getenv("TERM") == "dumb" { - MonochromeOutput = true - } else { - colorCapableTerminal := isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) - if !colorCapableTerminal { - MonochromeOutput = true - } - } +func printJsonToWriter(json string, monoOutput bool, w io.Writer) error { var v interface{} err := gojson.Unmarshal([]byte(json), &v) - e := NewEncoder(false, 2) + e := NewEncoder(false, 2, monoOutput) done := make(chan bool, 1) defer close(done) - if !MonochromeOutput { + if !monoOutput { go func() { select { case <-done: diff --git a/external/json/to_json.go b/external/json/to_json.go index 948e182c..4a7db19f 100644 --- a/external/json/to_json.go +++ b/external/json/to_json.go @@ -286,12 +286,19 @@ func RunJQWithArray(queryStr string, result interface{}) ([]interface{}, error) } return queryResult, nil } + +var TreatAsLiterals = regexp.MustCompile("^(-?[0-9]+(\\.[0-9]+)?|false|true|null)$") + +var QuotedString = regexp.MustCompile("^\\\".+\\\"$") + +var EmptyArray = regexp.MustCompile("^\\[]$") + func formatValue(v string) string { - if match, _ := regexp.MatchString("^(-?[0-9]+(\\.[0-9]+)?|false|true|null)$", v); match { + if match := TreatAsLiterals.MatchString(v); match { return v - } else if match, _ := regexp.MatchString("^\\\".+\\\"$", v); match { + } else if match := QuotedString.MatchString(v); match { return v - } else if match, _ := regexp.MatchString("^\\[\\]$", v); match { + } else if match := EmptyArray.MatchString(v); match { return v } else { v = strings.ReplaceAll(v, "\\", "\\\\") @@ -300,3 +307,11 @@ func formatValue(v string) string { return fmt.Sprintf("\"%s\"", v) } } + +func ValueNeedsQuotes(v string) bool { + if match := TreatAsLiterals.MatchString(v); match { + return true + } else { + return false + } +} diff --git a/external/resources/resources.go b/external/resources/resources.go index 27614957..cb31ebdb 100644 --- a/external/resources/resources.go +++ b/external/resources/resources.go @@ -63,6 +63,9 @@ type Resource struct { // This should only be used for cases where we manually fix things, or where // a store reset would clear a resource another way (e.g., the resource represents a projection). SuppressResetWarning bool `yaml:"suppress-reset-warning,omitempty"` + + // Exclude these json pointers from import + ExcludedJsonPointersFromImport []string `yaml:"excluded-json-pointers-from-import"` } type CrudEntityInfo struct { diff --git a/external/resources/resources_schema.json b/external/resources/resources_schema.json index 6445bca1..78eb645b 100644 --- a/external/resources/resources_schema.json +++ b/external/resources/resources_schema.json @@ -22,7 +22,16 @@ "enum": ["compliant", "legacy"] }, "alternate-json-type-for-aliases": { - "type":"array" + "type":"array", + "items": { + "type": "string" + } + }, + "excluded-json-pointers-from-import": { + "type": "array", + "items": { + "type": "string" + } }, "no-wrapping": { "type": "boolean" diff --git a/external/resources/yaml/resources.yaml b/external/resources/yaml/resources.yaml index 66bfb0c0..57b2d88e 100644 --- a/external/resources/yaml/resources.yaml +++ b/external/resources/yaml/resources.yaml @@ -1558,6 +1558,8 @@ pcm-hierarchies: type: STRING locales.en-US.description: type: STRING + excluded-json-pointers-from-import: + - relationships.children.links.related pcm-hierarchy-node-children: singular-name: "pcm-hierarchy-node-child" json-api-type: "node" @@ -1642,6 +1644,9 @@ pcm-nodes: type: ENUM:node relationships.parent.data.id: type: RESOURCE_ID:pcm-nodes + excluded-json-pointers-from-import: + - relationships.children. + - relationships.products. pcm-product-main-image: singular-name: "pcm-product-main-image" json-api-type: "file" diff --git a/external/rest/create.go b/external/rest/create.go index 576b65af..3d741aa6 100644 --- a/external/rest/create.go +++ b/external/rest/create.go @@ -118,7 +118,7 @@ func CreateInternal(ctx context.Context, overrides *httpclient.HttpParameterOver // Check if error response if resp.StatusCode >= 400 && resp.StatusCode <= 600 { - json.PrintJson(string(resBody)) + json.PrintJsonToStdout(string(resBody)) return "", fmt.Errorf(resp.Status) } diff --git a/external/rest/get.go b/external/rest/get.go index 48a20c53..fb908f6b 100644 --- a/external/rest/get.go +++ b/external/rest/get.go @@ -35,7 +35,7 @@ func GetInternal(ctx context.Context, overrides *httpclient.HttpParameterOverrid // Check if error response if resp.StatusCode >= 400 && resp.StatusCode <= 600 { - json.PrintJson(string(body)) + json.PrintJsonToStdout(string(body)) return "", fmt.Errorf(resp.Status) } diff --git a/external/rest/update.go b/external/rest/update.go index a8e83bb6..0bf1d3b6 100644 --- a/external/rest/update.go +++ b/external/rest/update.go @@ -82,7 +82,7 @@ func UpdateInternal(ctx context.Context, overrides *httpclient.HttpParameterOver // Check if error response if resp.StatusCode >= 400 && resp.StatusCode <= 600 { - json.PrintJson(string(resBody)) + json.PrintJsonToStdout(string(resBody)) return "", fmt.Errorf(resp.Status) } diff --git a/external/runbooks/account-management.epcc.yml b/external/runbooks/account-management.epcc.yml index bfbc2dfb..c26eb37d 100644 --- a/external/runbooks/account-management.epcc.yml +++ b/external/runbooks/account-management.epcc.yml @@ -10,6 +10,15 @@ actions: # Initialize alias for Authentication Realm - epcc get account-authentication-settings - epcc create password-profile related_authentication_realm_for_account_authentication_settings_last_read=entity name "Username and Password Authentication" + enable-self-signup-and-management: + description: + short: "Enable password authentication" + commands: + # Initialize alias for Authentication Realm + - epcc get account-authentication-settings + - | + epcc create password-profile related_authentication_realm_for_account_authentication_settings_last_read=entity name "Username and Password Authentication" + epcc update account-authentication-setting enable_self_signup true auto_create_account_for_account_members true account_member_self_management "update_only" create-deep-hierarchy: description: short: "Create a hierarchy" @@ -25,7 +34,7 @@ actions: description: short: "Width of the hierarchy" commands: - # language=YAML + # language=Yaml - |2 {{- range untilStep 0 $.depth 1 -}} {{- $d := . -}} diff --git a/external/runbooks/run-all-runbooks.sh b/external/runbooks/run-all-runbooks.sh index 920ac571..9055d580 100755 --- a/external/runbooks/run-all-runbooks.sh +++ b/external/runbooks/run-all-runbooks.sh @@ -96,6 +96,7 @@ echo "Starting Account Management Runbook" epcc reset-store .+ epcc runbooks run account-management enable-password-authentication +epcc runbooks run account-management enable-self-signup-and-management epcc runbooks run account-management create-singleton-account-member epcc runbooks run account-management catalog-rule-example epcc runbooks run account-management catalog-rule-example-reset diff --git a/external/runbooks/runbook_processing.go b/external/runbooks/runbook_processing.go new file mode 100644 index 00000000..868f3153 --- /dev/null +++ b/external/runbooks/runbook_processing.go @@ -0,0 +1 @@ +package runbooks diff --git a/external/runbooks/scripts/account-parallel-script.yml b/external/runbooks/scripts/account-parallel-script.yml new file mode 100644 index 00000000..1d63e7fa --- /dev/null +++ b/external/runbooks/scripts/account-parallel-script.yml @@ -0,0 +1,8 @@ +- | + epcc create -s account name Account1 --auto-fill + epcc create -s account name Account2 --auto-fill + +- | + epcc create -s account-address name=Account1 --auto-fill + epcc create -s account-address name=Account2 --auto-fill + diff --git a/external/runbooks/scripts/account-sequential-script.yml b/external/runbooks/scripts/account-sequential-script.yml new file mode 100644 index 00000000..1d78b16c --- /dev/null +++ b/external/runbooks/scripts/account-sequential-script.yml @@ -0,0 +1,4 @@ +- epcc create -s account name Account1 --auto-fill +- epcc create -s account name Account2 --auto-fill +- epcc create -s account-address name=Account1 --auto-fill +- epcc create -s account-address name=Account2 --auto-fill diff --git a/external/toposort/toposort.go b/external/toposort/toposort.go new file mode 100644 index 00000000..aa1a77fd --- /dev/null +++ b/external/toposort/toposort.go @@ -0,0 +1,100 @@ +package toposort + +import ( + "fmt" + "sort" +) + +type Graph struct { + edges map[string][]string + vertices map[string]struct{} + indegree map[string]int // Track number of dependencies +} + +func NewGraph() *Graph { + return &Graph{ + edges: make(map[string][]string), + vertices: make(map[string]struct{}), + indegree: make(map[string]int), + } +} + +func (g *Graph) AddNode(v string) { + g.vertices[v] = struct{}{} + if _, exists := g.edges[v]; !exists { + g.edges[v] = []string{} // Ensure it exists in the adjacency list + } +} + +func (g *Graph) AddEdge(u, v string) { + g.vertices[u] = struct{}{} + g.vertices[v] = struct{}{} + g.edges[u] = append(g.edges[u], v) + g.indegree[v]++ // Track dependencies +} + +// Determines parallel execution stages using Kahn's Algorithm +func (g *Graph) ParallelizableStages() ([][]string, error) { + indegree := make(map[string]int) + for v := range g.vertices { + indegree[v] = g.indegree[v] // Copy original indegrees + } + + queue := []string{} + for v, deg := range indegree { + if deg == 0 { + queue = append(queue, v) + } + } + + var levels [][]string + count := 0 + + for len(queue) > 0 { + sort.Strings(queue) // Sort the current stage before processing + var nextQueue []string + levels = append(levels, queue) // Nodes at current level + count += len(queue) + + for _, v := range queue { + for _, neighbor := range g.edges[v] { + indegree[neighbor]-- + if indegree[neighbor] == 0 { + nextQueue = append(nextQueue, neighbor) + } + } + } + + queue = nextQueue // Move to next level + } + + // If not all nodes were processed, there is a cycle + if count != len(g.vertices) { + var cycleNodes []string + for v, deg := range indegree { + if deg > 0 { + cycleNodes = append(cycleNodes, v) + } + } + sort.Strings(cycleNodes) + + return nil, fmt.Errorf("cycle detected in graph : %v", cycleNodes) + } + + return levels, nil +} + +// Topological Sort - Simply flattens ParallelizableStages output +func (g *Graph) TopologicalSort() ([]string, error) { + stages, err := g.ParallelizableStages() + if err != nil { + return nil, err + } + + var order []string + for _, stage := range stages { + order = append(order, stage...) + } + + return order, nil +} diff --git a/external/toposort/toposort_test.go b/external/toposort/toposort_test.go new file mode 100644 index 00000000..f2f985a7 --- /dev/null +++ b/external/toposort/toposort_test.go @@ -0,0 +1,77 @@ +package toposort + +import ( + "github.com/stretchr/testify/require" + "testing" +) + +func TestTopoSort(t *testing.T) { + // Fixture Setup + g := NewGraph() + // Execute SUT + g.AddEdge("c", "b") + g.AddEdge("b", "a") + + sort, err := g.TopologicalSort() + + // Verification + require.NoError(t, err) + require.Equal(t, []string{"c", "b", "a"}, sort) +} + +func TestTopoSortWithExtraNodes(t *testing.T) { + // Fixture Setup + g := NewGraph() + // Execute SUT + g.AddNode("c") + g.AddEdge("c", "b") + g.AddNode("d") + g.AddEdge("b", "a") + g.AddNode("a") + + sort, err := g.TopologicalSort() + + // Verification + require.NoError(t, err) + require.Equal(t, []string{"c", "d", "b", "a"}, sort) +} + +func TestTopoSortWithStagesAndExtraNodes(t *testing.T) { + // Fixture Setup + g := NewGraph() + // Execute SUT + g.AddNode("c") + g.AddEdge("c", "b") + g.AddNode("d") + g.AddEdge("b", "a") + g.AddNode("a") + g.AddEdge("b", "x") + g.AddEdge("y", "z") + g.AddEdge("x", "z") + g.AddEdge("f", "y") + g.AddEdge("w", "x") + + sort, err := g.ParallelizableStages() + + // Verification + require.NoError(t, err) + require.Equal(t, [][]string{{"c", "d", "f", "w"}, {"b", "y"}, {"a", "x"}, {"z"}}, sort) +} + +func TestTopoSortCircle(t *testing.T) { + // Fixture Setup + g := NewGraph() + + // Execute SUT + g.AddEdge("e", "c") + g.AddEdge("c", "b") + g.AddEdge("b", "a") + g.AddEdge("a", "c") + g.AddEdge("d", "b") + g.AddEdge("c", "f") + + _, err := g.TopologicalSort() + + // Verification + require.ErrorContains(t, err, "cycle detected in graph") +} diff --git a/go.mod b/go.mod index 300096e2..df427044 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/iancoleman/strcase v0.2.0 github.com/jolestar/go-commons-pool/v2 v2.1.2 github.com/mitchellh/mapstructure v1.5.0 + github.com/yukithm/json2csv v0.1.2 ) require dario.cat/mergo v1.0.1 // indirect diff --git a/go.sum b/go.sum index 17d71d5c..22e02c44 100644 --- a/go.sum +++ b/go.sum @@ -63,6 +63,7 @@ github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -99,6 +100,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/gox v1.0.1/go.mod h1:ED6BioOGXMswlXa2zxfh/xdd5QhwYliBFn9V18Ap4z4= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -156,6 +159,8 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/stevenle/topsort v0.2.0 h1:LLWgtp34HPX6/RBDRS0kElVxGOTzGBLI1lSAa5Lb46k= +github.com/stevenle/topsort v0.2.0/go.mod h1:ck2WG2/ZrOr6dLApQ/5Xrqy5wv3T0qhKYWE7r9tkibc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -167,12 +172,15 @@ github.com/thediveo/enumflag v0.10.1 h1:DB3Ag69VZ7BCv6jzKECrZ0ebZrHLzFRMIFYt96s4 github.com/thediveo/enumflag v0.10.1/go.mod h1:KyVhQUPzreSw85oJi2uSjFM0ODLKXBH0rPod7zc2pmI= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +github.com/yukithm/json2csv v0.1.2 h1:b2aIY9+TOY5Wss9lCku4wjqnQrENv5Ix1G0ZHN1FE2Q= +github.com/yukithm/json2csv v0.1.2/go.mod h1:Ul6ZenFV94YeUm08AqppOd+/hB9JsmiU4KXPs9ZvgwQ= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=