From 27c0c4e7c24a6ee7ccf85e9c9ccbfa0aa8535177 Mon Sep 17 00:00:00 2001 From: Amber Xu Date: Mon, 10 Nov 2025 14:50:03 -0800 Subject: [PATCH 1/6] run v4 cmd and unit test --- cmd/functions.go | 8 +- cmd/run.go | 495 +++++++++++++++++++++++++++++++-------------- cmd/run_v3_test.go | 247 ++++++++++++++++++++++ cmd/run_v4_test.go | 197 ++++++++++++++++++ 4 files changed, 793 insertions(+), 154 deletions(-) create mode 100644 cmd/run_v3_test.go create mode 100644 cmd/run_v4_test.go diff --git a/cmd/functions.go b/cmd/functions.go index 62968b7..5386829 100644 --- a/cmd/functions.go +++ b/cmd/functions.go @@ -48,8 +48,8 @@ func buildFmeFlowRequest(endpoint string, method string, body io.Reader) (http.R } // since the JSON for published parameters has subtypes, we need to implement this ourselves -func (f *JobRequest) UnmarshalJSON(b []byte) error { - type job JobRequest +func (f *JobRequestV3) UnmarshalJSON(b []byte) error { + type job JobRequestV3 err := json.Unmarshal(b, (*job)(f)) if err != nil { return err @@ -85,9 +85,9 @@ func (f *JobRequest) UnmarshalJSON(b []byte) error { return nil } -func (f *JobRequest) MarshalJSON() ([]byte, error) { +func (f *JobRequestV3) MarshalJSON() ([]byte, error) { - type job JobRequest + type job JobRequestV3 if f.PublishedParameters != nil { for _, v := range f.PublishedParameters { b, err := json.Marshal(v) diff --git a/cmd/run.go b/cmd/run.go index 69757c4..6ca843c 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -16,6 +16,7 @@ import ( "github.com/jedib0t/go-pretty/v6/table" "github.com/spf13/cobra" + "github.com/spf13/viper" ) type PublishedParameter struct { @@ -41,7 +42,20 @@ type JobId struct { Id int `json:"id"` } -type JobRequest struct { +type JobRequestV4 struct { + Directives map[string]string `json:"directives,omitempty"` + FailureTopics []string `json:"failureTopics,omitempty"` + SuccessTopics []string `json:"successTopics,omitempty"` + MaxJobRuntime int `json:"maxJobRuntime,omitempty"` + MaxTimeInQueue int `json:"maxTimeInQueue,omitempty"` + Queue string `json:"queue,omitempty"` + Repository string `json:"repository,omitempty"` + Workspace string `json:"workspace,omitempty"` + PublishedParameters map[string]interface{} `json:"publishedParameters,omitempty"` + MaxTotalLifeTime int `json:"maxTotalLifeTime,omitempty"` +} + +type JobRequestV3 struct { PublishedParameters []interface{} `json:"-"` RawPublishedParameters []json.RawMessage `json:"publishedParameters,omitempty"` TMDirectives struct { @@ -58,7 +72,19 @@ type JobRequest struct { } `json:"NMDirectives,omitempty"` } -type JobResult struct { +type JobResultV4 struct { + ID int `json:"id"` + FeatureOutputCount int `json:"featureOutputCount"` + RequesterHost string `json:"requesterHost"` + RequesterResultPort int `json:"requesterResultPort"` + Status string `json:"status"` + StatusMessage string `json:"statusMessage"` + TimeFinished time.Time `json:"timeFinished"` + TimeQueued time.Time `json:"timeQueued"` + TimeStarted time.Time `json:"timeStarted"` +} + +type JobResultV3 struct { TimeRequested time.Time `json:"timeRequested"` RequesterResultPort int `json:"requesterResultPort"` NumFeaturesOutput int `json:"numFeaturesOutput"` @@ -86,6 +112,11 @@ type runFlags struct { publishedParameter []string listPublishedParameter []string nodeManagerDirective []string + directive []string + queue string + maxJobRuntime int + maxTimeInQueue int + maxTotalLifeTime int outputType string noHeaders bool } @@ -121,17 +152,22 @@ func newRunCmd() *cobra.Command { cmd.Flags().StringVar(&f.repository, "repository", "", "The name of the repository containing the workspace to run.") cmd.Flags().StringVar(&f.workspace, "workspace", "", "The name of the workspace to run.") cmd.Flags().BoolVar(&f.wait, "wait", false, "Submit job and wait for it to finish.") - cmd.Flags().StringVar(&f.tag, "tag", "", "The job routing tag for the request") + cmd.Flags().StringVar(&f.tag, "tag", "", "The job routing tag for the request. For v3 API only.") cmd.Flags().StringArrayVar(&f.publishedParameter, "published-parameter", []string{}, "Published parameters defined for this workspace. Specify as Key=Value. Can be passed in multiple times. For list parameters, use the --list-published-parameter flag.") cmd.Flags().StringArrayVar(&f.listPublishedParameter, "published-parameter-list", []string{}, "A List-type published parameters defined for this workspace. Specify as Key=Value1,Value2. Can be passed in multiple times.") - cmd.Flags().StringVar(&f.sourceData, "file", "", "Upload a local file Source dataset to use to run the workspace. Note this causes the translation to run in synchonous mode whether the --wait flag is passed in or not.") - cmd.Flags().BoolVar(&f.rtc, "run-until-canceled", false, "Runs a job until it is explicitly canceled. The job will run again regardless of whether the job completed successfully, failed, or the server crashed or was shut down.") - cmd.Flags().IntVar(&f.ttc, "time-until-canceled", -1, "Time (in seconds) elapsed for a running job before it's cancelled. The minimum value is 1 second, values less than 1 second are ignored.") - cmd.Flags().IntVar(&f.ttl, "time-to-live", -1, "Time to live in the job queue (in seconds)") - cmd.Flags().StringVar(&f.description, "description", "", "Description of the request.") + cmd.Flags().StringVar(&f.sourceData, "file", "", "Upload a local file Source dataset to use to run the workspace. Note this causes the translation to run in synchonous mode whether the --wait flag is passed in or not. For v3 API only.") + cmd.Flags().BoolVar(&f.rtc, "run-until-canceled", false, "Runs a job until it is explicitly canceled. The job will run again regardless of whether the job completed successfully, failed, or the server crashed or was shut down. For v3 API only.") + cmd.Flags().IntVar(&f.ttc, "time-until-canceled", -1, "Time (in seconds) elapsed for a running job before it's cancelled. The minimum value is 1 second, values less than 1 second are ignored. For v3 API only.") + cmd.Flags().IntVar(&f.ttl, "time-to-live", -1, "Time to live in the job queue (in seconds). For v3 API only.") + cmd.Flags().StringVar(&f.description, "description", "", "Description of the request. For v3 API only.") cmd.Flags().StringArrayVar(&f.successTopics, "success-topic", []string{}, "Topics to notify when the job succeeds. Can be specified more than once.") cmd.Flags().StringArrayVar(&f.failureTopics, "failure-topic", []string{}, "Topics to notify when the job fails. Can be specified more than once.") - cmd.Flags().StringArrayVar(&f.nodeManagerDirective, "node-manager-directive", []string{}, "Additional NM Directives, which can include client-configured keys, to pass to the notification service for custom use by subscriptions. Specify as Key=Value Can be passed in multiple times.") + cmd.Flags().StringArrayVar(&f.nodeManagerDirective, "node-manager-directive", []string{}, "Additional NM Directives, which can include client-configured keys, to pass to the notification service for custom use by subscriptions. Specify as Key=Value Can be passed in multiple times. For v3 API only.") + cmd.Flags().StringArrayVar(&f.directive, "directive", []string{}, "Additional directives to pass to the job submission. Specify as Key=Value. Can be passed in multiple times. For v4 API only.") + cmd.Flags().StringVar(&f.queue, "queue", "", "Queue of the job to submit. For v4 API only.") + cmd.Flags().IntVar(&f.maxJobRuntime, "max-job-runtime", 0, "Time (in seconds) elapsed for a running job before it's cancelled. The minimum value is 1 second, values less than 1 second are ignored. For v4 API only.") + cmd.Flags().IntVar(&f.maxTimeInQueue, "max-time-in-queue", 0, "Time to live in the job queue (in seconds). For v4 API only.") + cmd.Flags().IntVar(&f.maxTotalLifeTime, "max-total-life-time", 0, "Time to live including both time in the queue and run time (in seconds). The maximum value is 86400 and the minimum value is 1. For v4 API only.") cmd.Flags().StringVarP(&f.outputType, "output", "o", "table", "Specify the output type. Should be one of table, json, or custom-columns") cmd.Flags().BoolVar(&f.noHeaders, "no-headers", false, "Don't print column headers") @@ -161,60 +197,47 @@ func runRun(f *runFlags) func(cmd *cobra.Command, args []string) error { Timeout: 604800 * time.Second, } - var result JobResult - var responseData []byte + if viper.GetInt("build") >= 26018 { + var result JobResultV4 + var responseData []byte - if f.sourceData == "" { - job := &JobRequest{} + job := &JobRequestV4{} + job.PublishedParameters = make(map[string]interface{}) // get published parameters for _, parameter := range f.publishedParameter { this_parameter := strings.SplitN(parameter, "=", 2) - var a SimpleParameter - a.Name = this_parameter[0] - a.Value = this_parameter[1] - job.PublishedParameters = append(job.PublishedParameters, a) + job.PublishedParameters[this_parameter[0]] = this_parameter[1] } // get list published parameters for _, parameter := range f.listPublishedParameter { this_parameter := strings.SplitN(parameter, "=", 2) - var a ListParameter - a.Name = this_parameter[0] - // split on commas, unless they are escaped - a.Value = splitEscapedString(this_parameter[1], ',') - job.PublishedParameters = append(job.PublishedParameters, a) - + job.PublishedParameters[this_parameter[0]] = splitEscapedString(this_parameter[1], ',') } - // get node manager directives - for _, directive := range f.nodeManagerDirective { - this_directive := strings.Split(directive, "=") - var a Directive - a.Name = this_directive[0] - a.Value = this_directive[1] - job.NMDirectives.Directives = append(job.NMDirectives.Directives, a) + job.Directives = make(map[string]string) + for _, directive := range f.directive { + this_directive := strings.SplitN(directive, "=", 2) + job.Directives[this_directive[0]] = this_directive[1] } - if f.ttc != -1 { - job.TMDirectives.Ttc = f.ttc - } - if f.ttl != -1 { - job.TMDirectives.TTL = f.ttl - } + job.SuccessTopics = append(job.SuccessTopics, f.successTopics...) + job.FailureTopics = append(job.FailureTopics, f.failureTopics...) + job.Queue = f.queue + job.Repository = f.repository + job.Workspace = f.workspace - if f.tag != "" { - job.TMDirectives.Tag = f.tag + if f.maxJobRuntime > 0 { + job.MaxJobRuntime = f.maxJobRuntime } - job.TMDirectives.Rtc = f.rtc - - // append slice to slice - job.NMDirectives.SuccessTopics = append(job.NMDirectives.SuccessTopics, f.successTopics...) - job.NMDirectives.FailureTopics = append(job.NMDirectives.FailureTopics, f.failureTopics...) + if f.maxTimeInQueue > 0 { + job.MaxTimeInQueue = f.maxTimeInQueue + } - if f.description != "" { - job.TMDirectives.Description = f.description + if f.wait && f.maxTotalLifeTime > 0 && f.maxTotalLifeTime < 86401 { + job.MaxTotalLifeTime = f.maxTotalLifeTime } jobJson, err := json.Marshal(job) @@ -222,12 +245,12 @@ func runRun(f *runFlags) func(cmd *cobra.Command, args []string) error { return err } - submitEndpoint := "submit" + syncEndpoint := "" if f.wait { - submitEndpoint = "transact" + syncEndpoint = "/sync" } - endpoint := "/fmerest/v3/transformations/" + submitEndpoint + "/" + f.repository + "/" + f.workspace + endpoint := "/fmeapiv4/jobs" + syncEndpoint request, err := buildFmeFlowRequest(endpoint, "POST", strings.NewReader(string(jobJson))) if err != nil { @@ -235,19 +258,12 @@ func runRun(f *runFlags) func(cmd *cobra.Command, args []string) error { } request.Header.Add("Content-Type", "application/json") - response, err := client.Do(&request) if err != nil { return err } else if response.StatusCode != 200 && response.StatusCode != 202 { - if response.StatusCode == 404 { - return fmt.Errorf("%w: check that the specified workspace and repository exist", errors.New(response.Status)) - } else if response.StatusCode == 422 { - return fmt.Errorf("%w: either job failed or published parameters are invalid", errors.New(response.Status)) - } else { - return errors.New(response.Status) - } + return errors.New(response.Status) } responseData, err = io.ReadAll(response.Body) @@ -275,146 +291,325 @@ func runRun(f *runFlags) func(cmd *cobra.Command, args []string) error { return err } } - } else { - // we are uploading a source file, so we want to send the file in the body as octet stream, and parameters as url parameters - file, err := os.Open(f.sourceData) - if err != nil { - return err - } - defer file.Close() - endpoint := "/fmerest/v3/transformations/transactdata/" + f.repository + "/" + f.workspace - request, err := buildFmeFlowRequest(endpoint, "POST", file) - if err != nil { - return err - } + if f.wait { + if f.outputType == "table" { + t := table.NewWriter() + t.SetStyle(defaultStyle) - q := request.URL.Query() + t.AppendHeader(table.Row{"ID", "Status", "Status Message", "Features Output"}) - if f.description != "" { - q.Add("opt_description", f.description) - } + t.AppendRow(table.Row{result.ID, result.Status, result.StatusMessage, result.FeatureOutputCount}) - for _, topic := range f.successTopics { - q.Add("opt_successtopics", topic) - } + if f.noHeaders { + t.ResetHeaders() + } + fmt.Fprintln(cmd.OutOrStdout(), t.Render()) - for _, topic := range f.failureTopics { - q.Add("opt_failuretopics", topic) - } + } else if f.outputType == "json" { + prettyJSON, err := prettyPrintJSON(responseData) + if err != nil { + return err + } + fmt.Fprintln(cmd.OutOrStdout(), prettyJSON) + } else if strings.HasPrefix(f.outputType, "custom-columns") { + // parse the columns and json queries + columnsString := "" + if strings.HasPrefix(f.outputType, "custom-columns=") { + columnsString = f.outputType[len("custom-columns="):] + } + if len(columnsString) == 0 { + return errors.New("custom-columns format specified but no custom columns given") + } - if f.description != "" { - endpoint += "opt_description=" + f.description - } + // we have to marshal the Items array, then create an array of marshalled items + // to pass to the creation of the table. + marshalledItems := [][]byte{} - if f.tag != "" { - q.Add("opt_tag", f.tag) - } + mJson, err := json.Marshal(result) + if err != nil { + return err + } - if f.ttl != -1 { - q.Add("opt_ttl", strconv.Itoa(f.ttl)) - } + marshalledItems = append(marshalledItems, mJson) - if f.ttc != -1 { - q.Add("opt_ttc", strconv.Itoa(f.ttc)) - } + columnsInput := strings.Split(columnsString, ",") + t, err := createTableFromCustomColumns(marshalledItems, columnsInput) + if err != nil { + return err + } + if f.noHeaders { + t.ResetHeaders() + } + fmt.Fprintln(cmd.OutOrStdout(), t.Render()) - for _, parameter := range f.publishedParameter { - this_parameter := strings.SplitN(parameter, "=", 2) - q.Add(this_parameter[0], this_parameter[1]) - } - for _, parameter := range f.listPublishedParameter { - this_parameter := strings.SplitN(parameter, "=", 2) - this_list := splitEscapedString(this_parameter[1], ',') - for _, item := range this_list { - q.Add(this_parameter[0], item) + } else { + return errors.New("invalid output format specified") } } + return nil - request.URL.RawQuery = q.Encode() + } else { - request.Header.Set("Content-Type", "application/octet-stream") + var result JobResultV3 + var responseData []byte - response, err := client.Do(&request) - if err != nil { - return err - } else if response.StatusCode != 200 { - if response.StatusCode == 404 { - return fmt.Errorf("%w: check that the specified workspace and repository exist", errors.New(response.Status)) - } else { - return errors.New(response.Status) + if f.sourceData == "" { + job := &JobRequestV3{} + + // get published parameters + for _, parameter := range f.publishedParameter { + this_parameter := strings.SplitN(parameter, "=", 2) + var a SimpleParameter + a.Name = this_parameter[0] + a.Value = this_parameter[1] + job.PublishedParameters = append(job.PublishedParameters, a) } - } + // get list published parameters + for _, parameter := range f.listPublishedParameter { + this_parameter := strings.SplitN(parameter, "=", 2) + var a ListParameter + a.Name = this_parameter[0] + // split on commas, unless they are escaped + a.Value = splitEscapedString(this_parameter[1], ',') + job.PublishedParameters = append(job.PublishedParameters, a) - responseData, err = io.ReadAll(response.Body) - if err != nil { - return err - } + } - if err := json.Unmarshal(responseData, &result); err != nil { - return err - } - } + // get node manager directives + for _, directive := range f.nodeManagerDirective { + this_directive := strings.Split(directive, "=") + var a Directive + a.Name = this_directive[0] + a.Value = this_directive[1] + job.NMDirectives.Directives = append(job.NMDirectives.Directives, a) + } + + if f.ttc != -1 { + job.TMDirectives.Ttc = f.ttc + } + if f.ttl != -1 { + job.TMDirectives.TTL = f.ttl + } - // the transactdata endpoint only runs synchonously - if f.wait || f.sourceData != "" { - if f.outputType == "table" { - t := table.NewWriter() - t.SetStyle(defaultStyle) + if f.tag != "" { + job.TMDirectives.Tag = f.tag + } - t.AppendHeader(table.Row{"ID", "Status", "Status Message", "Features Output"}) + job.TMDirectives.Rtc = f.rtc - t.AppendRow(table.Row{result.ID, result.Status, result.StatusMessage, result.NumFeaturesOutput}) + // append slice to slice + job.NMDirectives.SuccessTopics = append(job.NMDirectives.SuccessTopics, f.successTopics...) + job.NMDirectives.FailureTopics = append(job.NMDirectives.FailureTopics, f.failureTopics...) - if f.noHeaders { - t.ResetHeaders() + if f.description != "" { + job.TMDirectives.Description = f.description } - fmt.Fprintln(cmd.OutOrStdout(), t.Render()) - } else if f.outputType == "json" { - prettyJSON, err := prettyPrintJSON(responseData) + jobJson, err := json.Marshal(job) if err != nil { return err } - fmt.Fprintln(cmd.OutOrStdout(), prettyJSON) - } else if strings.HasPrefix(f.outputType, "custom-columns") { - // parse the columns and json queries - columnsString := "" - if strings.HasPrefix(f.outputType, "custom-columns=") { - columnsString = f.outputType[len("custom-columns="):] - } - if len(columnsString) == 0 { - return errors.New("custom-columns format specified but no custom columns given") + + submitEndpoint := "submit" + if f.wait { + submitEndpoint = "transact" } - // we have to marshal the Items array, then create an array of marshalled items - // to pass to the creation of the table. - marshalledItems := [][]byte{} + endpoint := "/fmerest/v3/transformations/" + submitEndpoint + "/" + f.repository + "/" + f.workspace - mJson, err := json.Marshal(result) + request, err := buildFmeFlowRequest(endpoint, "POST", strings.NewReader(string(jobJson))) if err != nil { return err } - marshalledItems = append(marshalledItems, mJson) + request.Header.Add("Content-Type", "application/json") + + response, err := client.Do(&request) - columnsInput := strings.Split(columnsString, ",") - t, err := createTableFromCustomColumns(marshalledItems, columnsInput) if err != nil { return err + } else if response.StatusCode != 200 && response.StatusCode != 202 { + if response.StatusCode == 404 { + return fmt.Errorf("%w: check that the specified workspace and repository exist", errors.New(response.Status)) + } else if response.StatusCode == 422 { + return fmt.Errorf("%w: either job failed or published parameters are invalid", errors.New(response.Status)) + } else { + return errors.New(response.Status) + } } - if f.noHeaders { - t.ResetHeaders() + + responseData, err = io.ReadAll(response.Body) + if err != nil { + return err } - fmt.Fprintln(cmd.OutOrStdout(), t.Render()) + if !f.wait { + var result JobId + if err := json.Unmarshal(responseData, &result); err != nil { + return err + } else { + if !jsonOutput { + fmt.Fprintln(cmd.OutOrStdout(), "Job submitted with id: "+strconv.Itoa(result.Id)) + } else { + prettyJSON, err := prettyPrintJSON(responseData) + if err != nil { + return err + } + fmt.Fprintln(cmd.OutOrStdout(), prettyJSON) + } + } + } else { + if err := json.Unmarshal(responseData, &result); err != nil { + return err + } + } } else { - return errors.New("invalid output format specified") + // we are uploading a source file, so we want to send the file in the body as octet stream, and parameters as url parameters + file, err := os.Open(f.sourceData) + if err != nil { + return err + } + defer file.Close() + + endpoint := "/fmerest/v3/transformations/transactdata/" + f.repository + "/" + f.workspace + request, err := buildFmeFlowRequest(endpoint, "POST", file) + if err != nil { + return err + } + + q := request.URL.Query() + + if f.description != "" { + q.Add("opt_description", f.description) + } + + for _, topic := range f.successTopics { + q.Add("opt_successtopics", topic) + } + + for _, topic := range f.failureTopics { + q.Add("opt_failuretopics", topic) + } + + if f.description != "" { + endpoint += "opt_description=" + f.description + } + + if f.tag != "" { + q.Add("opt_tag", f.tag) + } + + if f.ttl != -1 { + q.Add("opt_ttl", strconv.Itoa(f.ttl)) + } + + if f.ttc != -1 { + q.Add("opt_ttc", strconv.Itoa(f.ttc)) + } + + for _, parameter := range f.publishedParameter { + this_parameter := strings.SplitN(parameter, "=", 2) + q.Add(this_parameter[0], this_parameter[1]) + } + for _, parameter := range f.listPublishedParameter { + this_parameter := strings.SplitN(parameter, "=", 2) + this_list := splitEscapedString(this_parameter[1], ',') + for _, item := range this_list { + q.Add(this_parameter[0], item) + } + + } + + request.URL.RawQuery = q.Encode() + + request.Header.Set("Content-Type", "application/octet-stream") + + response, err := client.Do(&request) + if err != nil { + return err + } else if response.StatusCode != 200 { + if response.StatusCode == 404 { + return fmt.Errorf("%w: check that the specified workspace and repository exist", errors.New(response.Status)) + } else { + return errors.New(response.Status) + } + + } + + responseData, err = io.ReadAll(response.Body) + if err != nil { + return err + } + + if err := json.Unmarshal(responseData, &result); err != nil { + return err + } + } + + // the transactdata endpoint only runs synchonously + if f.wait || f.sourceData != "" { + if f.outputType == "table" { + t := table.NewWriter() + t.SetStyle(defaultStyle) + + t.AppendHeader(table.Row{"ID", "Status", "Status Message", "Features Output"}) + + t.AppendRow(table.Row{result.ID, result.Status, result.StatusMessage, result.NumFeaturesOutput}) + + if f.noHeaders { + t.ResetHeaders() + } + fmt.Fprintln(cmd.OutOrStdout(), t.Render()) + + } else if f.outputType == "json" { + prettyJSON, err := prettyPrintJSON(responseData) + if err != nil { + return err + } + fmt.Fprintln(cmd.OutOrStdout(), prettyJSON) + } else if strings.HasPrefix(f.outputType, "custom-columns") { + // parse the columns and json queries + columnsString := "" + if strings.HasPrefix(f.outputType, "custom-columns=") { + columnsString = f.outputType[len("custom-columns="):] + } + if len(columnsString) == 0 { + return errors.New("custom-columns format specified but no custom columns given") + } + + // we have to marshal the Items array, then create an array of marshalled items + // to pass to the creation of the table. + marshalledItems := [][]byte{} + + mJson, err := json.Marshal(result) + if err != nil { + return err + } + + marshalledItems = append(marshalledItems, mJson) + + columnsInput := strings.Split(columnsString, ",") + t, err := createTableFromCustomColumns(marshalledItems, columnsInput) + if err != nil { + return err + } + if f.noHeaders { + t.ResetHeaders() + } + fmt.Fprintln(cmd.OutOrStdout(), t.Render()) + + } else { + return errors.New("invalid output format specified") + } } + return nil } + return nil + } } diff --git a/cmd/run_v3_test.go b/cmd/run_v3_test.go new file mode 100644 index 0000000..bb08e22 --- /dev/null +++ b/cmd/run_v3_test.go @@ -0,0 +1,247 @@ +package cmd + +import ( + "net/http" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRun(t *testing.T) { + responseV3ASync := `{ + "id": 1 + }` + + responseV3Sync := `{ + "timeRequested": "2023-02-04T00:16:28Z", + "requesterResultPort": 37805, + "numFeaturesOutput": 1539, + "requesterHost": "10.1.113.39", + "timeStarted": "2023-02-04T00:16:28Z", + "id": 1, + "timeFinished": "2023-02-04T00:16:30Z", + "priority": -1, + "statusMessage": "Translation Successful", + "status": "SUCCESS" + }` + + dataFileContents := "Pretend backup file" + + // generate random file to restore from + f, err := os.CreateTemp("", "datafile") + require.NoError(t, err) + defer os.Remove(f.Name()) // clean up + err = os.WriteFile(f.Name(), []byte(dataFileContents), 0644) + require.NoError(t, err) + + cases := []testCase{ + { + name: "unknown flag", + statusCode: http.StatusOK, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--badflag"}, + wantErrOutputRegex: "unknown flag: --badflag", + }, + { + name: "500 bad status code", + statusCode: http.StatusInternalServerError, + wantErrText: "500 Internal Server Error", + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw"}, + }, + { + name: "repository flag required", + wantErrText: "required flag(s) \"repository\" not set", + args: []string{"run", "--workspace", "austinApartments.fmw"}, + }, + { + name: "workspace flag required", + wantErrText: "required flag(s) \"workspace\" not set", + args: []string{"run", "--repository", "Samples"}, + }, + { + name: "run sync job table output", + statusCode: http.StatusOK, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--wait"}, + body: responseV3Sync, + wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", + }, + { + name: "run async job regular output", + statusCode: http.StatusOK, + body: responseV3ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + }, + { + name: "run async job json", + statusCode: http.StatusOK, + body: responseV3ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--json"}, + wantOutputJson: responseV3ASync, + }, + { + name: "run sync job json output", + statusCode: http.StatusOK, + body: responseV3Sync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--json", "--wait"}, + wantOutputJson: responseV3Sync, + }, + { + name: "description flag async", + statusCode: http.StatusOK, + body: responseV3ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--description", "My Description"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + wantBodyRegEx: ".*\"TMDirectives\".*:[\\s]*{.*\"description\":\"My Description\".*", + }, + { + name: "failure topic flag async", + statusCode: http.StatusOK, + body: responseV3ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--failure-topic", "FAILURE_TOPIC"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + wantBodyRegEx: ".*\"NMDirectives\".*:[\\s]*{.*\"failureTopics\":\\[\"FAILURE_TOPIC\"\\].*", + }, + { + name: "success topic flag async", + statusCode: http.StatusOK, + body: responseV3ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--success-topic", "SUCCESS_TOPIC"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + wantBodyRegEx: ".*\"NMDirectives\".*:[\\s]*{.*\"successTopics\":\\[\"SUCCESS_TOPIC\"\\].*", + }, + { + name: "node manager directive flag async", + statusCode: http.StatusOK, + body: responseV3ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--node-manager-directive", "directive1=value1", "--node-manager-directive", "directive2=value2"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + wantBodyRegEx: ".*\"NMDirectives\".*:[\\s]*{.*\"directives\":\\[{\"name\":\"directive1\",\"value\":\"value1\"},{\"name\":\"directive2\",\"value\":\"value2\".*", + }, + { + name: "run until canceled flag async", + statusCode: http.StatusOK, + body: responseV3ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--run-until-canceled"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + wantBodyRegEx: ".*\"TMDirectives\":{\"rtc\":true}.*", + }, + { + name: "tag flag async", + statusCode: http.StatusOK, + body: responseV3ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--tag", "myqueue"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + wantBodyRegEx: ".*\"TMDirectives\":{.*\"tag\":\"myqueue\".*}.*", + }, + { + name: "time to live flag async", + statusCode: http.StatusOK, + body: responseV3ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--time-to-live", "60"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + wantBodyRegEx: ".*\"TMDirectives\":{.*\"ttl\":60.*}.*", + }, + { + name: "timeuntil canceled flag async", + statusCode: http.StatusOK, + body: responseV3ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--time-until-canceled", "60"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + wantBodyRegEx: ".*\"TMDirectives\":{.*\"ttc\":60.*}.*", + }, + { + name: "published parameter async", + statusCode: http.StatusOK, + body: responseV3ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--published-parameter", "COORDSYS=TX83-CF"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + wantBodyRegEx: ".*\"publishedParameters\":\\[{\"value\":\"TX83-CF\",\"name\":\"COORDSYS\".*}.*", + }, + { + name: "published parameter list async", + statusCode: http.StatusOK, + body: responseV3ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--published-parameter-list", "THEMES=railroad,airports"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + wantBodyRegEx: ".*\"publishedParameters\":\\[{\"value\":\\[\"railroad\",\"airports\"],\"name\":\"THEMES\".*}.*", + }, + + { + name: "description flag transact data", + statusCode: http.StatusOK, + body: responseV3Sync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--description", "My Description", "--file", f.Name()}, + wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", + wantFormParams: map[string]string{"opt_description": "My Description"}, + }, + { + name: "failure topic flag transact data", + statusCode: http.StatusOK, + body: responseV3Sync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--failure-topic", "FAILURE_TOPIC", "--file", f.Name()}, + wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", + wantFormParams: map[string]string{"opt_failuretopics": "FAILURE_TOPIC"}, + }, + { + name: "success topic flag transact data", + statusCode: http.StatusOK, + body: responseV3Sync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--success-topic", "SUCCESS_TOPIC", "--file", f.Name()}, + wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", + wantFormParams: map[string]string{"opt_successtopics": "SUCCESS_TOPIC"}, + }, + { + name: "tag flag transact data", + statusCode: http.StatusOK, + body: responseV3Sync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--tag", "myqueue", "--file", f.Name()}, + wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", + wantFormParams: map[string]string{"opt_tag": "myqueue"}, + }, + { + name: "time to live flag transact data", + statusCode: http.StatusOK, + body: responseV3Sync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--time-to-live", "60", "--file", f.Name()}, + wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", + wantFormParams: map[string]string{"opt_ttl": "60"}, + }, + { + name: "timeuntil canceled flag transact data", + statusCode: http.StatusOK, + body: responseV3Sync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--time-until-canceled", "60", "--file", f.Name()}, + wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", + wantFormParams: map[string]string{"opt_ttc": "60"}, + }, + { + name: "published parameter transact data", + statusCode: http.StatusOK, + body: responseV3Sync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--published-parameter", "COORDSYS=TX83-CF", "--file", f.Name()}, + wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", + wantFormParams: map[string]string{"COORDSYS": "TX83-CF"}, + }, + { + name: "published parameter list transact data", + statusCode: http.StatusOK, + body: responseV3Sync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--published-parameter-list", "THEMES=railroad,airports", "--file", f.Name()}, + wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", + wantFormParamsList: map[string][]string{"THEMES": {"railroad", "airports"}}, + }, + { + name: "transact data node manager mutually exclusive", + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--node-manager-directive", "directive1=value1", "--file", f.Name()}, + wantErrText: "if any flags in the group [file node-manager-directive] are set none of the others can be; [file node-manager-directive] were all set", + }, + { + name: "transact data run until canceled mutually exclusive", + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--run-until-canceled", "--file", f.Name()}, + wantErrText: "if any flags in the group [file run-until-canceled] are set none of the others can be; [file run-until-canceled] were all set", + }, + } + runTests(cases, t) + +} diff --git a/cmd/run_v4_test.go b/cmd/run_v4_test.go new file mode 100644 index 0000000..7f83ad4 --- /dev/null +++ b/cmd/run_v4_test.go @@ -0,0 +1,197 @@ +package cmd + +import ( + "net/http" + "testing" +) + +func TestRunV4(t *testing.T) { + responseV4ASync := `{ + "id": 1 + }` + + responseV4Sync := `{ + "id": 1, + "featureOutputCount": 1539, + "requesterHost": "10.1.113.39", + "requesterResultPort": 37805, + "status": "SUCCESS", + "statusMessage": "Translation Successful", + "timeFinished": "2023-02-04T00:16:30Z", + "timeQueued": "2023-02-04T00:16:28Z", + "timeStarted": "2023-02-04T00:16:28Z" + }` + + cases := []testCase{ + { + name: "unknown flag", + statusCode: http.StatusOK, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--badflag"}, + wantErrOutputRegex: "unknown flag: --badflag", + }, + { + name: "500 bad status code", + statusCode: http.StatusInternalServerError, + wantErrText: "500 Internal Server Error", + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw"}, + fmeflowBuild: 26018, + }, + { + name: "repository flag required", + wantErrText: "required flag(s) \"repository\" not set", + args: []string{"run", "--workspace", "austinApartments.fmw"}, + }, + { + name: "workspace flag required", + wantErrText: "required flag(s) \"workspace\" not set", + args: []string{"run", "--repository", "Samples"}, + }, + { + name: "run sync job table output", + statusCode: http.StatusOK, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--wait"}, + body: responseV4Sync, + wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", + fmeflowBuild: 26018, + }, + { + name: "run async job regular output", + statusCode: http.StatusOK, + body: responseV4ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + fmeflowBuild: 26018, + }, + { + name: "run async job json", + statusCode: http.StatusOK, + body: responseV4ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--json"}, + wantOutputJson: responseV4ASync, + fmeflowBuild: 26018, + }, + { + name: "run sync job json output", + statusCode: http.StatusOK, + body: responseV4Sync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--json", "--wait"}, + wantOutputJson: responseV4Sync, + fmeflowBuild: 26018, + }, + { + name: "failure topic flag async", + statusCode: http.StatusOK, + body: responseV4ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--failure-topic", "FAILURE_TOPIC"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + wantBodyRegEx: ".*\"failureTopics\":\\[\"FAILURE_TOPIC\"\\].*", + fmeflowBuild: 26018, + }, + { + name: "success topic flag async", + statusCode: http.StatusOK, + body: responseV4ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--success-topic", "SUCCESS_TOPIC"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + wantBodyRegEx: ".*\"successTopics\":\\[\"SUCCESS_TOPIC\"\\].*", + fmeflowBuild: 26018, + }, + { + name: "directive flag async", + statusCode: http.StatusOK, + body: responseV4ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--directive", "directive1=value1", "--directive", "directive2=value2"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + wantBodyRegEx: ".*\"directives\":{.*\"directive1\":\"value1\".*\"directive2\":\"value2\".*}.*", + fmeflowBuild: 26018, + }, + { + name: "published parameter async", + statusCode: http.StatusOK, + body: responseV4ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--published-parameter", "COORDSYS=TX83-CF"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + wantBodyRegEx: ".*\"publishedParameters\":{.*\"COORDSYS\":\"TX83-CF\".*}.*", + fmeflowBuild: 26018, + }, + { + name: "published parameter list async", + statusCode: http.StatusOK, + body: responseV4ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--published-parameter-list", "THEMES=railroad,airports"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + wantBodyRegEx: ".*\"publishedParameters\":{.*\"THEMES\":\\[\"railroad\",\"airports\"\\].*}.*", + fmeflowBuild: 26018, + }, + { + name: "max job runtime flag async", + statusCode: http.StatusOK, + body: responseV4ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--max-job-runtime", "10"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + wantBodyRegEx: ".*\"maxJobRuntime\":10.*", + fmeflowBuild: 26018, + }, + { + name: "max job runtime invalid value ignored", + statusCode: http.StatusOK, + body: responseV4ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--max-job-runtime", "-5"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + fmeflowBuild: 26018, + }, + { + name: "max time in queue flag async", + statusCode: http.StatusOK, + body: responseV4ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--max-time-in-queue", "60"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + wantBodyRegEx: ".*\"maxTimeInQueue\":60.*", + fmeflowBuild: 26018, + }, + { + name: "max time in queue invalid value ignored", + statusCode: http.StatusOK, + body: responseV4ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--max-time-in-queue", "-1"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + fmeflowBuild: 26018, + }, + { + name: "max total life time flag sync", + statusCode: http.StatusOK, + body: responseV4Sync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--wait", "--max-total-life-time", "300"}, + wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", + wantBodyRegEx: ".*\"maxTotalLifeTime\":300.*", + fmeflowBuild: 26018, + }, + { + name: "max total life time invalid value ignored", + statusCode: http.StatusOK, + body: responseV4Sync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--wait", "--max-total-life-time", "100000"}, + wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", + fmeflowBuild: 26018, + }, + { + name: "queue flag async", + statusCode: http.StatusOK, + body: responseV4ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--queue", "MyQueue"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + wantBodyRegEx: ".*\"queue\":\"MyQueue\".*", + fmeflowBuild: 26018, + }, + { + name: "published parameter and list combined", + statusCode: http.StatusOK, + body: responseV4ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--published-parameter", "COORDSYS=TX83-CF", "--published-parameter-list", "THEMES=railroad,airports"}, + wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", + wantBodyRegEx: ".*\"publishedParameters\":{.*\"COORDSYS\":\"TX83-CF\".*\"THEMES\":\\[\"railroad\",\"airports\"\\].*}.*", + fmeflowBuild: 26018, + }, + } + runTests(cases, t) +} From 7c857adecfbb5a48be48e38934896cbb8e96751b Mon Sep 17 00:00:00 2001 From: Amber Xu Date: Mon, 10 Nov 2025 14:52:49 -0800 Subject: [PATCH 2/6] updated doc --- docs/fmeflow_run.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/fmeflow_run.md b/docs/fmeflow_run.md index 4b16cbc..9348222 100644 --- a/docs/fmeflow_run.md +++ b/docs/fmeflow_run.md @@ -39,17 +39,22 @@ fmeflow run [flags] --repository string The name of the repository containing the workspace to run. --workspace string The name of the workspace to run. --wait Submit job and wait for it to finish. - --tag string The job routing tag for the request + --tag string The job routing tag for the request. For v3 API only. --published-parameter stringArray Published parameters defined for this workspace. Specify as Key=Value. Can be passed in multiple times. For list parameters, use the --list-published-parameter flag. --published-parameter-list stringArray A List-type published parameters defined for this workspace. Specify as Key=Value1,Value2. Can be passed in multiple times. - --file string Upload a local file Source dataset to use to run the workspace. Note this causes the translation to run in synchonous mode whether the --wait flag is passed in or not. - --run-until-canceled Runs a job until it is explicitly canceled. The job will run again regardless of whether the job completed successfully, failed, or the server crashed or was shut down. - --time-until-canceled int Time (in seconds) elapsed for a running job before it's cancelled. The minimum value is 1 second, values less than 1 second are ignored. (default -1) - --time-to-live int Time to live in the job queue (in seconds) (default -1) - --description string Description of the request. + --file string Upload a local file Source dataset to use to run the workspace. Note this causes the translation to run in synchonous mode whether the --wait flag is passed in or not. For v3 API only. + --run-until-canceled Runs a job until it is explicitly canceled. The job will run again regardless of whether the job completed successfully, failed, or the server crashed or was shut down. For v3 API only. + --time-until-canceled int Time (in seconds) elapsed for a running job before it's cancelled. The minimum value is 1 second, values less than 1 second are ignored. For v3 API only. (default -1) + --time-to-live int Time to live in the job queue (in seconds). For v3 API only. (default -1) + --description string Description of the request. For v3 API only. --success-topic stringArray Topics to notify when the job succeeds. Can be specified more than once. --failure-topic stringArray Topics to notify when the job fails. Can be specified more than once. - --node-manager-directive stringArray Additional NM Directives, which can include client-configured keys, to pass to the notification service for custom use by subscriptions. Specify as Key=Value Can be passed in multiple times. + --node-manager-directive stringArray Additional NM Directives, which can include client-configured keys, to pass to the notification service for custom use by subscriptions. Specify as Key=Value Can be passed in multiple times. For v3 API only. + --directive stringArray Additional directives to pass to the job submission. Specify as Key=Value. Can be passed in multiple times. For v4 API only. + --queue string Queue of the job to submit. For v4 API only. + --max-job-runtime int Time (in seconds) elapsed for a running job before it's cancelled. The minimum value is 1 second, values less than 1 second are ignored. For v4 API only. + --max-time-in-queue int Time to live in the job queue (in seconds). For v4 API only. + --max-total-life-time int Time to live including both time in the queue and run time (in seconds). The maximum value is 86400 and the minimum value is 1. For v4 API only. -o, --output string Specify the output type. Should be one of table, json, or custom-columns (default "table") --no-headers Don't print column headers -h, --help help for run From 29ff9cabd834cd2502757da14d9315359f444713 Mon Sep 17 00:00:00 2001 From: Amber Xu Date: Mon, 10 Nov 2025 14:54:09 -0800 Subject: [PATCH 3/6] renamed original run test to run v3 --- cmd/run_test.go | 247 ------------------------------------------------ 1 file changed, 247 deletions(-) delete mode 100644 cmd/run_test.go diff --git a/cmd/run_test.go b/cmd/run_test.go deleted file mode 100644 index bb08e22..0000000 --- a/cmd/run_test.go +++ /dev/null @@ -1,247 +0,0 @@ -package cmd - -import ( - "net/http" - "os" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestRun(t *testing.T) { - responseV3ASync := `{ - "id": 1 - }` - - responseV3Sync := `{ - "timeRequested": "2023-02-04T00:16:28Z", - "requesterResultPort": 37805, - "numFeaturesOutput": 1539, - "requesterHost": "10.1.113.39", - "timeStarted": "2023-02-04T00:16:28Z", - "id": 1, - "timeFinished": "2023-02-04T00:16:30Z", - "priority": -1, - "statusMessage": "Translation Successful", - "status": "SUCCESS" - }` - - dataFileContents := "Pretend backup file" - - // generate random file to restore from - f, err := os.CreateTemp("", "datafile") - require.NoError(t, err) - defer os.Remove(f.Name()) // clean up - err = os.WriteFile(f.Name(), []byte(dataFileContents), 0644) - require.NoError(t, err) - - cases := []testCase{ - { - name: "unknown flag", - statusCode: http.StatusOK, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--badflag"}, - wantErrOutputRegex: "unknown flag: --badflag", - }, - { - name: "500 bad status code", - statusCode: http.StatusInternalServerError, - wantErrText: "500 Internal Server Error", - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw"}, - }, - { - name: "repository flag required", - wantErrText: "required flag(s) \"repository\" not set", - args: []string{"run", "--workspace", "austinApartments.fmw"}, - }, - { - name: "workspace flag required", - wantErrText: "required flag(s) \"workspace\" not set", - args: []string{"run", "--repository", "Samples"}, - }, - { - name: "run sync job table output", - statusCode: http.StatusOK, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--wait"}, - body: responseV3Sync, - wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", - }, - { - name: "run async job regular output", - statusCode: http.StatusOK, - body: responseV3ASync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw"}, - wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", - }, - { - name: "run async job json", - statusCode: http.StatusOK, - body: responseV3ASync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--json"}, - wantOutputJson: responseV3ASync, - }, - { - name: "run sync job json output", - statusCode: http.StatusOK, - body: responseV3Sync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--json", "--wait"}, - wantOutputJson: responseV3Sync, - }, - { - name: "description flag async", - statusCode: http.StatusOK, - body: responseV3ASync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--description", "My Description"}, - wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", - wantBodyRegEx: ".*\"TMDirectives\".*:[\\s]*{.*\"description\":\"My Description\".*", - }, - { - name: "failure topic flag async", - statusCode: http.StatusOK, - body: responseV3ASync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--failure-topic", "FAILURE_TOPIC"}, - wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", - wantBodyRegEx: ".*\"NMDirectives\".*:[\\s]*{.*\"failureTopics\":\\[\"FAILURE_TOPIC\"\\].*", - }, - { - name: "success topic flag async", - statusCode: http.StatusOK, - body: responseV3ASync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--success-topic", "SUCCESS_TOPIC"}, - wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", - wantBodyRegEx: ".*\"NMDirectives\".*:[\\s]*{.*\"successTopics\":\\[\"SUCCESS_TOPIC\"\\].*", - }, - { - name: "node manager directive flag async", - statusCode: http.StatusOK, - body: responseV3ASync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--node-manager-directive", "directive1=value1", "--node-manager-directive", "directive2=value2"}, - wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", - wantBodyRegEx: ".*\"NMDirectives\".*:[\\s]*{.*\"directives\":\\[{\"name\":\"directive1\",\"value\":\"value1\"},{\"name\":\"directive2\",\"value\":\"value2\".*", - }, - { - name: "run until canceled flag async", - statusCode: http.StatusOK, - body: responseV3ASync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--run-until-canceled"}, - wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", - wantBodyRegEx: ".*\"TMDirectives\":{\"rtc\":true}.*", - }, - { - name: "tag flag async", - statusCode: http.StatusOK, - body: responseV3ASync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--tag", "myqueue"}, - wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", - wantBodyRegEx: ".*\"TMDirectives\":{.*\"tag\":\"myqueue\".*}.*", - }, - { - name: "time to live flag async", - statusCode: http.StatusOK, - body: responseV3ASync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--time-to-live", "60"}, - wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", - wantBodyRegEx: ".*\"TMDirectives\":{.*\"ttl\":60.*}.*", - }, - { - name: "timeuntil canceled flag async", - statusCode: http.StatusOK, - body: responseV3ASync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--time-until-canceled", "60"}, - wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", - wantBodyRegEx: ".*\"TMDirectives\":{.*\"ttc\":60.*}.*", - }, - { - name: "published parameter async", - statusCode: http.StatusOK, - body: responseV3ASync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--published-parameter", "COORDSYS=TX83-CF"}, - wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", - wantBodyRegEx: ".*\"publishedParameters\":\\[{\"value\":\"TX83-CF\",\"name\":\"COORDSYS\".*}.*", - }, - { - name: "published parameter list async", - statusCode: http.StatusOK, - body: responseV3ASync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--published-parameter-list", "THEMES=railroad,airports"}, - wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", - wantBodyRegEx: ".*\"publishedParameters\":\\[{\"value\":\\[\"railroad\",\"airports\"],\"name\":\"THEMES\".*}.*", - }, - - { - name: "description flag transact data", - statusCode: http.StatusOK, - body: responseV3Sync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--description", "My Description", "--file", f.Name()}, - wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", - wantFormParams: map[string]string{"opt_description": "My Description"}, - }, - { - name: "failure topic flag transact data", - statusCode: http.StatusOK, - body: responseV3Sync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--failure-topic", "FAILURE_TOPIC", "--file", f.Name()}, - wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", - wantFormParams: map[string]string{"opt_failuretopics": "FAILURE_TOPIC"}, - }, - { - name: "success topic flag transact data", - statusCode: http.StatusOK, - body: responseV3Sync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--success-topic", "SUCCESS_TOPIC", "--file", f.Name()}, - wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", - wantFormParams: map[string]string{"opt_successtopics": "SUCCESS_TOPIC"}, - }, - { - name: "tag flag transact data", - statusCode: http.StatusOK, - body: responseV3Sync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--tag", "myqueue", "--file", f.Name()}, - wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", - wantFormParams: map[string]string{"opt_tag": "myqueue"}, - }, - { - name: "time to live flag transact data", - statusCode: http.StatusOK, - body: responseV3Sync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--time-to-live", "60", "--file", f.Name()}, - wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", - wantFormParams: map[string]string{"opt_ttl": "60"}, - }, - { - name: "timeuntil canceled flag transact data", - statusCode: http.StatusOK, - body: responseV3Sync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--time-until-canceled", "60", "--file", f.Name()}, - wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", - wantFormParams: map[string]string{"opt_ttc": "60"}, - }, - { - name: "published parameter transact data", - statusCode: http.StatusOK, - body: responseV3Sync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--published-parameter", "COORDSYS=TX83-CF", "--file", f.Name()}, - wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", - wantFormParams: map[string]string{"COORDSYS": "TX83-CF"}, - }, - { - name: "published parameter list transact data", - statusCode: http.StatusOK, - body: responseV3Sync, - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--published-parameter-list", "THEMES=railroad,airports", "--file", f.Name()}, - wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", - wantFormParamsList: map[string][]string{"THEMES": {"railroad", "airports"}}, - }, - { - name: "transact data node manager mutually exclusive", - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--node-manager-directive", "directive1=value1", "--file", f.Name()}, - wantErrText: "if any flags in the group [file node-manager-directive] are set none of the others can be; [file node-manager-directive] were all set", - }, - { - name: "transact data run until canceled mutually exclusive", - args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--run-until-canceled", "--file", f.Name()}, - wantErrText: "if any flags in the group [file run-until-canceled] are set none of the others can be; [file run-until-canceled] were all set", - }, - } - runTests(cases, t) - -} From 794e882a96405059307e476a310fe4fd5b09709f Mon Sep 17 00:00:00 2001 From: Amber Xu Date: Mon, 10 Nov 2025 14:57:22 -0800 Subject: [PATCH 4/6] Updated struct name in jobs --- cmd/jobs.go | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/cmd/jobs.go b/cmd/jobs.go index 658984b..4a2f548 100644 --- a/cmd/jobs.go +++ b/cmd/jobs.go @@ -41,28 +41,28 @@ type JobStatusV4 struct { } type JobStatusV3 struct { - Request JobRequest `json:"request"` - TimeDelivered time.Time `json:"timeDelivered"` - Workspace string `json:"workspace"` - NumErrors int `json:"numErrors"` - NumLines int `json:"numLines"` - EngineHost string `json:"engineHost"` - TimeQueued time.Time `json:"timeQueued"` - CPUPct float64 `json:"cpuPct"` - Description string `json:"description"` - TimeStarted time.Time `json:"timeStarted"` - Repository string `json:"repository"` - UserName string `json:"userName"` - Result JobResult `json:"result"` - CPUTime int `json:"cpuTime"` - ID int `json:"id"` - TimeFinished time.Time `json:"timeFinished"` - EngineName string `json:"engineName"` - NumWarnings int `json:"numWarnings"` - TimeSubmitted time.Time `json:"timeSubmitted"` - ElapsedTime int `json:"elapsedTime"` - PeakMemUsage int `json:"peakMemUsage"` - Status string `json:"status"` + Request JobRequestV3 `json:"request"` + TimeDelivered time.Time `json:"timeDelivered"` + Workspace string `json:"workspace"` + NumErrors int `json:"numErrors"` + NumLines int `json:"numLines"` + EngineHost string `json:"engineHost"` + TimeQueued time.Time `json:"timeQueued"` + CPUPct float64 `json:"cpuPct"` + Description string `json:"description"` + TimeStarted time.Time `json:"timeStarted"` + Repository string `json:"repository"` + UserName string `json:"userName"` + Result JobResultV3 `json:"result"` + CPUTime int `json:"cpuTime"` + ID int `json:"id"` + TimeFinished time.Time `json:"timeFinished"` + EngineName string `json:"engineName"` + NumWarnings int `json:"numWarnings"` + TimeSubmitted time.Time `json:"timeSubmitted"` + ElapsedTime int `json:"elapsedTime"` + PeakMemUsage int `json:"peakMemUsage"` + Status string `json:"status"` } type JobsV4 struct { From ce5d976c129e932326923342bf8eb48247b72715 Mon Sep 17 00:00:00 2001 From: Amber Xu Date: Wed, 12 Nov 2025 23:30:51 -0800 Subject: [PATCH 5/6] Simplified reworded flags in version change --- cmd/run.go | 53 +++++++++++++++++++++++++-------------------- cmd/run_v3_test.go | 33 +++++++++++++++++++++++++--- cmd/run_v4_test.go | 27 +++++++++++++++++++++++ docs/fmeflow_run.md | 13 +++++------ 4 files changed, 92 insertions(+), 34 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index 6ca843c..5d8c64a 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -102,9 +102,6 @@ type runFlags struct { repository string wait bool rtc bool - ttc int - ttl int - tag string description string sourceData string successTopics []string @@ -135,7 +132,7 @@ func newRunCmd() *cobra.Command { fmeflow run --repository Samples --workspace austinApartments.fmw --wait # Submit a job to a specific queue and set a time to live in the queue - fmeflow run --repository Samples --workspace austinApartments.fmw --tag Queue1 --time-to-live 120 + fmeflow run --repository Samples --workspace austinApartments.fmw --queue Queue1 --max-time-in-queue 120 # Submit a job and pass in a few published parameters fmeflow run --repository Samples --workspace austinDownload.fmw --published-parameter-list THEMES=railroad,airports --published-parameter COORDSYS=TX83-CF @@ -152,22 +149,22 @@ func newRunCmd() *cobra.Command { cmd.Flags().StringVar(&f.repository, "repository", "", "The name of the repository containing the workspace to run.") cmd.Flags().StringVar(&f.workspace, "workspace", "", "The name of the workspace to run.") cmd.Flags().BoolVar(&f.wait, "wait", false, "Submit job and wait for it to finish.") - cmd.Flags().StringVar(&f.tag, "tag", "", "The job routing tag for the request. For v3 API only.") cmd.Flags().StringArrayVar(&f.publishedParameter, "published-parameter", []string{}, "Published parameters defined for this workspace. Specify as Key=Value. Can be passed in multiple times. For list parameters, use the --list-published-parameter flag.") cmd.Flags().StringArrayVar(&f.listPublishedParameter, "published-parameter-list", []string{}, "A List-type published parameters defined for this workspace. Specify as Key=Value1,Value2. Can be passed in multiple times.") cmd.Flags().StringVar(&f.sourceData, "file", "", "Upload a local file Source dataset to use to run the workspace. Note this causes the translation to run in synchonous mode whether the --wait flag is passed in or not. For v3 API only.") cmd.Flags().BoolVar(&f.rtc, "run-until-canceled", false, "Runs a job until it is explicitly canceled. The job will run again regardless of whether the job completed successfully, failed, or the server crashed or was shut down. For v3 API only.") - cmd.Flags().IntVar(&f.ttc, "time-until-canceled", -1, "Time (in seconds) elapsed for a running job before it's cancelled. The minimum value is 1 second, values less than 1 second are ignored. For v3 API only.") - cmd.Flags().IntVar(&f.ttl, "time-to-live", -1, "Time to live in the job queue (in seconds). For v3 API only.") cmd.Flags().StringVar(&f.description, "description", "", "Description of the request. For v3 API only.") cmd.Flags().StringArrayVar(&f.successTopics, "success-topic", []string{}, "Topics to notify when the job succeeds. Can be specified more than once.") cmd.Flags().StringArrayVar(&f.failureTopics, "failure-topic", []string{}, "Topics to notify when the job fails. Can be specified more than once.") cmd.Flags().StringArrayVar(&f.nodeManagerDirective, "node-manager-directive", []string{}, "Additional NM Directives, which can include client-configured keys, to pass to the notification service for custom use by subscriptions. Specify as Key=Value Can be passed in multiple times. For v3 API only.") cmd.Flags().StringArrayVar(&f.directive, "directive", []string{}, "Additional directives to pass to the job submission. Specify as Key=Value. Can be passed in multiple times. For v4 API only.") - cmd.Flags().StringVar(&f.queue, "queue", "", "Queue of the job to submit. For v4 API only.") - cmd.Flags().IntVar(&f.maxJobRuntime, "max-job-runtime", 0, "Time (in seconds) elapsed for a running job before it's cancelled. The minimum value is 1 second, values less than 1 second are ignored. For v4 API only.") - cmd.Flags().IntVar(&f.maxTimeInQueue, "max-time-in-queue", 0, "Time to live in the job queue (in seconds). For v4 API only.") - cmd.Flags().IntVar(&f.maxTotalLifeTime, "max-total-life-time", 0, "Time to live including both time in the queue and run time (in seconds). The maximum value is 86400 and the minimum value is 1. For v4 API only.") + cmd.Flags().StringVar(&f.queue, "queue", "", "Queue of the job to submit.") + cmd.Flags().StringVar(&f.queue, "tag", "", "The queue (job routing tag) for the request.") + cmd.Flags().IntVar(&f.maxJobRuntime, "max-job-runtime", -1, "Time (in seconds) elapsed for a running job before it's cancelled. The minimum value is 1 second, values less than 1 second are ignored.") + cmd.Flags().IntVar(&f.maxJobRuntime, "time-until-canceled", -1, "Time (in seconds) elapsed for a running job before it's cancelled. The minimum value is 1 second, values less than 1 second are ignored.") + cmd.Flags().IntVar(&f.maxTimeInQueue, "max-time-in-queue", -1, "Time to live in the job queue (in seconds).") + cmd.Flags().IntVar(&f.maxTimeInQueue, "time-to-live", -1, "Time to live in the job queue (in seconds).") + cmd.Flags().IntVar(&f.maxTotalLifeTime, "max-total-life-time", -1, "Time to live including both time in the queue and run time (in seconds). The maximum value is 86400 and the minimum value is 1. For v4 API only.") cmd.Flags().StringVarP(&f.outputType, "output", "o", "table", "Specify the output type. Should be one of table, json, or custom-columns") cmd.Flags().BoolVar(&f.noHeaders, "no-headers", false, "Don't print column headers") @@ -181,6 +178,16 @@ func newRunCmd() *cobra.Command { cmd.MarkFlagsMutuallyExclusive("file", "node-manager-directive") cmd.MarkFlagsMutuallyExclusive("file", "run-until-canceled") + // deprecated flags can't be used with the equavalent new flags + cmd.MarkFlagsMutuallyExclusive("tag", "queue") + cmd.MarkFlagsMutuallyExclusive("time-until-canceled", "max-job-runtime") + cmd.MarkFlagsMutuallyExclusive("time-to-live", "max-time-in-queue") + + // mark v3 deprecated flags + cmd.Flags().MarkDeprecated("tag", "please use --queue instead") + cmd.Flags().MarkDeprecated("time-until-canceled", "please use --max-job-runtime instead") + cmd.Flags().MarkDeprecated("time-to-live", "please use --max-time-in-queue instead") + return cmd } @@ -387,15 +394,15 @@ func runRun(f *runFlags) func(cmd *cobra.Command, args []string) error { job.NMDirectives.Directives = append(job.NMDirectives.Directives, a) } - if f.ttc != -1 { - job.TMDirectives.Ttc = f.ttc + if f.maxJobRuntime != -1 { + job.TMDirectives.Ttc = f.maxJobRuntime } - if f.ttl != -1 { - job.TMDirectives.TTL = f.ttl + if f.maxTimeInQueue != -1 { + job.TMDirectives.TTL = f.maxTimeInQueue } - if f.tag != "" { - job.TMDirectives.Tag = f.tag + if f.queue != "" { + job.TMDirectives.Tag = f.queue } job.TMDirectives.Rtc = f.rtc @@ -498,16 +505,16 @@ func runRun(f *runFlags) func(cmd *cobra.Command, args []string) error { endpoint += "opt_description=" + f.description } - if f.tag != "" { - q.Add("opt_tag", f.tag) + if f.queue != "" { + q.Add("opt_tag", f.queue) } - if f.ttl != -1 { - q.Add("opt_ttl", strconv.Itoa(f.ttl)) + if f.maxTimeInQueue != -1 { + q.Add("opt_ttl", strconv.Itoa(f.maxTimeInQueue)) } - if f.ttc != -1 { - q.Add("opt_ttc", strconv.Itoa(f.ttc)) + if f.maxJobRuntime != -1 { + q.Add("opt_ttc", strconv.Itoa(f.maxJobRuntime)) } for _, parameter := range f.publishedParameter { diff --git a/cmd/run_v3_test.go b/cmd/run_v3_test.go index bb08e22..32fdd4f 100644 --- a/cmd/run_v3_test.go +++ b/cmd/run_v3_test.go @@ -131,25 +131,52 @@ func TestRun(t *testing.T) { statusCode: http.StatusOK, body: responseV3ASync, args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--tag", "myqueue"}, + wantOutputRegex: "Job submitted with id: 1", + wantBodyRegEx: ".*\"TMDirectives\":{.*\"tag\":\"myqueue\".*}.*", + }, + { + name: "queue flag async", + statusCode: http.StatusOK, + body: responseV3ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--queue", "myqueue"}, wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", wantBodyRegEx: ".*\"TMDirectives\":{.*\"tag\":\"myqueue\".*}.*", }, + { name: "time to live flag async", statusCode: http.StatusOK, body: responseV3ASync, args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--time-to-live", "60"}, + wantOutputRegex: "Job submitted with id: 1", + wantBodyRegEx: ".*\"TMDirectives\":{.*\"ttl\":60.*}.*", + }, + { + name: "max time in queue flag async", + statusCode: http.StatusOK, + body: responseV3ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--max-time-in-queue", "60"}, wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", wantBodyRegEx: ".*\"TMDirectives\":{.*\"ttl\":60.*}.*", }, + { name: "timeuntil canceled flag async", statusCode: http.StatusOK, body: responseV3ASync, args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--time-until-canceled", "60"}, + wantOutputRegex: "Job submitted with id: 1", + wantBodyRegEx: ".*\"TMDirectives\":{.*\"ttc\":60.*}.*", + }, + { + name: "max job runtime flag async", + statusCode: http.StatusOK, + body: responseV3ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--max-job-runtime", "60"}, wantOutputRegex: "^[\\s]*Job submitted with id: 1[\\s]*$", wantBodyRegEx: ".*\"TMDirectives\":{.*\"ttc\":60.*}.*", }, + { name: "published parameter async", statusCode: http.StatusOK, @@ -196,7 +223,7 @@ func TestRun(t *testing.T) { statusCode: http.StatusOK, body: responseV3Sync, args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--tag", "myqueue", "--file", f.Name()}, - wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", + wantOutputRegex: "ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539", wantFormParams: map[string]string{"opt_tag": "myqueue"}, }, { @@ -204,7 +231,7 @@ func TestRun(t *testing.T) { statusCode: http.StatusOK, body: responseV3Sync, args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--time-to-live", "60", "--file", f.Name()}, - wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", + wantOutputRegex: "ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539", wantFormParams: map[string]string{"opt_ttl": "60"}, }, { @@ -212,7 +239,7 @@ func TestRun(t *testing.T) { statusCode: http.StatusOK, body: responseV3Sync, args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--time-until-canceled", "60", "--file", f.Name()}, - wantOutputRegex: "^[\\s]*ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539[\\s]*$", + wantOutputRegex: "ID[\\s]*STATUS[\\s]*STATUS MESSAGE[\\s]*FEATURES OUTPUT[\\s]*1[\\s]*SUCCESS[\\s]*Translation Successful[\\s]*1539", wantFormParams: map[string]string{"opt_ttc": "60"}, }, { diff --git a/cmd/run_v4_test.go b/cmd/run_v4_test.go index 7f83ad4..5e5ff66 100644 --- a/cmd/run_v4_test.go +++ b/cmd/run_v4_test.go @@ -132,6 +132,15 @@ func TestRunV4(t *testing.T) { wantBodyRegEx: ".*\"maxJobRuntime\":10.*", fmeflowBuild: 26018, }, + { + name: "time until canceled flag async deprecated", + statusCode: http.StatusOK, + body: responseV4ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--time-until-canceled", "10"}, + wantOutputRegex: "Flag --time-until-canceled has been deprecated, please use --max-job-runtime instead[\\s\\S]*Job submitted with id: 1", + wantBodyRegEx: ".*\"maxJobRuntime\":10.*", + fmeflowBuild: 26018, + }, { name: "max job runtime invalid value ignored", statusCode: http.StatusOK, @@ -149,6 +158,15 @@ func TestRunV4(t *testing.T) { wantBodyRegEx: ".*\"maxTimeInQueue\":60.*", fmeflowBuild: 26018, }, + { + name: "time to live flag async deprecated", + statusCode: http.StatusOK, + body: responseV4ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--time-to-live", "60"}, + wantOutputRegex: "Flag --time-to-live has been deprecated, please use --max-time-in-queue instead[\\s\\S]*Job submitted with id: 1", + wantBodyRegEx: ".*\"maxTimeInQueue\":60.*", + fmeflowBuild: 26018, + }, { name: "max time in queue invalid value ignored", statusCode: http.StatusOK, @@ -183,6 +201,15 @@ func TestRunV4(t *testing.T) { wantBodyRegEx: ".*\"queue\":\"MyQueue\".*", fmeflowBuild: 26018, }, + { + name: "tag flag async deprecated", + statusCode: http.StatusOK, + body: responseV4ASync, + args: []string{"run", "--repository", "Samples", "--workspace", "austinApartments.fmw", "--tag", "MyQueue"}, + wantOutputRegex: "Flag --tag has been deprecated, please use --queue instead[\\s\\S]*Job submitted with id: 1", + wantBodyRegEx: ".*\"queue\":\"MyQueue\".*", + fmeflowBuild: 26018, + }, { name: "published parameter and list combined", statusCode: http.StatusOK, diff --git a/docs/fmeflow_run.md b/docs/fmeflow_run.md index 9348222..38595dc 100644 --- a/docs/fmeflow_run.md +++ b/docs/fmeflow_run.md @@ -21,7 +21,7 @@ fmeflow run [flags] fmeflow run --repository Samples --workspace austinApartments.fmw --wait # Submit a job to a specific queue and set a time to live in the queue - fmeflow run --repository Samples --workspace austinApartments.fmw --tag Queue1 --time-to-live 120 + fmeflow run --repository Samples --workspace austinApartments.fmw --queue Queue1 --max-time-in-queue 120 # Submit a job and pass in a few published parameters fmeflow run --repository Samples --workspace austinDownload.fmw --published-parameter-list THEMES=railroad,airports --published-parameter COORDSYS=TX83-CF @@ -39,22 +39,19 @@ fmeflow run [flags] --repository string The name of the repository containing the workspace to run. --workspace string The name of the workspace to run. --wait Submit job and wait for it to finish. - --tag string The job routing tag for the request. For v3 API only. --published-parameter stringArray Published parameters defined for this workspace. Specify as Key=Value. Can be passed in multiple times. For list parameters, use the --list-published-parameter flag. --published-parameter-list stringArray A List-type published parameters defined for this workspace. Specify as Key=Value1,Value2. Can be passed in multiple times. --file string Upload a local file Source dataset to use to run the workspace. Note this causes the translation to run in synchonous mode whether the --wait flag is passed in or not. For v3 API only. --run-until-canceled Runs a job until it is explicitly canceled. The job will run again regardless of whether the job completed successfully, failed, or the server crashed or was shut down. For v3 API only. - --time-until-canceled int Time (in seconds) elapsed for a running job before it's cancelled. The minimum value is 1 second, values less than 1 second are ignored. For v3 API only. (default -1) - --time-to-live int Time to live in the job queue (in seconds). For v3 API only. (default -1) --description string Description of the request. For v3 API only. --success-topic stringArray Topics to notify when the job succeeds. Can be specified more than once. --failure-topic stringArray Topics to notify when the job fails. Can be specified more than once. --node-manager-directive stringArray Additional NM Directives, which can include client-configured keys, to pass to the notification service for custom use by subscriptions. Specify as Key=Value Can be passed in multiple times. For v3 API only. --directive stringArray Additional directives to pass to the job submission. Specify as Key=Value. Can be passed in multiple times. For v4 API only. - --queue string Queue of the job to submit. For v4 API only. - --max-job-runtime int Time (in seconds) elapsed for a running job before it's cancelled. The minimum value is 1 second, values less than 1 second are ignored. For v4 API only. - --max-time-in-queue int Time to live in the job queue (in seconds). For v4 API only. - --max-total-life-time int Time to live including both time in the queue and run time (in seconds). The maximum value is 86400 and the minimum value is 1. For v4 API only. + --queue string Queue of the job to submit. + --max-job-runtime int Time (in seconds) elapsed for a running job before it's cancelled. The minimum value is 1 second, values less than 1 second are ignored. (default -1) + --max-time-in-queue int Time to live in the job queue (in seconds). (default -1) + --max-total-life-time int Time to live including both time in the queue and run time (in seconds). The maximum value is 86400 and the minimum value is 1. For v4 API only. (default -1) -o, --output string Specify the output type. Should be one of table, json, or custom-columns (default "table") --no-headers Don't print column headers -h, --help help for run From 1730a98d61e04b9d26f71672b20008ca647ab294 Mon Sep 17 00:00:00 2001 From: Amber Xu Date: Thu, 13 Nov 2025 00:23:01 -0800 Subject: [PATCH 6/6] Updated doc to better help v3 users giving flags --- cmd/run.go | 6 +++--- docs/fmeflow_run.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index 5d8c64a..cd973fe 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -158,11 +158,11 @@ func newRunCmd() *cobra.Command { cmd.Flags().StringArrayVar(&f.failureTopics, "failure-topic", []string{}, "Topics to notify when the job fails. Can be specified more than once.") cmd.Flags().StringArrayVar(&f.nodeManagerDirective, "node-manager-directive", []string{}, "Additional NM Directives, which can include client-configured keys, to pass to the notification service for custom use by subscriptions. Specify as Key=Value Can be passed in multiple times. For v3 API only.") cmd.Flags().StringArrayVar(&f.directive, "directive", []string{}, "Additional directives to pass to the job submission. Specify as Key=Value. Can be passed in multiple times. For v4 API only.") - cmd.Flags().StringVar(&f.queue, "queue", "", "Queue of the job to submit.") + cmd.Flags().StringVar(&f.queue, "queue", "", "Queue of the job to submit. Equavalent to --tag (deprecated).") cmd.Flags().StringVar(&f.queue, "tag", "", "The queue (job routing tag) for the request.") - cmd.Flags().IntVar(&f.maxJobRuntime, "max-job-runtime", -1, "Time (in seconds) elapsed for a running job before it's cancelled. The minimum value is 1 second, values less than 1 second are ignored.") + cmd.Flags().IntVar(&f.maxJobRuntime, "max-job-runtime", -1, "Time (in seconds) elapsed for a running job before it's cancelled. The minimum value is 1 second, values less than 1 second are ignored. Equavalent to --time-until-canceled (deprecated).") cmd.Flags().IntVar(&f.maxJobRuntime, "time-until-canceled", -1, "Time (in seconds) elapsed for a running job before it's cancelled. The minimum value is 1 second, values less than 1 second are ignored.") - cmd.Flags().IntVar(&f.maxTimeInQueue, "max-time-in-queue", -1, "Time to live in the job queue (in seconds).") + cmd.Flags().IntVar(&f.maxTimeInQueue, "max-time-in-queue", -1, "Time to live in the job queue (in seconds). Equavalent to --time-to-live (deprecated).") cmd.Flags().IntVar(&f.maxTimeInQueue, "time-to-live", -1, "Time to live in the job queue (in seconds).") cmd.Flags().IntVar(&f.maxTotalLifeTime, "max-total-life-time", -1, "Time to live including both time in the queue and run time (in seconds). The maximum value is 86400 and the minimum value is 1. For v4 API only.") cmd.Flags().StringVarP(&f.outputType, "output", "o", "table", "Specify the output type. Should be one of table, json, or custom-columns") diff --git a/docs/fmeflow_run.md b/docs/fmeflow_run.md index 38595dc..694e550 100644 --- a/docs/fmeflow_run.md +++ b/docs/fmeflow_run.md @@ -48,9 +48,9 @@ fmeflow run [flags] --failure-topic stringArray Topics to notify when the job fails. Can be specified more than once. --node-manager-directive stringArray Additional NM Directives, which can include client-configured keys, to pass to the notification service for custom use by subscriptions. Specify as Key=Value Can be passed in multiple times. For v3 API only. --directive stringArray Additional directives to pass to the job submission. Specify as Key=Value. Can be passed in multiple times. For v4 API only. - --queue string Queue of the job to submit. - --max-job-runtime int Time (in seconds) elapsed for a running job before it's cancelled. The minimum value is 1 second, values less than 1 second are ignored. (default -1) - --max-time-in-queue int Time to live in the job queue (in seconds). (default -1) + --queue string Queue of the job to submit. Equavalent to --tag (deprecated). + --max-job-runtime int Time (in seconds) elapsed for a running job before it's cancelled. The minimum value is 1 second, values less than 1 second are ignored. Equavalent to --time-until-canceled (deprecated). (default -1) + --max-time-in-queue int Time to live in the job queue (in seconds). Equavalent to --time-to-live (deprecated). (default -1) --max-total-life-time int Time to live including both time in the queue and run time (in seconds). The maximum value is 86400 and the minimum value is 1. For v4 API only. (default -1) -o, --output string Specify the output type. Should be one of table, json, or custom-columns (default "table") --no-headers Don't print column headers