diff --git a/Dockerfile b/Dockerfile index 676f80b3..168d7530 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,12 @@ # docker build --force-rm --no-cache --rm -t mgrast/awe . # docker tag mgrast/awe mgrast/awe:${TAG} +# skycore push mgrast/awe:${TAG} +# docker create --name awe-temporary mgrast/awe:${TAG} +# docker cp awe-temporary:/go/bin/awe-worker . +# docker cp awe-temporary:/go/bin/awe-submitter . +# docker rm awe-temporary + FROM golang:1.7.6-alpine diff --git a/awe-submitter/awe-submitter.go b/awe-submitter/awe-submitter.go index 4207147e..36366d27 100644 --- a/awe-submitter/awe-submitter.go +++ b/awe-submitter/awe-submitter.go @@ -7,9 +7,15 @@ import ( "github.com/MG-RAST/AWE/lib/core/cwl" cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" "github.com/MG-RAST/AWE/lib/logger" + //"github.com/MG-RAST/AWE/lib/logger/event" + "bytes" "github.com/MG-RAST/AWE/lib/shock" "github.com/davecgh/go-spew/spew" "gopkg.in/yaml.v2" + "io" + "io/ioutil" + "mime/multipart" + "net/http" "net/url" "os" "path" @@ -54,6 +60,8 @@ func uploadFile(file *cwl_types.File, inputfile_path string) (err error) { file_path := file.Path + basename := path.Base(file_path) + if file_path == "" { return } @@ -75,35 +83,39 @@ func uploadFile(file *cwl_types.File, inputfile_path string) (err error) { } spew.Dump(node) - file.Location_url, err = url.Parse(conf.SERVER_URL + "/node/" + node.Id + "?download") + file.Location_url, err = url.Parse(conf.SHOCK_URL + "/node/" + node.Id + "?download") if err != nil { return } file.Location = file.Location_url.String() file.Path = "" + file.Basename = basename return } -func processInputData(native interface{}, inputfile_path string) (err error) { +func processInputData(native interface{}, inputfile_path string) (count int, err error) { + fmt.Printf("(processInputData) start\n") defer fmt.Printf("(processInputData) end\n") switch native.(type) { case *cwl.Job_document: - + fmt.Printf("found Job_document\n") job_doc_ptr := native.(*cwl.Job_document) job_doc := *job_doc_ptr - for key, value := range job_doc { + for _, value := range job_doc { - fmt.Printf("recurse into key: %s\n", key) - err = processInputData(value, inputfile_path) + id := value.GetId() + fmt.Printf("recurse into key: %s\n", id) + var sub_count int + sub_count, err = processInputData(value, inputfile_path) if err != nil { return } - + count += sub_count } return @@ -123,7 +135,7 @@ func processInputData(native interface{}, inputfile_path string) (err error) { if err != nil { return } - + count += 1 return default: spew.Dump(native) @@ -135,10 +147,19 @@ func processInputData(native interface{}, inputfile_path string) (err error) { } func main() { + err := main_wrapper() + if err != nil { + println(err.Error()) + os.Exit(1) + } + os.Exit(0) +} + +func main_wrapper() (err error) { conf.LOG_OUTPUT = "console" - err := conf.Init_conf("submitter") + err = conf.Init_conf("submitter") if err != nil { fmt.Fprintf(os.Stderr, "ERROR: error reading conf file: "+err.Error()) @@ -147,23 +168,24 @@ func main() { logger.Initialize("client") - if conf.CWL_JOB == "" { - logger.Error("cwl job file missing") - time.Sleep(time.Second) - os.Exit(1) + for _, value := range conf.ARGS { + println(value) } - inputfile_path := path.Dir(conf.CWL_JOB) + job_file := conf.ARGS[0] + workflow_file := conf.ARGS[1] + + inputfile_path := path.Dir(job_file) fmt.Printf("job path: %s\n", inputfile_path) // needed to resolve relative paths - job_doc, err := cwl.ParseJob(conf.CWL_JOB) + job_doc, err := cwl.ParseJobFile(job_file) if err != nil { logger.Error("error parsing cwl job: %v", err) time.Sleep(time.Second) os.Exit(1) } - fmt.Println("Job input fter reading from file:") + fmt.Println("Job input after reading from file:") spew.Dump(*job_doc) data, err := yaml.Marshal(*job_doc) @@ -172,15 +194,23 @@ func main() { os.Exit(1) } - fmt.Printf("yaml:\n%s\n", string(data[:])) + job_doc_string := string(data[:]) + fmt.Printf("job_doc_string: \"%s\"\n", job_doc_string) + if job_doc_string == "" { + fmt.Println("job_doc_string is empty") + os.Exit(1) + } + + fmt.Printf("yaml:\n%s\n", job_doc_string) // process input files - err = processInputData(job_doc, inputfile_path) + upload_count, err := processInputData(job_doc, inputfile_path) if err != nil { fmt.Printf("error: %s", err.Error()) os.Exit(1) } + fmt.Printf("%d files have been uploaded\n", upload_count) time.Sleep(2) fmt.Println("------------Job input after parsing:") @@ -192,4 +222,111 @@ func main() { fmt.Printf("yaml:\n%s\n", string(data[:])) + // job submission example: + // curl -X POST -F job=@test.yaml -F cwl=@/Users/wolfganggerlach/awe_data/pipeline/CWL/PackedWorkflow/preprocess-fasta.workflow.cwl http://localhost:8001/job + + //var b bytes.Buffer + //w := multipart.NewWriter(&b) + err = SubmitCWLJobToAWE(workflow_file, job_file, &data) + if err != nil { + return + } + + return +} + +func SubmitCWLJobToAWE(workflow_file string, job_file string, data *[]byte) (err error) { + multipart := NewMultipartWriter() + err = multipart.AddFile("cwl", workflow_file) + if err != nil { + return + } + err = multipart.AddDataAsFile("job", job_file, data) + if err != nil { + return + } + response, err := multipart.Send("POST", conf.SERVER_URL+"/job") + if err != nil { + return + } + responseData, err := ioutil.ReadAll(response.Body) + if err != nil { + return + } + responseString := string(responseData) + + fmt.Println(responseString) + return + +} + +type MultipartWriter struct { + b bytes.Buffer + w *multipart.Writer +} + +func NewMultipartWriter() *MultipartWriter { + m := &MultipartWriter{} + m.w = multipart.NewWriter(&m.b) + return m +} + +func (m *MultipartWriter) Send(method string, url string) (response *http.Response, err error) { + m.w.Close() + fmt.Println("------------") + spew.Dump(m.w) + fmt.Println("------------") + + req, err := http.NewRequest(method, url, &m.b) + if err != nil { + return + } + // Don't forget to set the content type, this will contain the boundary. + req.Header.Set("Content-Type", m.w.FormDataContentType()) + + // Submit the request + client := &http.Client{} + fmt.Printf("%s %s\n\n", method, url) + response, err = client.Do(req) + if err != nil { + return + } + + // Check the response + //if response.StatusCode != http.StatusOK { + // err = fmt.Errorf("bad status: %s", response.Status) + //} + return + +} + +func (m *MultipartWriter) AddDataAsFile(fieldname string, filepath string, data *[]byte) (err error) { + + fw, err := m.w.CreateFormFile(fieldname, filepath) + if err != nil { + return + } + _, err = fw.Write(*data) + if err != nil { + return + } + return +} + +func (m *MultipartWriter) AddFile(fieldname string, filepath string) (err error) { + + f, err := os.Open(filepath) + if err != nil { + return + } + defer f.Close() + fw, err := m.w.CreateFormFile(fieldname, filepath) + if err != nil { + return + } + if _, err = io.Copy(fw, f); err != nil { + return + } + + return } diff --git a/awe-worker/awe-worker.go b/awe-worker/awe-worker.go index d31f59e0..6d670892 100644 --- a/awe-worker/awe-worker.go +++ b/awe-worker/awe-worker.go @@ -78,7 +78,9 @@ func main() { os.Exit(1) } - var self *core.Client + core.SetClientProfile(profile) + self := core.Self + //var self *core.Client if worker.Client_mode == "online" { if conf.SERVER_URL == "" { fmt.Fprintf(os.Stderr, "AWE server url not configured or is empty. Please check the [Client]serverurl field in the configuration file.\n") @@ -89,20 +91,17 @@ func main() { os.Exit(1) } - self, err = worker.RegisterWithAuth(conf.SERVER_URL, profile) + err = worker.RegisterWithAuth(conf.SERVER_URL, profile) if err != nil { fmt.Fprintf(os.Stderr, "fail to register: %s\n", err.Error()) logger.Error("fail to register: %s\n", err.Error()) os.Exit(1) } - } else { - self = core.NewClient() } - core.SetClientProfile(self) if worker.Client_mode == "online" { - fmt.Printf("Client registered, name=%s, id=%s\n", self.Name, self.Id) + fmt.Printf("Client registered, name=%s, id=%s\n", self.WorkerRuntime.Name, self.Id) logger.Event(event.CLIENT_REGISTRATION, "clientid="+self.Id) } @@ -114,7 +113,7 @@ func main() { time.Sleep(time.Second) os.Exit(1) } - job_doc, err := cwl.ParseJob(conf.CWL_JOB) + job_doc, err := cwl.ParseJobFile(conf.CWL_JOB) if err != nil { logger.Error("error parsing cwl job: %v", err) time.Sleep(time.Second) @@ -126,13 +125,13 @@ func main() { os.Getwd() //https://golang.org/pkg/os/#Getwd - workunit := &core.Workunit{Id: "00000000-0000-0000-0000-000000000000_0_0", CWL: core.NewCWL_workunit()} + workunit := &core.Workunit{Id: "00000000-0000-0000-0000-000000000000_0_0", CWL_workunit: core.NewCWL_workunit()} - workunit.CWL.Job_input = job_doc - workunit.CWL.Job_input_filename = conf.CWL_JOB + workunit.CWL_workunit.Job_input = job_doc + workunit.CWL_workunit.Job_input_filename = conf.CWL_JOB - workunit.CWL.CWL_tool_filename = conf.CWL_TOOL - workunit.CWL.CWL_tool = &cwl.CommandLineTool{} // TODO parsing and testing ? + workunit.CWL_workunit.CWL_tool_filename = conf.CWL_TOOL + workunit.CWL_workunit.CWL_tool = &cwl.CommandLineTool{} // TODO parsing and testing ? current_working_directory, err := os.Getwd() if err != nil { @@ -142,13 +141,15 @@ func main() { } workunit.WorkPath = current_working_directory - workunit.Cmd = &core.Command{} - workunit.Cmd.Local = true // this makes sure the working directory is not deleted - workunit.Cmd.Name = "/usr/bin/cwl-runner" + cmd := &core.Command{} + cmd.Local = true // this makes sure the working directory is not deleted + cmd.Name = "/usr/bin/cwl-runner" - workunit.Cmd.ArgsArray = []string{"--leave-outputs", "--leave-tmpdir", "--tmp-outdir-prefix", "./tmp/", "--tmpdir-prefix", "./tmp/", "--disable-pull", "--rm-container", "--on-error", "stop", workunit.CWL.CWL_tool_filename, workunit.CWL.Job_input_filename} + cmd.ArgsArray = []string{"--leave-outputs", "--leave-tmpdir", "--tmp-outdir-prefix", "./tmp/", "--tmpdir-prefix", "./tmp/", "--disable-pull", "--rm-container", "--on-error", "stop", workunit.CWL_workunit.CWL_tool_filename, workunit.CWL_workunit.Job_input_filename} - workunit.WorkPerf = core.NewWorkPerf(workunit.Id) + workunit.Cmd = cmd + + workunit.WorkPerf = core.NewWorkPerf() workunit.WorkPerf.Checkout = time.Now().Unix() logger.Debug(1, "injecting cwl job into worker...") diff --git a/lib/cache/cache.go b/lib/cache/cache.go index 016308b8..82a8ada5 100644 --- a/lib/cache/cache.go +++ b/lib/cache/cache.go @@ -6,13 +6,18 @@ import ( "fmt" "github.com/MG-RAST/AWE/lib/conf" "github.com/MG-RAST/AWE/lib/core" + //"github.com/MG-RAST/AWE/lib/core/cwl" + cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" "github.com/MG-RAST/AWE/lib/logger" "github.com/MG-RAST/AWE/lib/logger/event" "github.com/MG-RAST/AWE/lib/shock" "github.com/MG-RAST/golib/httpclient" + "github.com/davecgh/go-spew/spew" "io" "io/ioutil" "os" + "path" + "reflect" "strings" "time" ) @@ -35,81 +40,185 @@ func StatCacheFilePath(id string) (file_path string, err error) { return file_path, err } -//fetch input data -func MoveInputData(work *core.Workunit) (size int64, err error) { - for _, io := range work.Inputs { - // skip if NoFile == true - if !io.NoFile { // is file ! - dataUrl, uerr := io.DataUrl() - if uerr != nil { - return 0, uerr - } - inputFilePath := fmt.Sprintf("%s/%s", work.Path(), io.FileName) - - if work.Rank == 0 { - if conf.CACHE_ENABLED && io.Node != "" { - if file_path, err := StatCacheFilePath(io.Node); err == nil { - //make a link in work dir from cached file - linkname := fmt.Sprintf("%s/%s", work.Path(), io.FileName) - fmt.Printf("input found in cache, making link: " + file_path + " -> " + linkname + "\n") - err = os.Symlink(file_path, linkname) - if err == nil { - logger.Event(event.FILE_READY, "workid="+work.Id+";url="+dataUrl) - } - return 0, err - } +func MoveInputIO(work *core.Workunit, io *core.IO, work_path string) (size int64, err error) { + + if !io.NoFile { // is file ! + dataUrl, uerr := io.DataUrl() + if uerr != nil { + err = uerr + return + } + if io.FileName == "" { + err = fmt.Errorf("io.Filename is empty") + return + } + + inputFilePath := path.Join(work_path, io.FileName) + + // create symlink if file has been cached + if work.Rank == 0 && conf.CACHE_ENABLED && io.Node != "" { + var file_path string + file_path, err = StatCacheFilePath(io.Node) + if err == nil { + //make a link in work dir from cached file + linkname := fmt.Sprintf("%s/%s", work_path, io.FileName) + fmt.Printf("input found in cache, making link: " + file_path + " -> " + linkname + "\n") + err = os.Symlink(file_path, linkname) + if err != nil { + return } + logger.Event(event.FILE_READY, "workid="+work.Id+";url="+dataUrl) + return } - // only get file Part based on work.Partition - if (work.Rank > 0) && (work.Partition != nil) && (work.Partition.Input == io.FileName) { - dataUrl = fmt.Sprintf("%s&index=%s&part=%s", dataUrl, work.Partition.Index, work.Part()) + + } + + // only get file Part based on work.Partition + if (work.Rank > 0) && (work.Partition != nil) && (work.Partition.Input == io.FileName) { + dataUrl = fmt.Sprintf("%s&index=%s&part=%s", dataUrl, work.Partition.Index, work.Part()) + } + logger.Debug(2, "mover: fetching input file from url:"+dataUrl) + logger.Event(event.FILE_IN, "workid="+work.Id+";url="+dataUrl) + + // download file + retry := 1 + for true { + datamoved, _, err := shock.FetchFile(inputFilePath, dataUrl, work.Info.DataToken, io.Uncompress, false) + if err != nil { + if !strings.Contains(err.Error(), "Node has no file") { + logger.Debug(3, "(MoveInputData) got: %s", err.Error()) + return size, err + } + logger.Debug(3, "(MoveInputData) got: Node has no file") + if retry >= 3 { + return size, err + } + logger.Warning("(MoveInputData) Will retry download, got this error: %s", err.Error()) + time.Sleep(time.Second * 20) + retry += 1 + continue } - logger.Debug(2, "mover: fetching input file from url:"+dataUrl) - logger.Event(event.FILE_IN, "workid="+work.Id+";url="+dataUrl) - // download file - retry := 1 - for true { - datamoved, _, err := shock.FetchFile(inputFilePath, dataUrl, work.Info.DataToken, io.Uncompress, false) + size += datamoved + break + } + logger.Event(event.FILE_READY, "workid="+work.Id+";url="+dataUrl) + } + + // download node attributes if requested + if io.AttrFile != "" { + // get node + node, xerr := shock.ShockGet(io.Host, io.Node, work.Info.DataToken) + if xerr != nil { + //return size, err + err = errors.New("shock.ShockGet (node attributes) returned: " + xerr.Error()) + return + } + logger.Debug(2, "mover: fetching input attributes from node:"+node.Id) + logger.Event(event.ATTR_IN, "workid="+work.Id+";node="+node.Id) + // print node attributes + work_path, yerr := work.Path() + if yerr != nil { + + return 0, yerr + } + attrFilePath := fmt.Sprintf("%s/%s", work_path, io.AttrFile) + attr_json, _ := json.Marshal(node.Attributes) + err = ioutil.WriteFile(attrFilePath, attr_json, 0644) + if err != nil { + return + } + logger.Event(event.ATTR_READY, "workid="+work.Id+";path="+attrFilePath) + } + return +} + +func CWL_File_2_AWE_IO(file *cwl_types.File) (io *core.IO, err error) { + + url_obj := file.Location_url + + // example: http://localhost:8001/node/429a47aa-85e4-4575-9347-a78cfacb6979?download + + io = core.NewIO() + + url_string := url_obj.String() + + io.Url = url_string + err = io.Url2Shock() // populates Host and Node + if err != nil { + err = fmt.Errorf("(CWL_File2AWE_IO) %s", err.Error()) + } + + if file.Basename == "" { + basename := path.Base(url_string) + io.FileName = basename + } else { + io.FileName = file.Basename + } + + return +} + +//fetch input data +func MoveInputData(work *core.Workunit) (size int64, err error) { + + work_path, xerr := work.Path() + if xerr != nil { + err = xerr + return + } + + if work.CWL_workunit != nil { + + job_input := work.CWL_workunit.Job_input + spew.Dump(job_input) + + for input_name, input := range *job_input { + fmt.Println(input_name) + spew.Dump(input) + switch input.(type) { + case *cwl_types.File: + file := input.(*cwl_types.File) + spew.Dump(*file) + fmt.Printf("file: %+v\n", *file) + + var io *core.IO + io, err = CWL_File_2_AWE_IO(file) + if err != nil { + return + } + + var io_size int64 + io_size, err = MoveInputIO(work, io, work_path) if err != nil { - if !strings.Contains(err.Error(), "Node has no file") { - logger.Debug(3, "(MoveInputData) got: err.Error()") - return size, err - } - logger.Debug(3, "(MoveInputData) got: Node has no file") - if retry >= 3 { - return size, err - } - logger.Warning("(MoveInputData) Will retry download, got this error: %s", err.Error()) - time.Sleep(time.Second * 20) - retry += 1 - continue + err = fmt.Errorf("(MoveInputData) MoveInputIO returns %s", err.Error()) + return } + spew.Dump(io) + size += io_size - size += datamoved - break + continue + case *cwl_types.String: + continue + default: + err = fmt.Errorf("(MoveInputData) type %s not supoorted yet", reflect.TypeOf(input)) + return } - logger.Event(event.FILE_READY, "workid="+work.Id+";url="+dataUrl) } - // download node attributes if requested - if io.AttrFile != "" { - // get node - node, err := shock.ShockGet(io.Host, io.Node, work.Info.DataToken) - if err != nil { - //return size, err - return size, errors.New("shock.ShockGet (node attributes) returned: " + err.Error()) - } - logger.Debug(2, "mover: fetching input attributes from node:"+node.Id) - logger.Event(event.ATTR_IN, "workid="+work.Id+";node="+node.Id) - // print node attributes - attrFilePath := fmt.Sprintf("%s/%s", work.Path(), io.AttrFile) - attr_json, _ := json.Marshal(node.Attributes) - if err := ioutil.WriteFile(attrFilePath, attr_json, 0644); err != nil { - return size, err - } - logger.Event(event.ATTR_READY, "workid="+work.Id+";path="+attrFilePath) + return + } + + for _, io := range work.Inputs { + // skip if NoFile == true + var io_size int64 + io_size, err = MoveInputIO(work, io, work_path) + if err != nil { + err = fmt.Errorf("(MoveInputData) MoveInputIO returns %s", err.Error()) + return } + + size += io_size } return } @@ -160,26 +269,29 @@ func fetchFile_deprecated(filename string, url string, token string) (size int64 func UploadOutputData(work *core.Workunit) (size int64, err error) { logger.Info("Processing %d outputs for uploading", len(work.Outputs)) - + work_path, err := work.Path() + if err != nil { + return + } for _, io := range work.Outputs { name := io.FileName var local_filepath string //local file name generated by the cmd var file_path string //file name to be uploaded to shock if io.Directory != "" { - local_filepath = fmt.Sprintf("%s/%s/%s", work.Path(), io.Directory, name) + local_filepath = fmt.Sprintf("%s/%s/%s", work_path, io.Directory, name) //if specified, rename the local file name to the specified shock node file name //otherwise use the local name as shock file name file_path = local_filepath if io.ShockFilename != "" { - file_path = fmt.Sprintf("%s/%s/%s", work.Path(), io.Directory, io.ShockFilename) + file_path = fmt.Sprintf("%s/%s/%s", work_path, io.Directory, io.ShockFilename) os.Rename(local_filepath, file_path) } } else { - local_filepath = fmt.Sprintf("%s/%s", work.Path(), name) + local_filepath = fmt.Sprintf("%s/%s", work_path, name) file_path = local_filepath if io.ShockFilename != "" { - file_path = fmt.Sprintf("%s/%s", work.Path(), io.ShockFilename) + file_path = fmt.Sprintf("%s/%s", work_path, io.ShockFilename) os.Rename(local_filepath, file_path) } } @@ -211,7 +323,7 @@ func UploadOutputData(work *core.Workunit) (size int64, err error) { //upload attribute file to shock IF attribute file is specified in outputs AND it is found in local directory. var attrfile_path string = "" if io.AttrFile != "" { - attrfile_path = fmt.Sprintf("%s/%s", work.Path(), io.AttrFile) + attrfile_path = fmt.Sprintf("%s/%s", work_path, io.AttrFile) if fi, err := os.Stat(attrfile_path); err != nil || fi.Size() == 0 { attrfile_path = "" } diff --git a/lib/conf/conf.go b/lib/conf/conf.go index 4c57aeb5..39b69520 100644 --- a/lib/conf/conf.go +++ b/lib/conf/conf.go @@ -13,7 +13,7 @@ import ( "time" ) -const VERSION string = "0.9.51" +const VERSION string = "0.9.53" var GIT_COMMIT_HASH string // use -ldflags "-X github.com/MG-RAST/AWE/lib/conf.GIT_COMMIT_HASH " const BasePriority int = 1 @@ -171,6 +171,8 @@ var ( PRINT_HELP bool // full usage SHOW_HELP bool // simple usage SHOW_GIT_COMMIT_HASH bool + CPUPROFILE string + MEMPROFILE string // used to track changes in data structures VERSIONS = make(map[string]int) @@ -192,73 +194,10 @@ var ( // internal config control FAKE_VAR = false -) - -// writes to target only if has been defined in config -// avoids overwriting of default values if config is not defined -func getDefinedValueInt(c *config.Config, section string, key string, target *int) { - if c.HasOption(section, key) { - if int_value, err := c.Int(section, key); err == nil { - *target = int_value - } - } -} - -func getDefinedValueBool(c *config.Config, section string, key string, target *bool) { - if c.HasOption(section, key) { - if bool_value, err := c.Bool(section, key); err == nil { - *target = bool_value - } - } -} - -func getDefinedValueString(c *config.Config, section string, key string, target *string) { - if string_value, err := c.String(section, key); err == nil { - string_value = os.ExpandEnv(string_value) - *target = string_value - } - -} - -type Config_value struct { - Conf_type string - Conf_str *Config_value_string - Conf_int *Config_value_int - Conf_bool *Config_value_bool -} -type Config_value_string struct { - Target *string - Default_value string - Section string - Key string - Descr_short string - Descr_long string -} - -type Config_value_int struct { - Target *int - Default_value int - Section string - Key string - Descr_short string - Descr_long string -} - -type Config_value_bool struct { - Target *bool - Default_value bool - Section string - Key string - Descr_short string - Descr_long string -} - -type Config_store struct { - Store []*Config_value - Fs *flag.FlagSet - Con *config.Config -} + // + ARGS []string +) type LoginResource struct { Icon string `json:"icon"` @@ -269,121 +208,6 @@ type LoginResource struct { Bearer string `json:"bearer"` } -func NewCS(c *config.Config) *Config_store { - cs := &Config_store{Store: make([]*Config_value, 0, 100), Con: c} // length 0, capacity 100 - cs.Fs = flag.NewFlagSet("name", flag.ContinueOnError) - cs.Fs.BoolVar(&FAKE_VAR, "fake_var", true, "ignore this") - cs.Fs.BoolVar(&SHOW_HELP, "h", false, "ignore this") // for help: -h - return cs -} - -func (this *Config_store) AddString(target *string, - default_value string, - section string, - key string, - descr_short string, - descr_long string) { - - *target = default_value - new_val := &Config_value{Conf_type: "string"} - new_val.Conf_str = &Config_value_string{target, default_value, section, key, descr_short, descr_long} - - this.Store = append(this.Store, new_val) -} - -func (this *Config_store) AddInt(target *int, - default_value int, - section string, - key string, - descr_short string, - descr_long string) { - - *target = default_value - new_val := &Config_value{Conf_type: "int"} - new_val.Conf_int = &Config_value_int{target, default_value, section, key, descr_short, descr_long} - - this.Store = append(this.Store, new_val) -} - -func (this *Config_store) AddBool(target *bool, - default_value bool, - section string, - key string, - descr_short string, - descr_long string) { - - *target = default_value - new_val := &Config_value{Conf_type: "bool"} - new_val.Conf_bool = &Config_value_bool{target, default_value, section, key, descr_short, descr_long} - - this.Store = append(this.Store, new_val) -} - -func (this Config_store) Parse() { - c := this.Con - f := this.Fs - for _, val := range this.Store { - if val.Conf_type == "string" { - get_my_config_string(c, f, val.Conf_str) - } else if val.Conf_type == "int" { - get_my_config_int(c, f, val.Conf_int) - } else if val.Conf_type == "bool" { - get_my_config_bool(c, f, val.Conf_bool) - } - } - err := this.Fs.Parse(os.Args[1:]) - if err != nil { - this.PrintHelp() - fmt.Fprintf(os.Stderr, "error parsing command line args: "+err.Error()+"\n") - os.Exit(1) - } -} - -func (this Config_store) PrintHelp() { - current_section := "" - prefix := "--" - if PRINT_HELP { - prefix = "" - } - for _, val := range this.Store { - if val.Conf_type == "string" { - d := val.Conf_str - if current_section != d.Section { - current_section = d.Section - fmt.Printf("\n[%s]\n", current_section) - } - fmt.Printf("%s%-27s %s (default: \"%s\")\n", prefix, d.Key+"=", d.Descr_short, d.Default_value) - - if PRINT_HELP && d.Descr_long != "" { - fmt.Printf(" %s\n", d.Descr_long) - } - } else if val.Conf_type == "int" { - d := val.Conf_int - if current_section != d.Section { - current_section = d.Section - fmt.Printf("\n[%s]\n", current_section) - } - fmt.Printf("%s%-27s %s (default: %d)\n", prefix, d.Key+"=", d.Descr_short, d.Default_value) - - if PRINT_HELP && d.Descr_long != "" { - fmt.Printf(" %s\n", d.Descr_long) - } - } else if val.Conf_type == "bool" { - d := val.Conf_bool - if current_section != d.Section { - current_section = d.Section - fmt.Printf("\n[%s]\n", current_section) - } - fmt.Printf("%s%-27s %s (default: %t)\n", prefix, d.Key+"=", d.Descr_short, d.Default_value) - - if PRINT_HELP && d.Descr_long != "" { - fmt.Printf(" %s\n", d.Descr_long) - } - } - } - -} - func get_my_config_string(c *config.Config, f *flag.FlagSet, val *Config_value_string) { //overwrite variable if defined in config file if c != nil { @@ -561,6 +385,8 @@ func getConfiguration(c *config.Config, mode string) (c_store *Config_store) { c_store.AddBool(&SHOW_GIT_COMMIT_HASH, false, "Other", "show_git_commit_hash", "", "") c_store.AddBool(&PRINT_HELP, false, "Other", "fullhelp", "show detailed usage without \"--\"-prefixes", "") c_store.AddBool(&SHOW_HELP, false, "Other", "help", "show usage", "") + c_store.AddString(&CPUPROFILE, "", "Other", "cpuprofile", "e.g. create cpuprofile.prof", "") + c_store.AddString(&MEMPROFILE, "", "Other", "memprofile", "e.g. create memprofile.prof", "") c_store.Parse() return @@ -594,6 +420,8 @@ func Init_conf(mode string) (err error) { // ####### at this point configuration variables are set ######## + ARGS = c_store.Fs.Args() + if FAKE_VAR == false { return errors.New("config was not parsed") } diff --git a/lib/conf/config_store.go b/lib/conf/config_store.go new file mode 100644 index 00000000..e2c57b04 --- /dev/null +++ b/lib/conf/config_store.go @@ -0,0 +1,189 @@ +package conf + +import ( + "flag" + "fmt" + "github.com/MG-RAST/golib/goconfig/config" + "os" +) + +type Config_store struct { + Store []*Config_value + Fs *flag.FlagSet + Con *config.Config +} + +type Config_value struct { + Conf_type string + Conf_str *Config_value_string + Conf_int *Config_value_int + Conf_bool *Config_value_bool +} + +type Config_value_string struct { + Target *string + Default_value string + Section string + Key string + Descr_short string + Descr_long string +} + +type Config_value_int struct { + Target *int + Default_value int + Section string + Key string + Descr_short string + Descr_long string +} + +type Config_value_bool struct { + Target *bool + Default_value bool + Section string + Key string + Descr_short string + Descr_long string +} + +func NewCS(c *config.Config) *Config_store { + cs := &Config_store{Store: make([]*Config_value, 0, 100), Con: c} // length 0, capacity 100 + cs.Fs = flag.NewFlagSet("name", flag.ContinueOnError) + cs.Fs.BoolVar(&FAKE_VAR, "fake_var", true, "ignore this") + cs.Fs.BoolVar(&SHOW_HELP, "h", false, "ignore this") // for help: -h + return cs +} + +func (this *Config_store) AddString(target *string, + default_value string, + section string, + key string, + descr_short string, + descr_long string) { + + *target = default_value + new_val := &Config_value{Conf_type: "string"} + new_val.Conf_str = &Config_value_string{target, default_value, section, key, descr_short, descr_long} + + this.Store = append(this.Store, new_val) +} + +func (this *Config_store) AddInt(target *int, + default_value int, + section string, + key string, + descr_short string, + descr_long string) { + + *target = default_value + new_val := &Config_value{Conf_type: "int"} + new_val.Conf_int = &Config_value_int{target, default_value, section, key, descr_short, descr_long} + + this.Store = append(this.Store, new_val) +} + +func (this *Config_store) AddBool(target *bool, + default_value bool, + section string, + key string, + descr_short string, + descr_long string) { + + *target = default_value + new_val := &Config_value{Conf_type: "bool"} + new_val.Conf_bool = &Config_value_bool{target, default_value, section, key, descr_short, descr_long} + + this.Store = append(this.Store, new_val) +} + +func (this Config_store) Parse() { + c := this.Con + f := this.Fs + for _, val := range this.Store { + if val.Conf_type == "string" { + get_my_config_string(c, f, val.Conf_str) + } else if val.Conf_type == "int" { + get_my_config_int(c, f, val.Conf_int) + } else if val.Conf_type == "bool" { + get_my_config_bool(c, f, val.Conf_bool) + } + } + err := this.Fs.Parse(os.Args[1:]) + if err != nil { + this.PrintHelp() + fmt.Fprintf(os.Stderr, "error parsing command line args: "+err.Error()+"\n") + os.Exit(1) + } +} + +func (this Config_store) PrintHelp() { + current_section := "" + prefix := "--" + if PRINT_HELP { + prefix = "" + } + for _, val := range this.Store { + if val.Conf_type == "string" { + d := val.Conf_str + if current_section != d.Section { + current_section = d.Section + fmt.Printf("\n[%s]\n", current_section) + } + fmt.Printf("%s%-27s %s (default: \"%s\")\n", prefix, d.Key+"=", d.Descr_short, d.Default_value) + + if PRINT_HELP && d.Descr_long != "" { + fmt.Printf(" %s\n", d.Descr_long) + } + } else if val.Conf_type == "int" { + d := val.Conf_int + if current_section != d.Section { + current_section = d.Section + fmt.Printf("\n[%s]\n", current_section) + } + fmt.Printf("%s%-27s %s (default: %d)\n", prefix, d.Key+"=", d.Descr_short, d.Default_value) + + if PRINT_HELP && d.Descr_long != "" { + fmt.Printf(" %s\n", d.Descr_long) + } + } else if val.Conf_type == "bool" { + d := val.Conf_bool + if current_section != d.Section { + current_section = d.Section + fmt.Printf("\n[%s]\n", current_section) + } + fmt.Printf("%s%-27s %s (default: %t)\n", prefix, d.Key+"=", d.Descr_short, d.Default_value) + + if PRINT_HELP && d.Descr_long != "" { + fmt.Printf(" %s\n", d.Descr_long) + } + } + } + +} + +// writes to target only if has been defined in config +// avoids overwriting of default values if config is not defined +func getDefinedValueInt(c *config.Config, section string, key string, target *int) { + if c.HasOption(section, key) { + if int_value, err := c.Int(section, key); err == nil { + *target = int_value + } + } +} + +func getDefinedValueBool(c *config.Config, section string, key string, target *bool) { + if c.HasOption(section, key) { + if bool_value, err := c.Bool(section, key); err == nil { + *target = bool_value + } + } +} + +func getDefinedValueString(c *config.Config, section string, key string, target *string) { + if string_value, err := c.String(section, key); err == nil { + string_value = os.ExpandEnv(string_value) + *target = string_value + } + +} diff --git a/lib/controller/clientController.go b/lib/controller/clientController.go index c1e381c5..1f1d7518 100644 --- a/lib/controller/clientController.go +++ b/lib/controller/clientController.go @@ -1,6 +1,7 @@ package controller import ( + "encoding/json" "fmt" "github.com/MG-RAST/AWE/lib/conf" "github.com/MG-RAST/AWE/lib/core" @@ -10,6 +11,7 @@ import ( "github.com/MG-RAST/AWE/lib/request" "github.com/MG-RAST/AWE/lib/user" "github.com/MG-RAST/golib/goweb" + "io/ioutil" "net/http" "strconv" "strings" @@ -30,20 +32,10 @@ func (cr *ClientController) Create(cx *goweb.Context) { // Log Request and check for Auth LogRequest(cx.Request) - cg, err := request.AuthenticateClientGroup(cx.Request) - if err != nil { - if err.Error() == e.NoAuth || err.Error() == e.UnAuth || err.Error() == e.InvalidAuth { - if conf.CLIENT_AUTH_REQ == true { - cx.RespondWithError(http.StatusUnauthorized) - return - } - } else { - logger.Error("Err@AuthenticateClientGroup: " + err.Error()) - cx.RespondWithError(http.StatusInternalServerError) - return - } + cg, done := GetClientGroup(cx) + if done { + return } - // Parse uploaded form _, files, err := ParseMultipartForm(cx.Request) @@ -65,7 +57,7 @@ func (cr *ClientController) Create(cx *goweb.Context) { } //log event about client registration (CR) - logger.Event(event.CLIENT_REGISTRATION, "clientid="+client.Id+";name="+client.Name+";host="+client.Host+";group="+client.Group+";instance_id="+client.InstanceId+";instance_type="+client.InstanceType+";domain="+client.Domain) + logger.Event(event.CLIENT_REGISTRATION, "clientid="+client.Id+";host="+client.Host+";group="+client.Group+";instance_id="+client.InstanceId+";instance_type="+client.InstanceType+";domain="+client.Domain) rlock, err := client.RLockNamed("ClientController/Create") if err != nil { @@ -75,35 +67,17 @@ func (cr *ClientController) Create(cx *goweb.Context) { } defer client.RUnlockNamed(rlock) cx.RespondWithData(client) + return } // GET: /client/{id} func (cr *ClientController) Read(id string, cx *goweb.Context) { // Gather query params - query := &Query{Li: cx.Request.URL.Query()} - - if query.Has("heartbeat") { //handle heartbeat - cg, err := request.AuthenticateClientGroup(cx.Request) - if err != nil { - if err.Error() == e.NoAuth || err.Error() == e.UnAuth || err.Error() == e.InvalidAuth { - if conf.CLIENT_AUTH_REQ == true { - cx.RespondWithError(http.StatusUnauthorized) - return - } - } else { - logger.Error("Err@AuthenticateClientGroup: " + err.Error()) - cx.RespondWithError(http.StatusInternalServerError) - return - } - } - hbmsg, err := core.QMgr.ClientHeartBeat(id, cg) - if err != nil { - cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) - } else { - cx.RespondWithData(hbmsg) - } + query := &Query{Li: cx.Request.URL.Query()} + if query.Has("heartbeat") { // OLD heartbeat + cx.RespondWithErrorMessage("Please update to newer version of awe-worker. Old heartbeat mechanism not supported anymore.", http.StatusBadRequest) return } @@ -179,7 +153,7 @@ func (cr *ClientController) ReadMany(cx *goweb.Context) { if query.Has("busy") { for _, client := range clients { - work_length, err := client.Current_work_length(true) + work_length, err := client.Current_work.Length(true) if err != nil { continue } @@ -195,7 +169,7 @@ func (cr *ClientController) ReadMany(cx *goweb.Context) { } } else if query.Has("status") { for _, client := range clients { - status, xerr := client.Get_Status(false) + status, xerr := client.Get_New_Status(false) if xerr != nil { continue } @@ -231,25 +205,52 @@ func (cr *ClientController) ReadMany(cx *goweb.Context) { func (cr *ClientController) Update(id string, cx *goweb.Context) { LogRequest(cx.Request) - // Try to authenticate user. - u, err := request.Authenticate(cx.Request) - if err != nil && err.Error() != e.NoAuth { - cx.RespondWithErrorMessage(err.Error(), http.StatusUnauthorized) - return - } + // Gather query params + query := &Query{Li: cx.Request.URL.Query()} - // If no auth was provided, and anonymous read is allowed, use the public user - if u == nil { - if conf.ANON_WRITE == true { - u = &user.User{Uuid: "public"} - } else { - cx.RespondWithErrorMessage(e.NoAuth, http.StatusUnauthorized) + if query.Has("heartbeat") { //handle heartbeat + + cg, done := GetClientGroup(cx) + if done { + return + } + + const MAX_MEMORY = 1024 + + r := cx.Request + worker_status_bytes, err := ioutil.ReadAll(r.Body) + defer r.Body.Close() + + if err != nil { + cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) + return + } + + worker_status := core.WorkerState{} + err = json.Unmarshal(worker_status_bytes, &worker_status) + if err != nil { + err = fmt.Errorf("%s, worker_status_bytes: %s", err, string(worker_status_bytes[:])) + cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) + //cx.Respond(data interface{}, statusCode int, []string{err.Error()}, cx) return } + worker_status.Current_work.Init("Current_work") + + hbmsg, err := core.QMgr.ClientHeartBeat(id, cg, worker_status) + if err != nil { + cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) + } else { + cx.RespondWithData(hbmsg) + } + return + + } + + u, done := GetAuthorizedUser(cx) + if done { + return } - // Gather query params - query := &Query{Li: cx.Request.URL.Query()} if query.Has("subclients") { //update the number of subclients for a proxy if count, err := strconv.Atoi(query.Value("subclients")); err != nil { cx.RespondWithError(http.StatusNotImplemented) @@ -260,7 +261,7 @@ func (cr *ClientController) Update(id string, cx *goweb.Context) { return } if query.Has("suspend") { //resume the suspended client - if err := core.QMgr.SuspendClientByUser(id, u); err != nil { + if err := core.QMgr.SuspendClientByUser(id, u, "request by api call"); err != nil { cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) } else { cx.RespondWithData("client suspended") @@ -308,7 +309,10 @@ func (cr *ClientController) UpdateMany(cx *goweb.Context) { return } if query.Has("suspendall") { //resume the suspended client - num := core.QMgr.SuspendAllClientsByUser(u) + num, err := core.QMgr.SuspendAllClientsByUser(u, "request by api call") + if err != nil { + cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) + } cx.RespondWithData(fmt.Sprintf("%d clients suspended", num)) return } @@ -319,29 +323,30 @@ func (cr *ClientController) UpdateMany(cx *goweb.Context) { // DELETE: /client/{id} func (cr *ClientController) Delete(id string, cx *goweb.Context) { LogRequest(cx.Request) + cx.RespondWithErrorMessage("this functionality has been removed from AWE", http.StatusBadRequest) // Try to authenticate user. - u, err := request.Authenticate(cx.Request) - if err != nil && err.Error() != e.NoAuth { - cx.RespondWithErrorMessage(err.Error(), http.StatusUnauthorized) - return - } + //u, err := request.Authenticate(cx.Request) + //if err != nil && err.Error() != e.NoAuth { + // cx.RespondWithErrorMessage(err.Error(), http.StatusUnauthorized) + // return + //} // If no auth was provided, and anonymous read is allowed, use the public user - if u == nil { - if conf.ANON_DELETE == true { - u = &user.User{Uuid: "public"} - } else { - cx.RespondWithErrorMessage(e.NoAuth, http.StatusUnauthorized) - return - } - } - - if err := core.QMgr.DeleteClientByUser(id, u); err != nil { - cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) - } else { - cx.RespondWithData("client deleted") - } + //if u == nil { + // if conf.ANON_DELETE == true { + // u = &user.User{Uuid: "public"} + // } else { + // cx.RespondWithErrorMessage(e.NoAuth, http.StatusUnauthorized) + // return + // } + //} + + //if err := core.QMgr.DeleteClientByUser(id, u); err != nil { + // cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) + //} else { + // cx.RespondWithData("client deleted") + //} return } diff --git a/lib/controller/jobController.go b/lib/controller/jobController.go index b170b84a..c7688631 100644 --- a/lib/controller/jobController.go +++ b/lib/controller/jobController.go @@ -74,8 +74,8 @@ func (cr *JobController) Create(cx *goweb.Context) { _, has_import := files["import"] _, has_upload := files["upload"] _, has_awf := files["awf"] - _, has_cwl := files["cwl"] // TODO I could overload 'upload' - _, has_job := files["job"] // input data for an CWL workflow + cwl_file, has_cwl := files["cwl"] // TODO I could overload 'upload' + job_file, has_job := files["job"] // input data for an CWL workflow var job *core.Job job = nil @@ -93,19 +93,30 @@ func (cr *JobController) Create(cx *goweb.Context) { if !has_job { logger.Error("job missing") - cx.RespondWithErrorMessage("job missing", http.StatusBadRequest) + cx.RespondWithErrorMessage("cwl job missing", http.StatusBadRequest) return } + workflow_filename := cwl_file.Name + collection := cwl.NewCWL_collection() - // 1) parse job - //job_input, err := cwl.ParseJob(&collection, files["job"].Path) - //if err != nil { - // logger.Error("ParseJob: " + err.Error()) - // cx.RespondWithErrorMessage("error in reading job yaml/json file: "+err.Error(), http.StatusBadRequest) - // return - //} + //1) parse job + + job_stream, err := ioutil.ReadFile(job_file.Path) + if err != nil { + cx.RespondWithErrorMessage("error in reading job yaml/json file: "+err.Error(), http.StatusBadRequest) + return + } + + //job_str := string(job_stream[:]) + + job_input, err := cwl.ParseJob(&job_stream) + if err != nil { + logger.Error("ParseJob: " + err.Error()) + cx.RespondWithErrorMessage("error in reading job yaml/json file: "+err.Error(), http.StatusBadRequest) + return + } //collection.Job_input = job_input @@ -113,7 +124,7 @@ func (cr *JobController) Create(cx *goweb.Context) { logger.Debug(1, "got CWL") // get CWL as byte[] - yamlstream, err := ioutil.ReadFile(files["cwl"].Path) + yamlstream, err := ioutil.ReadFile(cwl_file.Path) if err != nil { logger.Error("CWL error: " + err.Error()) cx.RespondWithErrorMessage("error in reading workflow file: "+err.Error(), http.StatusBadRequest) @@ -123,12 +134,18 @@ func (cr *JobController) Create(cx *goweb.Context) { // convert CWL to string yaml_str := string(yamlstream[:]) - err = cwl.Parse_cwl_document(&collection, yaml_str) + object_array, cwl_version, err := cwl.Parse_cwl_document(yaml_str) if err != nil { - logger.Error("Parse_cwl_document error: " + err.Error()) cx.RespondWithErrorMessage("error in parsing cwl workflow yaml file: "+err.Error(), http.StatusBadRequest) return } + + err = cwl.Add_to_collection(&collection, object_array) + if err != nil { + logger.Error("Parse_cwl_document error: " + err.Error()) + cx.RespondWithErrorMessage("error in adding cwl objects to collection: "+err.Error(), http.StatusBadRequest) + return + } logger.Debug(1, "Parse_cwl_document done") cwl_workflow, ok := collection.Workflows["#main"] @@ -144,11 +161,28 @@ func (cr *JobController) Create(cx *goweb.Context) { } fmt.Println("\n\n\n--------------------------------- Create AWE Job:\n") - //job, err = core.CWL2AWE(_user, files, cwl_workflow, &collection) - //if err != nil { - // cx.RespondWithErrorMessage("Error: "+err.Error(), http.StatusBadRequest) - // return - //} + job, err = core.CWL2AWE(_user, files, job_input, cwl_workflow, &collection) + if err != nil { + cx.RespondWithErrorMessage("Error: "+err.Error(), http.StatusBadRequest) + return + } + + job.IsCWL = true + job.CWL_objects = object_array + job.CwlVersion = cwl_version + //job.CWL_collection = &collection + job.Info.Name = job_file.Name + job.Info.Pipeline = workflow_filename + + //job.CWL_workflow_interface = cwl_workflow + //job.CWL_job_input_interface = job_input + + //job.CWL_workflow = cwl_workflow + job.CWL_job_input = job_input + //job.Set_CWL_workflow_b64(yaml_str) + + //job.Set_CWL_job_input_b64(job_str) + logger.Debug(1, "CWL2AWE done") } else if !has_upload && !has_awf { @@ -179,7 +213,7 @@ func (cr *JobController) Create(cx *goweb.Context) { logger.Debug(3, "job %s got token", job.Id) } - err = job.Save() + err = job.Save() // note that the job only goes into mongo, not into memory yet (EnqueueTasksByJobId is dowing that) if err != nil { cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) return @@ -197,7 +231,11 @@ func (cr *JobController) Create(cx *goweb.Context) { // don't enqueue imports if !has_import { - core.QMgr.EnqueueTasksByJobId(job.Id) + err = core.QMgr.EnqueueTasksByJobId(job.Id) + if err != nil { + cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) + return + } } //cx.RespondWithData(job) diff --git a/lib/controller/queueController.go b/lib/controller/queueController.go index 678360f1..188fd07a 100644 --- a/lib/controller/queueController.go +++ b/lib/controller/queueController.go @@ -111,14 +111,14 @@ func (cr *QueueController) ReadMany(cx *goweb.Context) { for _, client := range client_list { if client.Group == cg.Name { - current_work_array, err := client.Get_current_work(true) + current_work_array, err := client.Current_work.Get_list(true) if err != nil { logger.Error("(queue/ReadMany) %s", err.Error()) continue } for _, wid := range current_work_array { - jid, _ := core.GetJobIdByWorkId(wid) + jid := wid.JobId if job, err := core.GetJob(jid); err == nil { jobs = append(jobs, job) } diff --git a/lib/controller/util.go b/lib/controller/util.go index 9c506415..c00d8012 100644 --- a/lib/controller/util.go +++ b/lib/controller/util.go @@ -5,7 +5,10 @@ import ( "fmt" "github.com/MG-RAST/AWE/lib/conf" "github.com/MG-RAST/AWE/lib/core" + e "github.com/MG-RAST/AWE/lib/errors" "github.com/MG-RAST/AWE/lib/logger" + "github.com/MG-RAST/AWE/lib/request" + "github.com/MG-RAST/AWE/lib/user" "github.com/MG-RAST/golib/goweb" "io" "math/rand" @@ -128,6 +131,7 @@ type resource struct { V string `json:"version"` Time string `json:"server_time"` GitCommitHash string `json:"git_commit_hash"` + Uptime string `json:"uptime"` } func ResourceDescription(cx *goweb.Context) { @@ -163,6 +167,7 @@ func ResourceDescription(cx *goweb.Context) { V: conf.VERSION, Time: time.Now().Format(longDateForm), GitCommitHash: conf.GIT_COMMIT_HASH, + Uptime: time.Since(core.Start_time).String(), } if core.Service == "server" { @@ -311,3 +316,48 @@ func RespondPrivateEnvInHeader(cx *goweb.Context, Envs map[string]string) (err e cx.Respond(nil, http.StatusOK, nil, cx) return } + +func GetAuthorizedUser(cx *goweb.Context) (u *user.User, done bool) { + // Try to authenticate user. + + done = false + + u, err := request.Authenticate(cx.Request) + if err != nil && err.Error() != e.NoAuth { + cx.RespondWithErrorMessage(err.Error(), http.StatusUnauthorized) + done = true + return + } + + // If no auth was provided, and anonymous read is allowed, use the public user + if u == nil { + if conf.ANON_WRITE == true { + u = &user.User{Uuid: "public"} + } else { + cx.RespondWithErrorMessage(e.NoAuth, http.StatusUnauthorized) + done = true + return + } + } + return +} + +func GetClientGroup(cx *goweb.Context) (cg *core.ClientGroup, done bool) { + done = false + cg, err := request.AuthenticateClientGroup(cx.Request) + if err != nil { + if err.Error() == e.NoAuth || err.Error() == e.UnAuth || err.Error() == e.InvalidAuth { + if conf.CLIENT_AUTH_REQ == true { + cx.RespondWithError(http.StatusUnauthorized) + done = true + return + } + } else { + logger.Error("Err@AuthenticateClientGroup: " + err.Error()) + cx.RespondWithError(http.StatusInternalServerError) + done = true + return + } + } + return +} diff --git a/lib/controller/workController.go b/lib/controller/workController.go index cb2b9b9a..ddadfed0 100644 --- a/lib/controller/workController.go +++ b/lib/controller/workController.go @@ -32,6 +32,12 @@ func (cr *WorkController) Options(cx *goweb.Context) { func (cr *WorkController) Read(id string, cx *goweb.Context) { LogRequest(cx.Request) + work_id, err := core.New_Workunit_Unique_Identifier(id) + if err != nil { + cx.RespondWithErrorMessage("error parsing workunit identifier: "+id, http.StatusBadRequest) + return + } + // Gather query params query := &Query{Li: cx.Request.URL.Query()} @@ -66,7 +72,8 @@ func (cr *WorkController) Read(id string, cx *goweb.Context) { } if query.Has("datatoken") { //a client is requesting data token for this job - token, err := core.QMgr.FetchDataToken(id, clientid) + + token, err := core.QMgr.FetchDataToken(work_id, clientid) if err != nil { cx.RespondWithErrorMessage("error in getting token for job "+id, http.StatusBadRequest) return @@ -77,7 +84,7 @@ func (cr *WorkController) Read(id string, cx *goweb.Context) { } if query.Has("privateenv") { //a client is requesting data token for this job - envs, err := core.QMgr.FetchPrivateEnv(id, clientid) + envs, err := core.QMgr.FetchPrivateEnv(work_id, clientid) if err != nil { cx.RespondWithErrorMessage("error in getting token for job "+id+" :"+err.Error(), http.StatusBadRequest) return @@ -105,11 +112,7 @@ func (cr *WorkController) Read(id string, cx *goweb.Context) { } } - jobid, err := core.GetJobIdByWorkId(id) - if err != nil { - cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) - return - } + jobid := work_id.JobId //job, err := core.LoadJob(jobid) //if err != nil { @@ -136,7 +139,7 @@ func (cr *WorkController) Read(id string, cx *goweb.Context) { } if query.Has("report") { //retrieve report: stdout or stderr or worknotes - reportmsg, err := core.QMgr.GetReportMsg(id, query.Value("report")) + reportmsg, err := core.QMgr.GetReportMsg(work_id, query.Value("report")) if err != nil { cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) return @@ -154,7 +157,12 @@ func (cr *WorkController) Read(id string, cx *goweb.Context) { } // Base case respond with workunit in json - workunit, err := core.QMgr.GetWorkById(id) + id_wui, err := core.New_Workunit_Unique_Identifier(id) + if err != nil { + cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) + return + } + workunit, err := core.QMgr.GetWorkById(id_wui) if err != nil { cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) return @@ -301,10 +309,11 @@ func (cr *WorkController) ReadMany(cx *goweb.Context) { if err != nil { if err.Error() != e.QueueEmpty && err.Error() != e.QueueSuspend && err.Error() != e.NoEligibleWorkunitFound && err.Error() != e.ClientNotFound && err.Error() != e.ClientSuspended { if !strings.Contains(err.Error(), "Too many work requests") { - logger.Error("Err@work_ReadMany:core.QMgr.GetWorkByFCFS(): " + err.Error() + ";client=" + clientid) + logger.Error("Err@work_ReadMany:core.QMgr.GetWorkByFCFS(): %s;client=%s", err.Error(), clientid) } } - logger.Debug(3, fmt.Sprintf("Error in CheckoutWorkunits: clientid=%s;available=%d;error=%s", clientid, availableBytes, err.Error())) + err = fmt.Errorf("(ReadMany GET /work) CheckoutWorkunits returns: clientid=%s;available=%d;error=%s", clientid, availableBytes, err.Error()) + logger.Error(err.Error()) cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest) return } @@ -327,6 +336,13 @@ func (cr *WorkController) ReadMany(cx *goweb.Context) { // PUT: /work/{id} -> status update func (cr *WorkController) Update(id string, cx *goweb.Context) { LogRequest(cx.Request) + + work_id, err := core.New_Workunit_Unique_Identifier(id) + if err != nil { + cx.RespondWithErrorMessage("error parsing workunit identifier: "+id, http.StatusBadRequest) + return + } + // Gather query params query := &Query{Li: cx.Request.URL.Query()} if !query.Has("client") { @@ -366,7 +382,7 @@ func (cr *WorkController) Update(id string, cx *goweb.Context) { } if query.Has("status") && query.Has("client") { //notify execution result: "done" or "fail" - notice := core.Notice{WorkId: id, Status: query.Value("status"), ClientId: query.Value("client"), Notes: ""} + notice := core.Notice{WorkId: work_id, Status: query.Value("status"), ClientId: query.Value("client"), Notes: ""} if query.Has("computetime") { if comptime, err := strconv.Atoi(query.Value("computetime")); err == nil { notice.ComputeTime = comptime @@ -375,7 +391,7 @@ func (cr *WorkController) Update(id string, cx *goweb.Context) { if query.Has("report") { // if "report" is specified in query, parse performance statistics or errlog if _, files, err := ParseMultipartForm(cx.Request); err == nil { if _, ok := files["perf"]; ok { - core.QMgr.FinalizeWorkPerf(id, files["perf"].Path) + core.QMgr.FinalizeWorkPerf(work_id, files["perf"].Path) } for _, log := range conf.WORKUNIT_LOGS { if _, ok := files[log]; ok { @@ -397,7 +413,7 @@ func (cr *WorkController) Update(id string, cx *goweb.Context) { } } // move / save log file - core.QMgr.SaveStdLog(id, log, files[log].Path) + core.QMgr.SaveStdLog(work_id, log, files[log].Path) } } } diff --git a/lib/core/client.go b/lib/core/client.go index 133f27a7..0ee77077 100644 --- a/lib/core/client.go +++ b/lib/core/client.go @@ -9,45 +9,63 @@ import ( "time" ) -const ( - CLIENT_STAT_ACTIVE_BUSY = "active-busy" - CLIENT_STAT_ACTIVE_IDLE = "active-idle" - CLIENT_STAT_SUSPEND = "suspend" - CLIENT_STAT_DELETED = "deleted" -) +// states +//online bool // server defined +//suspended bool // server defined +//busy bool // this is the Worker type Client struct { - coAckChannel chan CoAck `bson:"-" json:"-"` //workunit checkout item including data and err (qmgr.Handler -> WorkController) - RWMutex - Id string `bson:"id" json:"id"` // this is a uuid (the only relevant identifier) - Name string `bson:"name" json:"name"` // this can be anything you want - Group string `bson:"group" json:"group"` - User string `bson:"user" json:"user"` - Domain string `bson:"domain" json:"domain"` - InstanceId string `bson:"instance_id" json:"instance_id"` // Openstack specific - InstanceType string `bson:"instance_type" json:"instance_type"` // Openstack specific - Host string `bson:"host" json:"host"` // deprecated - Hostname string `bson:"hostname" json:"hostname"` - Host_ip string `bson:"host_ip" json:"host_ip"` // Host can be physical machine or VM, whatever is helpful for management - CPUs int `bson:"cores" json:"cores"` - Apps []string `bson:"apps" json:"apps"` - RegTime time.Time `bson:"regtime" json:"regtime"` - LastCompleted time.Time `bson:"lastcompleted" json:"lastcompleted"` // time of last time a job was completed (can be used to compute idle time) - Serve_time string `bson:"serve_time" json:"serve_time"` - //Idle_time int `bson:"idle_time" json:"idle_time"` - Status string `bson:"Status" json:"Status"` - Total_checkout int `bson:"total_checkout" json:"total_checkout"` - Total_completed int `bson:"total_completed" json:"total_completed"` - Total_failed int `bson:"total_failed" json:"total_failed"` - Current_work map[string]bool `bson:"current_work" json:"current_work"` // the bool in the mapping is deprecated. It used to indicate completed work that could not be returned to server - Skip_work []string `bson:"skip_work" json:"skip_work"` - Last_failed int `bson:"-" json:"-"` - Tag bool `bson:"-" json:"-"` - Proxy bool `bson:"proxy" json:"proxy"` - SubClients int `bson:"subclients" json:"subclients"` - GitCommitHash string `bson:"git_commit_hash" json:"git_commit_hash"` - Version string `bson:"version" json:"version"` + coAckChannel chan CoAck `bson:"-" json:"-"` //workunit checkout item including data and err (qmgr.Handler -> WorkController) + RWMutex `bson:"-" json:"-"` + WorkerRuntime `bson:",inline" json:",inline"` + WorkerState `bson:",inline" json:",inline"` + RegTime time.Time `bson:"regtime" json:"regtime"` + LastCompleted time.Time `bson:"lastcompleted" json:"lastcompleted"` // time of last time a job was completed (can be used to compute idle time) + Serve_time string `bson:"serve_time" json:"serve_time"` + Total_checkout int `bson:"total_checkout" json:"total_checkout"` + Total_completed int `bson:"total_completed" json:"total_completed"` + Total_failed int `bson:"total_failed" json:"total_failed"` + Skip_work []string `bson:"skip_work" json:"skip_work"` + Last_failed int `bson:"-" json:"-"` + Tag bool `bson:"-" json:"-"` + Proxy bool `bson:"proxy" json:"proxy"` + SubClients int `bson:"subclients" json:"subclients"` + Online bool `bson:"online" json:"online"` // a state + Suspended bool `bson:"suspended" json:"suspended"` // a state + Suspend_reason string `bson:"suspend_reason" json:"suspend_reason"` // a state + Status string `bson:"Status" json:"Status"` // 1) suspended? 2) busy ? 3) online (call is idle) 4) offline + Assigned_work *WorkunitList `bson:"assigned_work" json:"assigned_work"` // this is for exporting into json +} + +// worker info that does not change at runtime +type WorkerRuntime struct { + Id string `bson:"id" json:"id"` // this is a uuid (the only relevant identifier) + Name string `bson:"name" json:"name"` // this can be anything you want + Group string `bson:"group" json:"group"` + User string `bson:"user" json:"user"` + Domain string `bson:"domain" json:"domain"` + InstanceId string `bson:"instance_id" json:"instance_id"` // Openstack specific + InstanceType string `bson:"instance_type" json:"instance_type"` // Openstack specific + Host string `bson:"host" json:"host"` // deprecated + Hostname string `bson:"hostname" json:"hostname"` + Host_ip string `bson:"host_ip" json:"host_ip"` // Host can be physical machine or VM, whatever is helpful for management + CPUs int `bson:"cores" json:"cores"` + Apps []string `bson:"apps" json:"apps"` + GitCommitHash string `bson:"git_commit_hash" json:"git_commit_hash"` + Version string `bson:"version" json:"version"` +} + +// changes at runtime +type WorkerState struct { + Busy bool `bson:"busy" json:"busy"` // a state + Current_work *WorkunitList `bson:"current_work" json:"current_work"` +} + +func NewWorkerState() (ws *WorkerState) { + ws = &WorkerState{} + ws.Current_work = NewWorkunitList() + return } // invoked by NewClient or manually after unmarshalling @@ -70,9 +88,10 @@ func (client *Client) Init() { if client.Skip_work == nil { client.Skip_work = []string{} } - if client.Current_work == nil { - client.Current_work = map[string]bool{} - } + + client.Assigned_work.Init("Assigned_work") + + client.Current_work.Init("Current_work") } @@ -81,10 +100,14 @@ func NewClient() (client *Client) { Total_checkout: 0, Total_completed: 0, Total_failed: 0, + Serve_time: "0", + Last_failed: 0, + Suspended: false, + Status: "offline", + Assigned_work: NewWorkunitList(), - Serve_time: "0", - Last_failed: 0, - Status: CLIENT_STAT_ACTIVE_IDLE, + WorkerRuntime: WorkerRuntime{}, + WorkerState: *NewWorkerState(), } client.Init() @@ -110,7 +133,8 @@ func NewProfileClient(filepath string) (client *Client, err error) { err = json.Unmarshal(jsonstream, client) if err != nil { err = fmt.Errorf("failed to unmashal json stream for client profile (error: %s) (file: %s) json: %s", err.Error(), filepath, string(jsonstream[:])) - return nil, err + client = nil + return } client.Init() @@ -118,6 +142,16 @@ func NewProfileClient(filepath string) (client *Client, err error) { return } +func (this *Client) Add(workid Workunit_Unique_Identifier) (err error) { + err = this.Assigned_work.Add(workid) + if err != nil { + return + } + + this.Total_checkout += 1 // TODO add lock ???? + return +} + func (cl *Client) Get_Ack() (ack CoAck, err error) { start_time := time.Now() timeout := make(chan bool, 1) @@ -139,18 +173,17 @@ func (cl *Client) Get_Ack() (ack CoAck, err error) { return } -func (cl *Client) Append_Skip_work(workid string, write_lock bool) (err error) { +func (cl *Client) Append_Skip_work(workid Workunit_Unique_Identifier, write_lock bool) (err error) { if write_lock { err = cl.LockNamed("Append_Skip_work") if err != nil { return } + defer cl.Unlock() } - cl.Skip_work = append(cl.Skip_work, workid) - if write_lock { - cl.Unlock() - } + cl.Skip_work = append(cl.Skip_work, workid.String()) + return } @@ -172,7 +205,8 @@ func (cl *Client) Get_Id(do_read_lock bool) (s string, err error) { return } -func (cl *Client) Get_Status(do_read_lock bool) (s string, err error) { +// this function should not be used internally, this is only for backwards-compatibility and human readability +func (cl *Client) Get_New_Status(do_read_lock bool) (s string, err error) { if do_read_lock { read_lock, xerr := cl.RLockNamed("Get_Status") if xerr != nil { @@ -185,7 +219,20 @@ func (cl *Client) Get_Status(do_read_lock bool) (s string, err error) { return } -func (cl *Client) Set_Status(s string, write_lock bool) (err error) { +func (cl *Client) Get_Group(do_read_lock bool) (g string, err error) { + if do_read_lock { + read_lock, xerr := cl.RLockNamed("Get_Status") + if xerr != nil { + err = xerr + return + } + defer cl.RUnlockNamed(read_lock) + } + g = cl.Group + return +} + +func (cl *Client) Set_Status_deprecated(s string, write_lock bool) (err error) { if write_lock { err = cl.LockNamed("Set_Status") if err != nil { @@ -198,6 +245,123 @@ func (cl *Client) Set_Status(s string, write_lock bool) (err error) { return } +func (cl *Client) Update_Status(write_lock bool) (err error) { + if write_lock { + err = cl.LockNamed("Update_Status") + if err != nil { + return + } + defer cl.Unlock() + } + + // 1) suspended? 2) busy ? 3) online (call is idle) 4) offline + + if cl.Suspended { + cl.Status = "suspended" + return + } + + if cl.Busy { + cl.Status = "busy" + return + } + + if cl.Online { + cl.Status = "online" + return + } + + cl.Status = "offline" + return +} + +func (cl *Client) Set_Suspended(s bool, reason string, write_lock bool) (err error) { + if write_lock { + err = cl.LockNamed("Set_Suspended") + if err != nil { + return + } + defer cl.Unlock() + } + + if cl.Suspended != s { + cl.Suspended = s + if s { + if reason == "" { + panic("suspending without providing eason not allowed !") + } + } + cl.Suspend_reason = reason + cl.Update_Status(false) + } + return +} + +func (cl *Client) Suspend(reason string, write_lock bool) (err error) { + return cl.Set_Suspended(true, reason, write_lock) +} + +func (cl *Client) Resume(write_lock bool) (err error) { + return cl.Set_Suspended(false, "", write_lock) +} + +func (cl *Client) Get_Suspended(do_read_lock bool) (s bool, err error) { + if do_read_lock { + read_lock, xerr := cl.RLockNamed("Get_Suspended") + if xerr != nil { + err = xerr + return + } + defer cl.RUnlockNamed(read_lock) + } + s = cl.Suspended + return +} + +func (cl *Client) Set_Online(o bool, write_lock bool) (err error) { + if write_lock { + err = cl.LockNamed("Set_Online") + if err != nil { + return + } + defer cl.Unlock() + } + + if cl.Online != o { + cl.Online = o + cl.Update_Status(false) + } + return +} + +func (cl *Client) Set_Busy(b bool, do_write_lock bool) (err error) { + if do_write_lock { + err = cl.LockNamed("Set_Busy") + if err != nil { + return + } + defer cl.Unlock() + } + if cl.Busy != b { + cl.Busy = b + cl.Update_Status(false) + } + return +} + +func (cl *Client) Get_Busy(do_read_lock bool) (b bool, err error) { + if do_read_lock { + read_lock, xerr := cl.RLockNamed("Get_Busy") + if xerr != nil { + err = xerr + return + } + defer cl.RUnlockNamed(read_lock) + } + b = cl.Busy + return +} + func (cl *Client) Get_Total_checkout() (count int, err error) { read_lock, err := cl.RLockNamed("Get_Total_checkout") if err != nil { @@ -290,119 +454,35 @@ func (cl *Client) Get_Last_failed() (count int, err error) { return } -func (cl *Client) Current_work_delete(workid string, write_lock bool) (err error) { - if write_lock { - err = cl.LockNamed("Current_work_delete") - defer cl.Unlock() - } - delete(cl.Current_work, workid) - cw_length, err := cl.Current_work_length(false) - if err != nil { - return - } - - if cw_length == 0 && cl.Status == CLIENT_STAT_ACTIVE_BUSY { - cl.Status = CLIENT_STAT_ACTIVE_IDLE - } - return -} - -func (cl *Client) Current_work_has(workid string) (ok bool, err error) { - - err = cl.LockNamed("Current_work_has") - defer cl.Unlock() - - _, ok = cl.Current_work[workid] - - return -} - -func (cl *Client) Get_current_work(do_read_lock bool) (current_work_ids []string, err error) { - current_work_ids = []string{} - if do_read_lock { - read_lock, xerr := cl.RLockNamed("Get_current_work") - if xerr != nil { - err = xerr - return - } - defer cl.RUnlockNamed(read_lock) - } - for id := range cl.Current_work { - current_work_ids = append(current_work_ids, id) - } - return -} - -func (cl *Client) Set_current_work(current_work_ids []string, do_write_lock bool) (err error) { - current_work_ids = []string{} - if do_write_lock { - err = cl.LockNamed("Set_current_work") - if err != nil { - return - } - defer cl.Unlock() - } - - cl.Current_work = make(map[string]bool) - for _, workid := range current_work_ids { - cl.Current_work[workid] = true - } - - return -} +//func (cl *Client) Set_current_work(current_work_ids []string, do_write_lock bool) (err error) { +// current_work_ids = []string{} +// if do_write_lock { +// err = cl.LockNamed("Set_current_work") +// if err != nil { +// return +// } +// defer cl.Unlock() +// } +// +// cl.Assigned_work = make(map[string]bool) +// for _, workid := range current_work_ids { +// cl.Assigned_work[workid] = true +// } +// +// return +//} // TODO: Wolfgang: Can we use delete instead ? -//func (cl *Client) Current_work_false(workid string) (err error) { -// err = cl.LockNamed("Current_work_false") +//func (cl *Client) Assigned_work_false(workid string) (err error) { +// err = cl.LockNamed("Assigned_work_false") // if err != nil { // return // } // defer cl.Unlock() -// cl.Current_work[workid] = false +// cl.Assigned_work[workid] = false // return //} -// lock always -func (cl *Client) Add_work(workid string) (err error) { - - err = cl.LockNamed("Add_work") - if err != nil { - return - } - defer cl.Unlock() - - cl.Current_work[workid] = true - cl.Total_checkout += 1 - return -} - -func (cl *Client) Current_work_length(lock bool) (clength int, err error) { - if lock { - read_lock, xerr := cl.RLockNamed("Current_work_length") - if xerr != nil { - err = xerr - return - } - defer cl.RUnlockNamed(read_lock) - } - clength = len(cl.Current_work) - - return -} - -func (cl *Client) IsBusy(lock bool) (busy bool, err error) { - cw_length, err := cl.Current_work_length(lock) - if err != nil { - return - } - if cw_length > 0 { - busy = true - return - } - busy = false - return -} - func (cl *Client) Marshal() (result []byte, err error) { read_lock, err := cl.RLockNamed("Marshal") if err != nil { diff --git a/lib/core/command.go b/lib/core/command.go index bc03156e..69406545 100644 --- a/lib/core/command.go +++ b/lib/core/command.go @@ -9,16 +9,16 @@ package core //} type Command struct { - Name string `bson:"name" json:"name"` - Args string `bson:"args" json:"args"` - ArgsArray []string `bson:"args_array" json:"args_array"` // use this instead of Args, which is just a string - Dockerimage string `bson:"Dockerimage" json:"Dockerimage"` // for Shock (TODO rename this !) - DockerPull string `bson:"dockerPull" json:"dockerPull"` // docker pull - Cmd_script []string `bson:"cmd_script" json:"cmd_script"` - Environ Envs `bson:"environ" json:"environ"` - HasPrivateEnv bool `bson:"has_private_env" json:"has_private_env"` - Description string `bson:"description" json:"description"` - ParsedArgs []string `bson:"-" json:"-"` + Name string `bson:"name" json:"name" mapstructure:"name"` + Args string `bson:"args" json:"args" mapstructure:"args"` + ArgsArray []string `bson:"args_array" json:"args_array" mapstructure:"args_array"` // use this instead of Args, which is just a string + Dockerimage string `bson:"Dockerimage" json:"Dockerimage" mapstructure:"Dockerimage"` // for Shock (TODO rename this !) + DockerPull string `bson:"dockerPull" json:"dockerPull" mapstructure:"dockerPull"` // docker pull + Cmd_script []string `bson:"cmd_script" json:"cmd_script" mapstructure:"cmd_script"` + Environ Envs `bson:"environ" json:"environ" mapstructure:"environ"` + HasPrivateEnv bool `bson:"has_private_env" json:"has_private_env" mapstructure:"has_private_env"` + Description string `bson:"description" json:"description" mapstructure:"description"` + ParsedArgs []string `bson:"-" json:"-" mapstructure:"-"` Local bool // indicates local execution, i.e. working directory is same as current working directory (do not delete !) } diff --git a/lib/core/core.go b/lib/core/core.go index 21513a56..06125aff 100644 --- a/lib/core/core.go +++ b/lib/core/core.go @@ -27,22 +27,28 @@ var ( ProxyWorkChan chan bool Server_UUID string JM *JobMap + Start_time time.Time ) +type BaseResponse struct { + Status int `json:"status"` + Error []string `json:"error"` +} + type StandardResponse struct { - S int `json:"status"` - D interface{} `json:"data"` - E []string `json:"error"` + Status int `json:"status"` + Data interface{} `json:"data"` + Error []string `json:"error"` } func InitResMgr(service string) { if service == "server" { QMgr = NewServerMgr() } else if service == "proxy" { - QMgr = NewProxyMgr() + //QMgr = NewProxyMgr() } Service = service - + Start_time = time.Now() } func SetClientProfile(profile *Client) { @@ -68,7 +74,7 @@ type CoReq struct { } type Notice struct { - WorkId string + WorkId Workunit_Unique_Identifier Status string ClientId string ComputeTime int @@ -91,7 +97,7 @@ type FormFile struct { //heartbeat response from awe-server to awe-worker //used for issue operation request to client, e.g. discard suspended workunits -type HBmsg map[string]string //map[op]obj1,obj2 e.g. map[discard]=work1,work2 +type HeartbeatInstructions map[string]string //map[op]obj1,obj2 e.g. map[discard]=work1,work2 func CreateJobUpload(u *user.User, files FormFiles) (job *Job, err error) { @@ -306,11 +312,7 @@ func JobDepToJob(jobDep *JobDep) (job *Job, err error) { return } - task, xerr := NewTask(job, taskDep.Id) - if xerr != nil { - err = xerr - return - } + task := NewTask(job, taskDep.Id) _, err = task.Init(job) if err != nil { @@ -370,7 +372,7 @@ func JobDepToJob(jobDep *JobDep) (job *Job, err error) { //misc -func GetJobIdByTaskId(taskid string) (jobid string, err error) { +func GetJobIdByTaskId_deprecated(taskid string) (jobid string, err error) { // job_id is embedded in task struct parts := strings.Split(taskid, "_") if len(parts) == 2 { return parts[0], nil @@ -378,7 +380,7 @@ func GetJobIdByTaskId(taskid string) (jobid string, err error) { return "", errors.New("invalid task id: " + taskid) } -func GetJobIdByWorkId(workid string) (jobid string, err error) { +func GetJobIdByWorkId_deprecated(workid string) (jobid string, err error) { parts := strings.Split(workid, "_") if len(parts) == 3 { jobid = parts[0] @@ -388,7 +390,7 @@ func GetJobIdByWorkId(workid string) (jobid string, err error) { return } -func GetTaskIdByWorkId(workid string) (taskid string, err error) { +func GetTaskIdByWorkId_deprecated(workid string) (taskid string, err error) { parts := strings.Split(workid, "_") if len(parts) == 3 { return fmt.Sprintf("%s_%s", parts[0], parts[1]), nil @@ -406,8 +408,8 @@ func IsFirstTask(taskid string) bool { return false } -//update job state to "newstate" only if the current state is in one of the "oldstates" -func UpdateJobState(jobid string, newstate string, oldstates []string) (err error) { +//update job state to "newstate" only if the current state is in one of the "oldstates" // TODO make this a job.SetState function +func UpdateJobState_deprecated(jobid string, newstate string, oldstates []string) (err error) { job, err := GetJob(jobid) if err != nil { return @@ -426,11 +428,13 @@ func UpdateJobState(jobid string, newstate string, oldstates []string) (err erro } } if !matched { - return errors.New("old state not matching one of the required ones") - } - if err := job.SetState(newstate); err != nil { - return err + oldstates_str := strings.Join(oldstates, ",") + err = fmt.Errorf("(UpdateJobState) old state %s does not match one of the required ones (required: %s)", job_state, oldstates_str) + return } + //if err := job.SetState(newstate); err != nil { + // return err + //} return } @@ -446,7 +450,7 @@ func contains(list []string, elem string) bool { //functions for REST API communication (=deprecated=) //notify AWE server a workunit is finished with status either "failed" or "done", and with perf statistics if "done" func NotifyWorkunitProcessed(work *Workunit, perf *WorkPerf) (err error) { - target_url := fmt.Sprintf("%s/work/%s?status=%s&client=%s", conf.SERVER_URL, work.Id, work.State, Self.Id) + target_url := fmt.Sprintf("%s/work/%s?workid=%s&jobid=%s&status=%s&client=%s", conf.SERVER_URL, work.Id, work.TaskId, work.JobId, work.State, Self.Id) argv := []string{} argv = append(argv, "-X") @@ -470,7 +474,7 @@ func NotifyWorkunitProcessed(work *Workunit, perf *WorkPerf) (err error) { } func NotifyWorkunitProcessedWithLogs(work *Workunit, perf *WorkPerf, sendstdlogs bool) (response *StandardResponse, err error) { - target_url := fmt.Sprintf("%s/work/%s?status=%s&client=%s&computetime=%d", conf.SERVER_URL, work.Id, work.State, Self.Id, work.ComputeTime) + target_url := fmt.Sprintf("%s/work/%s?status=%s&client=%s&computetime=%d", conf.SERVER_URL, work.String(), work.State, Self.Id, work.ComputeTime) form := httpclient.NewForm() hasreport := false if work.State == WORK_STAT_DONE && perf != nil { @@ -530,8 +534,8 @@ func NotifyWorkunitProcessedWithLogs(work *Workunit, perf *WorkPerf, sendstdlogs err = fmt.Errorf("(NotifyWorkunitProcessedWithLogs) failed to marshal response:\"%s\"", jsonstream) return } - if len(response.E) > 0 { - err = errors.New(strings.Join(response.E, ",")) + if len(response.Error) > 0 { + err = errors.New(strings.Join(response.Error, ",")) return } @@ -545,20 +549,25 @@ func PushOutputData(work *Workunit) (size int64, err error) { var local_filepath string //local file name generated by the cmd var file_path string //file name to be uploaded to shock + work_path, xerr := work.Path() + if xerr != nil { + err = xerr + return + } if io.Directory != "" { - local_filepath = fmt.Sprintf("%s/%s/%s", work.Path(), io.Directory, name) + local_filepath = fmt.Sprintf("%s/%s/%s", work_path, io.Directory, name) //if specified, rename the local file name to the specified shock node file name //otherwise use the local name as shock file name file_path = local_filepath if io.ShockFilename != "" { - file_path = fmt.Sprintf("%s/%s/%s", work.Path(), io.Directory, io.ShockFilename) + file_path = fmt.Sprintf("%s/%s/%s", work_path, io.Directory, io.ShockFilename) os.Rename(local_filepath, file_path) } } else { - local_filepath = fmt.Sprintf("%s/%s", work.Path(), name) + local_filepath = fmt.Sprintf("%s/%s", work_path, name) file_path = local_filepath if io.ShockFilename != "" { - file_path = fmt.Sprintf("%s/%s", work.Path(), io.ShockFilename) + file_path = fmt.Sprintf("%s/%s", work_path, io.ShockFilename) os.Rename(local_filepath, file_path) } } @@ -588,7 +597,7 @@ func PushOutputData(work *Workunit) (size int64, err error) { //upload attribute file to shock IF attribute file is specified in outputs AND it is found in local directory. var attrfile_path string = "" if io.AttrFile != "" { - attrfile_path = fmt.Sprintf("%s/%s", work.Path(), io.AttrFile) + attrfile_path = fmt.Sprintf("%s/%s", work_path, io.AttrFile) if fi, err := os.Stat(attrfile_path); err != nil || fi.Size() == 0 { attrfile_path = "" } @@ -655,13 +664,21 @@ func getPerfFilePath(work *Workunit, perfstat *WorkPerf) (reportPath string, err if err != nil { return reportPath, err } - reportPath = fmt.Sprintf("%s/%s.perf", work.Path(), work.Id) + work_path, err := work.Path() + if err != nil { + return + } + reportPath = fmt.Sprintf("%s/%s.perf", work_path, work.Id) err = ioutil.WriteFile(reportPath, []byte(perfJsonstream), 0644) return } func getStdOutPath(work *Workunit) (stdoutFilePath string, err error) { - stdoutFilePath = fmt.Sprintf("%s/%s", work.Path(), conf.STDOUT_FILENAME) + work_path, err := work.Path() + if err != nil { + return + } + stdoutFilePath = fmt.Sprintf("%s/%s", work_path, conf.STDOUT_FILENAME) fi, err := os.Stat(stdoutFilePath) if err != nil { return stdoutFilePath, err @@ -673,7 +690,11 @@ func getStdOutPath(work *Workunit) (stdoutFilePath string, err error) { } func getStdErrPath(work *Workunit) (stderrFilePath string, err error) { - stderrFilePath = fmt.Sprintf("%s/%s", work.Path(), conf.STDERR_FILENAME) + work_path, err := work.Path() + if err != nil { + return + } + stderrFilePath = fmt.Sprintf("%s/%s", work_path, conf.STDERR_FILENAME) fi, err := os.Stat(stderrFilePath) if err != nil { return stderrFilePath, err @@ -685,7 +706,11 @@ func getStdErrPath(work *Workunit) (stderrFilePath string, err error) { } func getWorkNotesPath(work *Workunit) (worknotesFilePath string, err error) { - worknotesFilePath = fmt.Sprintf("%s/%s", work.Path(), conf.WORKNOTES_FILENAME) + work_path, err := work.Path() + if err != nil { + return + } + worknotesFilePath = fmt.Sprintf("%s/%s", work_path, conf.WORKNOTES_FILENAME) if len(work.Notes) == 0 { return worknotesFilePath, errors.New("work notes empty") } @@ -809,16 +834,19 @@ func createOrUpdate_deprecated(opts shock.Opts, host string, nodeid string, toke func GetJob(id string) (job *Job, err error) { job, ok, err := JM.Get(id, true) if err != nil { + err = fmt.Errorf("(GetJob) JM.Get failed: %s", err.Error()) return } if !ok { // load job if not already in memory job, err = LoadJob(id) if err != nil { + err = fmt.Errorf("(GetJob) LoadJob failed: %s", err.Error()) return } err = JM.Add(job) if err != nil { + err = fmt.Errorf("(GetJob) JM.Add failed: %s", err.Error()) return } } diff --git a/lib/core/cqmgr.go b/lib/core/cqmgr.go index 907e9e69..f0f192e8 100644 --- a/lib/core/cqmgr.go +++ b/lib/core/cqmgr.go @@ -8,8 +8,11 @@ import ( "github.com/MG-RAST/AWE/lib/logger" "github.com/MG-RAST/AWE/lib/logger/event" "github.com/MG-RAST/AWE/lib/user" + //"github.com/davecgh/go-spew/spew" "gopkg.in/mgo.v2/bson" "os" + "runtime" + "runtime/pprof" "strings" "time" ) @@ -77,7 +80,7 @@ func (qm *CQMgr) RemoveClient(id string, lock bool) (err error) { } //now client must be gone as tag set to false 30 seconds ago and no heartbeat received thereafter - logger.Event(event.CLIENT_UNREGISTER, "clientid="+client.Id+";name="+client.Name) + logger.Event(event.CLIENT_UNREGISTER, "clientid="+client.Id) //requeue unfinished workunits associated with the failed client err = qm.ReQueueWorkunitByClient(client, true) if err != nil { @@ -88,30 +91,30 @@ func (qm *CQMgr) RemoveClient(id string, lock bool) (err error) { return } -func (qm *CQMgr) DeleteClient(client *Client) (err error) { - err = qm.ClientStatusChange(client, CLIENT_STAT_DELETED, true) - return -} +//func (qm *CQMgr) DeleteClient(client *Client) (err error) { +// err = qm.ClientStatusChange(client, CLIENT_STAT_DELETED, true) +// return +//} -func (qm *CQMgr) DeleteClientById(id string) (err error) { - err = qm.ClientIdStatusChange(id, CLIENT_STAT_DELETED, true) - return -} +//func (qm *CQMgr) DeleteClientById(id string) (err error) { +// err = qm.ClientIdStatusChange(id, CLIENT_STAT_DELETED, true) +// return +//} -func (qm *CQMgr) ClientIdStatusChange(id string, new_status string, client_write_lock bool) (err error) { - client, ok, err := qm.clientMap.Get(id, true) - if err != nil { - return - } - if ok { - err = client.Set_Status(new_status, client_write_lock) - return - } - return errors.New(e.ClientNotFound) -} +// func (qm *CQMgr) ClientIdStatusChange_deprecated(id string, new_status string, client_write_lock bool) (err error) { +// client, ok, err := qm.clientMap.Get(id, true) +// if err != nil { +// return +// } +// if ok { +// //err = client.Set_Status(new_status, client_write_lock) +// return +// } +// returnerrors.New(e.ClientNotFound) +//} -func (qm *CQMgr) ClientStatusChange(client *Client, new_status string, client_write_lock bool) (err error) { - client.Set_Status(new_status, client_write_lock) +func (qm *CQMgr) ClientStatusChange_deprecated(client *Client, new_status string, client_write_lock bool) (err error) { + //client.Set_Status(new_status, client_write_lock) return } @@ -148,8 +151,12 @@ func (qm *CQMgr) CheckClient(client *Client) (ok bool, err error) { } defer client.Unlock() + logger.Debug(3, "(CheckClient) client: %s", client.Id) + if client.Tag == true { // *** Client is active + client.Online = true + logger.Debug(3, "(CheckClient) client %s active", client.Id) client.Tag = false total_minutes := int(time.Now().Sub(client.RegTime).Minutes()) @@ -157,52 +164,81 @@ func (qm *CQMgr) CheckClient(client *Client) (ok bool, err error) { minutes := total_minutes % 60 client.Serve_time = fmt.Sprintf("%dh%dm", hours, minutes) - current_work, xerr := client.Get_current_work(false) + //spew.Dump(client) + + current_work, xerr := client.WorkerState.Current_work.Get_list(false) if xerr != nil { logger.Error("(CheckClient) Get_current_work: %s", xerr.Error()) return } + + logger.Debug(3, "(CheckClient) client %s has %d workunits", client.Id, len(current_work)) + for _, work_id := range current_work { + logger.Debug(3, "(CheckClient) client %s has work %s", client.Id, work_id) work, ok, zerr := qm.workQueue.all.Get(work_id) if zerr != nil { + logger.Warning("(CheckClient) failed getting work %s from workQueue: %s", work_id, zerr.Error()) continue } if !ok { + logger.Error("(CheckClient) work %s not in workQueue", work_id) // this could happen wehen server was restarted but worker does not know yet continue } + logger.Debug(3, "(CheckClient) work.State: %s", work.State) if work.State == WORK_STAT_RESERVED { - qm.workQueue.StatusChange(work_id, work, WORK_STAT_CHECKOUT) + qm.workQueue.StatusChange(work_id, work, WORK_STAT_CHECKOUT, "") } } } else { - // *** Client is NOT active + client.Online = false - //delete the client from client map - //qm.RemoveClient(client.Id) ok = false } return } func (qm *CQMgr) ClientChecker() { - logger.Info("ClientChecker starting") + logger.Info("(ClientChecker) starting") + if conf.CPUPROFILE != "" { + f, err := os.Create(conf.CPUPROFILE) + if err != nil { + logger.Error(err.Error()) + } + pprof.StartCPUProfile(f) + defer pprof.StopCPUProfile() + } + for { time.Sleep(30 * time.Second) + + if conf.MEMPROFILE != "" { + f, err := os.Create(conf.MEMPROFILE) + if err != nil { + logger.Error("could not create memory profile: ", err) + } + runtime.GC() // get up-to-date statistics + if err := pprof.WriteHeapProfile(f); err != nil { + logger.Error("could not write memory profile: ", err) + } + f.Close() + } + logger.Debug(3, "(ClientChecker) time to update client list....") delete_clients := []string{} client_list, xerr := qm.clientMap.GetClients() // this uses a list of pointers to prevent long locking of the CLientMap if xerr != nil { - logger.Error("ClientChecker/GetClients: %s", xerr.Error()) + logger.Error("(ClientChecker) GetClients: %s", xerr.Error()) continue } logger.Debug(3, "(ClientChecker) check %d clients", len(client_list)) for _, client := range client_list { ok, xerr := qm.CheckClient(client) if xerr != nil { - logger.Error("ClientChecker/CheckClient: %s", xerr.Error()) + logger.Error("(ClientChecker) CheckClient: %s", xerr.Error()) continue } if !ok { @@ -226,7 +262,7 @@ func (qm *CQMgr) DeleteClients(delete_clients []string) { } -func (qm *CQMgr) ClientHeartBeat(id string, cg *ClientGroup) (hbmsg HBmsg, err error) { +func (qm *CQMgr) ClientHeartBeat(id string, cg *ClientGroup, workerstate WorkerState) (hbmsg HeartbeatInstructions, err error) { hbmsg = make(map[string]string, 1) client, ok, xerr := qm.GetClient(id, true) if xerr != nil { @@ -250,11 +286,16 @@ func (qm *CQMgr) ClientHeartBeat(id string, cg *ClientGroup) (hbmsg HBmsg, err e return nil, errors.New(e.ClientGroupBadName) } client.Tag = true + client.Set_Online(true, false) - logger.Debug(3, "HeartBeatFrom:"+"clientid="+id+",name="+client.Name) + workerstate.Current_work.FillMap() // fix struct by moving values from Data array into internal map (was not exported) + + client.WorkerState = workerstate // TODO could do a comparsion with assigned state here + + logger.Debug(3, "HeartBeatFrom:"+"clientid="+id) //get suspended workunit that need the client to discard - current_work, xerr := client.Get_current_work(false) + current_work, xerr := client.Current_work.Get_list(false) suspended := []string{} for _, work_id := range current_work { @@ -275,9 +316,9 @@ func (qm *CQMgr) ClientHeartBeat(id string, cg *ClientGroup) (hbmsg HBmsg, err e if len(suspended) > 0 { hbmsg["discard"] = strings.Join(suspended, ",") } - if client.Status == CLIENT_STAT_DELETED { - hbmsg["stop"] = id - } + //if client.Status == CLIENT_STAT_DELETED { + // hbmsg["stop"] = id + //} hbmsg["server-uuid"] = Server_UUID @@ -339,127 +380,23 @@ func (qm *CQMgr) RegisterNewClient(files FormFiles, cg *ClientGroup) (client *Cl return nil, errors.New(e.ClientGroupBadName) } - all_workunits_map := make(map[string]bool) - - // collect current_work from new client object - current_work, err := client.Get_current_work(true) - if err != nil { - return - } - - if len(current_work) > 1 { - logger.Error("Client %s reports %d elements in Current_work", client_id, len(current_work)) // TODO this is a temprorary check. Remove this if you want to support multiple workunits per client. - err = fmt.Errorf("Client reports too many current workunits") - return - } - - for _, workid := range current_work { - all_workunits_map[workid] = true - } - // check if client is already known old_client, old_client_exists, err := qm.GetClient(client_id, true) if err != nil { return } - if old_client_exists { - - // client is already known. Check current_work, then remove old client object - - old_client_current_work, xerr := old_client.Get_current_work(true) - if xerr != nil { - err = xerr - return - } - - for _, work_id := range old_client_current_work { - - _, ok := all_workunits_map[work_id] - if ok { - // all good, new client is still working on the same workunit the old client did work on. - continue - } - - // the client does not know about this workunit anymore, make sure the workunit also knows that - work, ok, xerr := qm.workQueue.Get(work_id) - if xerr != nil { - logger.Error("(RegisterNewClient) %s", xerr.Error()) - continue - } - if !ok { - // workunit not found, that is ok... - continue - } - - if work.Client == client_id { - work.Client = "" - work_state := work.State - if (work_state != WORK_STAT_SUSPEND) && (work_state != WORK_STAT_QUEUED) { - qm.workQueue.StatusChange(work_id, work, WORK_STAT_QUEUED) - } - } - - continue - - } - - // keep old client, copy relevant values - //err = qm.RemoveClient(client_id, true) - //if err != nil { - // return - //} - } - - keep_work := []string{} - - for workid, _ := range all_workunits_map { - work, ok, xerr := qm.workQueue.Get(workid) - if xerr != nil { - logger.Error("(RegisterNewClient) %s", xerr.Error()) - continue - } - - if !ok { - // client has workunit the server does not know about !? (e.g. after reboot) - continue - } - - if work.Client == client_id { - // ok: workunits exists and knows where it belongs to - work_state := work.State - - // fix state if needed - if (work_state != WORK_STAT_SUSPEND) && (work_state != WORK_STAT_CHECKOUT) { - qm.workQueue.StatusChange(workid, work, WORK_STAT_CHECKOUT) - } - - keep_work = append(keep_work, workid) - continue - } - - if work.Client == "" { - // reclaim work unit - work.Client = client_id - qm.workQueue.StatusChange(workid, work, WORK_STAT_CHECKOUT) - keep_work = append(keep_work, workid) - continue - } - - // work.Client != client.Id (workunit seems to be owned by another client, leave it alone) - - continue - } if old_client_exists { // copy values from new client to old client + old_client.Current_work = client.Current_work - old_client.Set_current_work(keep_work, true) old_client.Tag = true // new client struct will be deleted afterwards } else { - client.Set_current_work(keep_work, true) + client.Tag = true - err = qm.AddClient(client, true) // locks clientMap + + err = qm.AddClient(client, true) if err != nil { return } @@ -520,35 +457,35 @@ func (qm *CQMgr) GetAllClientsByUser(u *user.User) (clients []*Client, err error return } -func (qm *CQMgr) DeleteClientByUser(id string, u *user.User) (err error) { - // Get all clientgroups that user owns or that are publicly owned, or all if user is admin - q := bson.M{} - clientgroups := new(ClientGroups) - dbFindClientGroups(q, clientgroups) - filtered_clientgroups := map[string]bool{} - for _, cg := range *clientgroups { - if (u.Uuid != "public" && (cg.Acl.Owner == u.Uuid || u.Admin == true || cg.Acl.Owner == "public")) || - (u.Uuid == "public" && conf.CLIENT_AUTH_REQ == false && cg.Acl.Owner == "public") { - filtered_clientgroups[cg.Name] = true - } - } - - client, ok, err := qm.GetClient(id, true) - if err != nil { - return - } - if ok { - if val, exists := filtered_clientgroups[client.Group]; exists == true && val == true { - err = qm.DeleteClient(client) - return - } - return errors.New(e.UnAuth) - } - return errors.New(e.ClientNotFound) -} +// func (qm *CQMgr) DeleteClientByUser(id string, u *user.User) (err error) { +// // Get all clientgroups that user owns or that are publicly owned, or all if user is admin +// q := bson.M{} +// clientgroups := new(ClientGroups) +// dbFindClientGroups(q, clientgroups) +// filtered_clientgroups := map[string]bool{} +// for _, cg := range *clientgroups { +// if (u.Uuid != "public" && (cg.Acl.Owner == u.Uuid || u.Admin == true || cg.Acl.Owner == "public")) || +// (u.Uuid == "public" && conf.CLIENT_AUTH_REQ == false && cg.Acl.Owner == "public") { +// filtered_clientgroups[cg.Name] = true +// } +// } +// +// client, ok, err := qm.GetClient(id, true) +// if err != nil { +// return +// } +// if ok { +// if val, exists := filtered_clientgroups[client.Group]; exists == true && val == true { +// err = qm.DeleteClient(client) +// return +// } +// return errors.New(e.UnAuth) +// } +// return errors.New(e.ClientNotFound) +// } // use id OR client -func (qm *CQMgr) SuspendClient(id string, client *Client, client_write_lock bool) (err error) { +func (qm *CQMgr) SuspendClient(id string, client *Client, reason string, client_write_lock bool) (err error) { if client == nil { var ok bool @@ -570,37 +507,41 @@ func (qm *CQMgr) SuspendClient(id string, client *Client, client_write_lock bool defer client.Unlock() } - status, err := client.Get_Status(false) + is_suspended, err := client.Get_Suspended(false) if err != nil { return } - if status == CLIENT_STAT_ACTIVE_IDLE || status == CLIENT_STAT_ACTIVE_BUSY { - client.Set_Status(CLIENT_STAT_SUSPEND, false) - //if err = qm.ClientStatusChange(id, CLIENT_STAT_SUSPEND); err != nil { - // return - //} - qm.ReQueueWorkunitByClient(client, false) + if is_suspended { + err = errors.New(e.ClientNotActive) return } - return errors.New(e.ClientNotActive) + + client.Suspend(reason, false) + + err = qm.ReQueueWorkunitByClient(client, false) + if err != nil { + return + } + + return } -func (qm *CQMgr) SuspendAllClients() (count int, err error) { +func (qm *CQMgr) SuspendAllClients(reason string) (count int, err error) { clients, err := qm.ListClients() if err != nil { return } for _, id := range clients { - if err := qm.SuspendClient(id, nil, true); err == nil { + if err := qm.SuspendClient(id, nil, reason, true); err == nil { count += 1 } } return } -func (qm *CQMgr) SuspendClientByUser(id string, u *user.User) (err error) { +func (qm *CQMgr) SuspendClientByUser(id string, u *user.User, reason string) (err error) { // Get all clientgroups that user owns or that are publicly owned, or all if user is admin q := bson.M{} clientgroups := new(ClientGroups) @@ -618,35 +559,24 @@ func (qm *CQMgr) SuspendClientByUser(id string, u *user.User) (err error) { return } if ok { - return errors.New(e.ClientNotFound) + err = errors.New(e.ClientNotFound) + return } - if val, exists := filtered_clientgroups[client.Group]; exists == true && val == true { - err = client.LockNamed("SuspendClientByUser") + val, exists := filtered_clientgroups[client.Group] + if exists == true && val == true { + + err = qm.SuspendClient("", client, reason, true) if err != nil { return } - defer client.Unlock() - status, xerr := client.Get_Status(false) - if xerr != nil { - err = xerr - return - } - if status == CLIENT_STAT_ACTIVE_IDLE || status == CLIENT_STAT_ACTIVE_BUSY { - client.Set_Status(CLIENT_STAT_SUSPEND, false) - //if err = qm.ClientStatusChange(id, CLIENT_STAT_SUSPEND); err != nil { - // return - //} - qm.ReQueueWorkunitByClient(client, false) - return - } - return errors.New(e.ClientNotActive) + } return errors.New(e.UnAuth) } -func (qm *CQMgr) SuspendAllClientsByUser(u *user.User) (count int) { +func (qm *CQMgr) SuspendAllClientsByUser(u *user.User, reason string) (count int, err error) { // Get all clientgroups that user owns or that are publicly owned, or all if user is admin q := bson.M{} clientgroups := new(ClientGroups) @@ -664,22 +594,29 @@ func (qm *CQMgr) SuspendAllClientsByUser(u *user.User) (count int) { return } for _, client := range clients { - err = client.LockNamed("SuspendAllClientsByUser") - if err != nil { - continue + + is_suspended, xerr := client.Get_Suspended(true) + if xerr != nil { + err = xerr + return } - status, xerr := client.Get_Status(false) + + group, xerr := client.Get_Group(true) if xerr != nil { - continue + return } - if val, exists := filtered_clientgroups[client.Group]; exists == true && val == true && (status == CLIENT_STAT_ACTIVE_IDLE || status == CLIENT_STAT_ACTIVE_BUSY) { - qm.SuspendClient("", client, false) + + if val, exists := filtered_clientgroups[group]; exists == true && val == true && is_suspended { + err = qm.SuspendClient("", client, reason, true) + if err != nil { + return + } count += 1 } - client.Unlock() + } - return count + return } func (qm *CQMgr) ResumeClient(id string) (err error) { @@ -691,17 +628,13 @@ func (qm *CQMgr) ResumeClient(id string) (err error) { return errors.New(e.ClientNotFound) } - err = client.LockNamed("ResumeClient") + err = client.Resume(true) + if err != nil { return } - defer client.Unlock() - if client.Status == CLIENT_STAT_SUSPEND { - //err = qm.ClientStatusChange(id, CLIENT_STAT_ACTIVE_IDLE) - client.Status = CLIENT_STAT_ACTIVE_IDLE - return - } - return errors.New(e.ClientNotSuspended) + + return } @@ -726,19 +659,13 @@ func (qm *CQMgr) ResumeClientByUser(id string, u *user.User) (err error) { return errors.New(e.ClientNotFound) } - err = client.LockNamed("ResumeClientByUser") - if err != nil { - return - } - defer client.Unlock() - if val, exists := filtered_clientgroups[client.Group]; exists == true && val == true { - if client.Status == CLIENT_STAT_SUSPEND { - //err = qm.ClientStatusChange(id, CLIENT_STAT_ACTIVE_IDLE) - client.Status = CLIENT_STAT_ACTIVE_IDLE + err = client.Resume(true) + if err != nil { return } - return errors.New(e.ClientNotSuspended) + + return } return errors.New(e.UnAuth) @@ -751,16 +678,22 @@ func (qm *CQMgr) ResumeSuspendedClients() (count int, err error) { return } for _, client := range clients { - err = client.LockNamed("ResumeSuspendedClients") - if err != nil { - continue + + is_suspended, xerr := client.Get_Suspended(true) + if xerr != nil { + err = xerr + return } - if client.Status == CLIENT_STAT_SUSPEND { + + if is_suspended { //qm.ClientStatusChange(client.Id, CLIENT_STAT_ACTIVE_IDLE) - client.Status = CLIENT_STAT_ACTIVE_IDLE + err = client.Resume(true) + if err != nil { + return + } count += 1 } - client.Unlock() + } return @@ -788,9 +721,20 @@ func (qm *CQMgr) ResumeSuspendedClientsByUser(u *user.User) (count int) { if err != nil { continue } - if val, exists := filtered_clientgroups[client.Group]; exists == true && val == true && client.Status == CLIENT_STAT_SUSPEND { + + is_suspended, xerr := client.Get_Suspended(true) + if xerr != nil { + err = xerr + return + } + + if val, exists := filtered_clientgroups[client.Group]; exists == true && val == true && is_suspended { //qm.ClientStatusChange(client.Id, CLIENT_STAT_ACTIVE_IDLE) - client.Status = CLIENT_STAT_ACTIVE_IDLE + + err = client.Resume(true) + if err != nil { + return + } count += 1 } client.Unlock() @@ -856,10 +800,10 @@ func (qm *CQMgr) CheckoutWorkunits(req_policy string, client_id string, client * if err != nil { return } - status := client.Status + //status := client.Status response_channel := client.coAckChannel - work_length, _ := client.Current_work_length(false) + work_length, _ := client.Current_work.Length(false) client.Unlock() if work_length > 0 { @@ -867,14 +811,21 @@ func (qm *CQMgr) CheckoutWorkunits(req_policy string, client_id string, client * return nil, errors.New(e.ClientBusy) } - if status == CLIENT_STAT_SUSPEND { - return nil, errors.New(e.ClientSuspended) + is_suspended, err := client.Get_Suspended(true) + if err != nil { + return } - if status == CLIENT_STAT_DELETED { - qm.RemoveClient(client_id, false) - return nil, errors.New(e.ClientDeleted) + + if is_suspended { + err = errors.New(e.ClientSuspended) + return } + //if status == CLIENT_STAT_DELETED { + // qm.RemoveClient(client_id, false) + // return nil, errors.New(e.ClientDeleted) + //} + //req := CoReq{policy: req_policy, fromclient: client_id, available: available_bytes, count: num, response: client.coAckChannel} req := CoReq{policy: req_policy, fromclient: client_id, available: available_bytes, count: num, response: response_channel} @@ -922,33 +873,32 @@ func (qm *CQMgr) CheckoutWorkunits(req_policy string, client_id string, client * added_work := 0 for _, work := range ack.workunits { - work_id := work.Id - err = client.Add_work(work_id) + work_id, xerr := New_Workunit_Unique_Identifier(work.Id) + if xerr != nil { + return + } + err = client.Assigned_work.Add(work_id) if err != nil { return } added_work += 1 } - status, xerr := client.Get_Status(true) - if xerr != nil { - err = xerr - return - } - if added_work > 0 && status == CLIENT_STAT_ACTIVE_IDLE { - client.Set_Status(CLIENT_STAT_ACTIVE_BUSY, true) - } + + //if added_work > 0 && status == CLIENT_STAT_ACTIVE_IDLE { + // client.Set_Status(CLIENT_STAT_ACTIVE_BUSY, true) + //} logger.Debug(3, "(CheckoutWorkunits) %s finished", client_id) return ack.workunits, ack.err } -func (qm *CQMgr) GetWorkById(id string) (workunit *Workunit, err error) { +func (qm *CQMgr) GetWorkById(id Workunit_Unique_Identifier) (workunit *Workunit, err error) { workunit, ok, err := qm.workQueue.Get(id) if err != nil { return } if !ok { - err = errors.New(fmt.Sprintf("no workunit found with id %s", id)) + err = errors.New(fmt.Sprintf("no workunit found with id %s", id.String())) } return } @@ -976,6 +926,7 @@ func (qm *CQMgr) popWorks(req CoReq) (client_specific_workunits []*Workunit, err filtered, err := qm.filterWorkByClient(client) if err != nil { + err = fmt.Errorf("(popWorks) filterWorkByClient returned: %s", err.Error()) return } logger.Debug(3, "(popWorks) filterWorkByClient returned: %d (0 meansNoEligibleWorkunitFound)", len(filtered)) @@ -985,6 +936,7 @@ func (qm *CQMgr) popWorks(req CoReq) (client_specific_workunits []*Workunit, err } client_specific_workunits, err = qm.workQueue.selectWorkunits(filtered, req.policy, req.available, req.count) if err != nil { + err = fmt.Errorf("(popWorks) selectWorkunits returned: %s", err.Error()) return } //get workunits successfully, put them into coWorkMap @@ -992,10 +944,10 @@ func (qm *CQMgr) popWorks(req CoReq) (client_specific_workunits []*Workunit, err work.Client = client_id work.CheckoutTime = time.Now() //qm.workQueue.Put(work) TODO isn't that already in the queue ? - qm.workQueue.StatusChange(work.Id, work, WORK_STAT_CHECKOUT) + qm.workQueue.StatusChange(work.Workunit_Unique_Identifier, work, WORK_STAT_CHECKOUT, "") } - logger.Debug(3, "done with popWorks() for client: %s ", client_id) + logger.Debug(3, "(popWorks) done with client: %s ", client_id) return } @@ -1018,6 +970,7 @@ func (qm *CQMgr) filterWorkByClient(client *Client) (workunits WorkList, err err workunit_list, err := qm.workQueue.Queue.GetWorkunits() if err != nil { + err = fmt.Errorf("(filterWorkByClient) qm.workQueue.Queue.GetWorkunits retruns: %s", err.Error()) return } @@ -1072,7 +1025,7 @@ func (qm *CQMgr) filterWorkByClient(client *Client) (workunits WorkList, err err //} //handle feedback from a client about the execution of a workunit -func (qm *CQMgr) handleWorkStatusChange(notice Notice) (err error) { +func (qm *CQMgr) handleNoticeWorkDelivered(notice Notice) (err error) { //to be implemented for proxy or server return } @@ -1108,16 +1061,17 @@ func (qm *CQMgr) ShowWorkunitsByUser(status string, u *user.User) (workunits []* workunits = append(workunits, work) } } else { - if jobid, err := GetJobIdByWorkId(work.Id); err == nil { - if job, err := GetJob(jobid); err == nil { - rights := job.Acl.Check(u.Uuid) - if job.Acl.Owner == u.Uuid || rights["read"] == true { - if work.State == status || status == "" { - workunits = append(workunits, work) - } + jobid := work.JobId + + if job, err := GetJob(jobid); err == nil { + rights := job.Acl.Check(u.Uuid) + if job.Acl.Owner == u.Uuid || rights["read"] == true { + if work.State == status || status == "" { + workunits = append(workunits, work) } } } + } } return workunits @@ -1130,7 +1084,7 @@ func (qm *CQMgr) EnqueueWorkunit(work *Workunit) (err error) { func (qm *CQMgr) ReQueueWorkunitByClient(client *Client, client_write_lock bool) (err error) { - worklist, err := client.Get_current_work(client_write_lock) + worklist, err := client.Current_work.Get_list(client_write_lock) if err != nil { return } @@ -1147,12 +1101,14 @@ func (qm *CQMgr) ReQueueWorkunitByClient(client *Client, client_write_lock bool) continue } - jobid, err := GetJobIdByWorkId(workid) - if err != nil { - logger.Error("(ReQueueWorkunitByClient) GetJobIdByWorkId: %s", err.Error()) - continue + jobid := work.JobId + + job, xerr := GetJob(jobid) + if xerr != nil { + err = xerr + return } - job_state, err := dbGetJobFieldString(jobid, "state") + job_state, err := job.GetState(true) if err != nil { logger.Error("(ReQueueWorkunitByClient) dbGetJobField: %s", err.Error()) continue @@ -1160,8 +1116,8 @@ func (qm *CQMgr) ReQueueWorkunitByClient(client *Client, client_write_lock bool) if contains(JOB_STATS_ACTIVE, job_state) { //only requeue workunits belonging to active jobs (rule out suspended jobs) if work.Client == client.Id { - qm.workQueue.StatusChange(workid, work, WORK_STAT_QUEUED) - logger.Event(event.WORK_REQUEUE, "workid="+workid) + qm.workQueue.StatusChange(workid, work, WORK_STAT_QUEUED, "") + logger.Event(event.WORK_REQUEUE, "workid="+workid.String()) } else { other_client_id := work.Client @@ -1173,14 +1129,14 @@ func (qm *CQMgr) ReQueueWorkunitByClient(client *Client, client_write_lock bool) } if ok { // other_client exists (if otherclient does not exist, that is ok....) - oc_has_work, err := other_client.Current_work_has(workid) + oc_has_work, err := other_client.Current_work.Has(workid) if err != nil { logger.Error("(ReQueueWorkunitByClient) Current_work_has: %s ", err) continue } if !oc_has_work { // other client has not this workunit, - qm.workQueue.StatusChange(workid, work, WORK_STAT_SUSPEND) + qm.workQueue.StatusChange(workid, work, WORK_STAT_SUSPEND, "workunit has wrong client info") continue } } @@ -1189,7 +1145,7 @@ func (qm *CQMgr) ReQueueWorkunitByClient(client *Client, client_write_lock bool) } } else { - qm.workQueue.StatusChange(workid, work, WORK_STAT_SUSPEND) + qm.workQueue.StatusChange(workid, work, WORK_STAT_SUSPEND, "workunit does not belong to an actove job") } } diff --git a/lib/core/cwl/collection.go b/lib/core/cwl/collection.go index 0267e0da..8755acf9 100644 --- a/lib/core/cwl/collection.go +++ b/lib/core/cwl/collection.go @@ -17,8 +17,9 @@ type CWL_collection struct { Strings map[string]*cwl_types.String Ints map[string]*cwl_types.Int Booleans map[string]*cwl_types.Boolean - All map[string]*cwl_types.CWL_object - Job_input *Job_document + All map[string]*cwl_types.CWL_object // everything goes in here + //Job_input *Job_document + Job_input_map *map[string]cwl_types.CWLType } func (c CWL_collection) Evaluate(raw string) (parsed string) { @@ -112,6 +113,31 @@ func (c CWL_collection) Get(id string) (obj *cwl_types.CWL_object, err error) { return } +func (c CWL_collection) GetType(id string) (obj cwl_types.CWLType, err error) { + var ok bool + obj, ok = c.Files[id] + if ok { + return + } + obj, ok = c.Strings[id] + if ok { + return + } + + obj, ok = c.Ints[id] + if ok { + return + } + obj, ok = c.Booleans[id] + if ok { + return + } + + err = fmt.Errorf("(GetType) %s not found", id) + return + +} + func (c CWL_collection) GetFile(id string) (obj *cwl_types.File, err error) { obj, ok := c.Files[id] if !ok { @@ -144,6 +170,14 @@ func (c CWL_collection) GetWorkflowStepInput(id string) (obj *WorkflowStepInput, return } +func (c CWL_collection) GetCommandLineTool(id string) (obj *CommandLineTool, err error) { + obj, ok := c.CommandLineTools[id] + if !ok { + err = fmt.Errorf("(GetCommandLineTool) item %s not found in collection", id) + } + return +} + func NewCWL_collection() (collection CWL_collection) { collection = CWL_collection{} diff --git a/lib/core/cwl/commandInputParameter.go b/lib/core/cwl/commandInputParameter.go index f0b86e10..6e42e9b8 100644 --- a/lib/core/cwl/commandInputParameter.go +++ b/lib/core/cwl/commandInputParameter.go @@ -5,18 +5,60 @@ import ( cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" "github.com/davecgh/go-spew/spew" "github.com/mitchellh/mapstructure" + "gopkg.in/mgo.v2/bson" + "reflect" ) +// http://www.commonwl.org/v1.0/CommandLineTool.html#CommandInputParameter + type CommandInputParameter struct { - Id string `yaml:"id"` - SecondaryFiles []string `yaml:"secondaryFiles"` // TODO string | Expression | array - Format []string `yaml:"format"` - Streamable bool `yaml:"streamable"` - Type []CommandInputParameterType `yaml:"type"` // TODO CWLType | CommandInputRecordSchema | CommandInputEnumSchema | CommandInputArraySchema | string | array - Label string `yaml:"label"` - Description string `yaml:"description"` - InputBinding CommandLineBinding `yaml:"inputBinding"` - Default *cwl_types.Any `yaml:"default"` + Id string `yaml:"id,omitempty" bson:"id,omitempty" json:"id,omitempty"` + SecondaryFiles []string `yaml:"secondaryFiles,omitempty" bson:"secondaryFiles,omitempty" json:"secondaryFiles,omitempty"` // TODO string | Expression | array + Format []string `yaml:"format,omitempty" bson:"format,omitempty" json:"format,omitempty"` + Streamable bool `yaml:"streamable,omitempty" bson:"streamable,omitempty" json:"streamable,omitempty"` + Type interface{} `yaml:"type,omitempty" bson:"type,omitempty" json:"type,omitempty"` // []CommandInputParameterType CWLType | CommandInputRecordSchema | CommandInputEnumSchema | CommandInputArraySchema | string | array + Label string `yaml:"label,omitempty" bson:"label,omitempty" json:"label,omitempty"` + Description string `yaml:"description,omitempty" bson:"description,omitempty" json:"description,omitempty"` + InputBinding CommandLineBinding `yaml:"inputBinding,omitempty" bson:"inputBinding,omitempty" json:"inputBinding,omitempty"` + Default *cwl_types.Any `yaml:"default,omitempty" bson:"default,omitempty" json:"default,omitempty"` +} + +func makeStringMap(v interface{}) (result interface{}, err error) { + + switch v.(type) { + case bson.M: + + original_map := v.(bson.M) + + new_map := make(map[string]interface{}) + + for key_str, value := range original_map { + + new_map[key_str] = value + } + + result = new_map + return + case map[interface{}]interface{}: + + v_map, ok := v.(map[interface{}]interface{}) + if !ok { + err = fmt.Errorf("casting problem (b)") + return + } + v_string_map := make(map[string]interface{}) + + for key, value := range v_map { + key_string := key.(string) + v_string_map[key_string] = value + } + + result = v_string_map + return + + } + result = v + return } func NewCommandInputParameter(v interface{}) (input_parameter *CommandInputParameter, err error) { @@ -24,12 +66,45 @@ func NewCommandInputParameter(v interface{}) (input_parameter *CommandInputParam fmt.Println("NewCommandInputParameter:\n") spew.Dump(v) + v, err = makeStringMap(v) + if err != nil { + return + } + switch v.(type) { - case map[interface{}]interface{}: - v_map, ok := v.(map[interface{}]interface{}) + // case bson.M: + // + // original_map := v.(bson.M) + // + // new_map := make(map[string]interface{}) + // + // for key_str, value := range original_map { + // + // new_map[key_str] = value + // } + // + // return NewCommandInputParameter(new_map) + // case map[interface{}]interface{}: + // + // v_map, ok := v.(map[interface{}]interface{}) + // if !ok { + // err = fmt.Errorf("casting problem (b)") + // return + // } + // v_string_map := make(map[string]interface{}) + // + // for key, value := range v_map { + // key_string := key.(string) + // v_string_map[key_string] = value + // } + // + // return NewCommandInputParameter(v_string_map) + + case map[string]interface{}: + v_map, ok := v.(map[string]interface{}) if !ok { - err = fmt.Errorf("casting problem") + err = fmt.Errorf("casting problem (a)") return } @@ -49,11 +124,13 @@ func NewCommandInputParameter(v interface{}) (input_parameter *CommandInputParam type_value, ok := v_map["type"] if ok { - type_value, err = CreateCommandInputParameterTypeArray(type_value) + + type_value, err = NewCommandInputParameterType(type_value) if err != nil { - err = fmt.Errorf("(NewCommandInputParameter) CreateCommandInputParameterTypeArray error: %s", err.Error()) + err = fmt.Errorf("(NewCommandInputParameter) NewCommandInputParameter error: %s", err.Error()) return } + v_map["type"] = type_value } @@ -73,9 +150,13 @@ func NewCommandInputParameter(v interface{}) (input_parameter *CommandInputParam return } + case string: + v_string := v.(string) + err = fmt.Errorf("(NewCommandInputParameter) got string %s", v_string) + return default: - err = fmt.Errorf("(NewCommandInputParameter) type unknown") + err = fmt.Errorf("(NewCommandInputParameter) type %s unknown", reflect.TypeOf(v)) return } @@ -83,12 +164,14 @@ func NewCommandInputParameter(v interface{}) (input_parameter *CommandInputParam } // keyname will be converted into 'Id'-field +// array | map | map func CreateCommandInputArray(original interface{}) (err error, new_array []*CommandInputParameter) { fmt.Println("CreateCommandInputArray:\n") spew.Dump(original) switch original.(type) { + case map[interface{}]interface{}: for k, v := range original.(map[interface{}]interface{}) { diff --git a/lib/core/cwl/commandInputParameterType.go b/lib/core/cwl/commandInputParameterType.go index 0a453859..184fd25a 100644 --- a/lib/core/cwl/commandInputParameterType.go +++ b/lib/core/cwl/commandInputParameterType.go @@ -4,25 +4,80 @@ import ( "fmt" cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" "github.com/davecgh/go-spew/spew" - //"strings" + "strings" //"github.com/mitchellh/mapstructure" + "reflect" ) type CommandInputParameterType struct { Type string } -func NewCommandInputParameterType(original interface{}) (cipt_ptr *CommandInputParameterType, err error) { +// CWLType | CommandInputRecordSchema | CommandInputEnumSchema | CommandInputArraySchema | string | array + +func NewCommandInputParameterType(original interface{}) (result interface{}, err error) { // Try CWL_Type - var cipt CommandInputParameterType + //var cipt CommandInputParameterType switch original.(type) { + case []interface{}: + + case map[string]interface{}: + + original_map, ok := original.(map[string]interface{}) + if !ok { + err = fmt.Errorf("(NewCommandInputParameterType) type error") + return + } + + type_str, ok := original_map["Type"] + if !ok { + type_str, ok = original_map["type"] + } + + if !ok { + err = fmt.Errorf("(NewCommandInputParameterType) type error, field type not found") + return + } + + switch type_str { + case "array": + schema, xerr := NewCommandOutputArraySchema(original_map) + if xerr != nil { + err = xerr + return + } + result = schema + return + case "enum": + + schema, xerr := NewCommandOutputEnumSchema(original_map) + if xerr != nil { + err = xerr + return + } + result = schema + return + + case "record": + schema, xerr := NewCommandOutputRecordSchema(original_map) + if xerr != nil { + err = xerr + return + } + result = schema + return + + } + err = fmt.Errorf("(NewCommandInputParameterType) type %s unknown", type_str) + return + case string: original_str := original.(string) - switch original_str { + switch strings.ToLower(original_str) { case cwl_types.CWL_null: case cwl_types.CWL_boolean: @@ -31,63 +86,25 @@ func NewCommandInputParameterType(original interface{}) (cipt_ptr *CommandInputP case cwl_types.CWL_float: case cwl_types.CWL_double: case cwl_types.CWL_string: - case cwl_types.CWL_File: - case cwl_types.CWL_Directory: + case strings.ToLower(cwl_types.CWL_File): + case strings.ToLower(cwl_types.CWL_Directory): default: err = fmt.Errorf("(NewCommandInputParameterType) type %s is unknown", original_str) return } - - cipt.Type = original_str - cipt_ptr = &cipt + result = original_str return } fmt.Printf("unknown type") spew.Dump(original) - err = fmt.Errorf("(NewCommandInputParameterType) Type unknown") + err = fmt.Errorf("(NewCommandInputParameterType) Type %s unknown", reflect.TypeOf(original)) return } -func CreateCommandInputParameterTypeArray(v interface{}) (cipt_array_ptr *[]CommandInputParameterType, err error) { - - cipt_array := []CommandInputParameterType{} - - array, ok := v.([]interface{}) - - if ok { - //handle array case - for _, v := range array { - - cipt, xerr := NewCommandInputParameterType(v) - if xerr != nil { - err = xerr - return - } - - cipt_array = append(cipt_array, *cipt) - } - cipt_array_ptr = &cipt_array - return - } - - // handle non-arrary case - - cipt, err := NewCommandInputParameterType(v) - if err != nil { - err = fmt.Errorf("(CreateCommandInputParameterTypeArray) NewCommandInputParameterType returns %s", err.Error()) - return - } - - cipt_array = append(cipt_array, *cipt) - cipt_array_ptr = &cipt_array - - return -} - func HasCommandInputParameterType(array *[]CommandInputParameterType, search_type string) (ok bool) { for _, v := range *array { if v.Type == search_type { diff --git a/lib/core/cwl/commandLineBinding.go b/lib/core/cwl/commandLineBinding.go index fecd1a91..8b86d645 100644 --- a/lib/core/cwl/commandLineBinding.go +++ b/lib/core/cwl/commandLineBinding.go @@ -3,31 +3,52 @@ package cwl import ( "fmt" //"github.com/davecgh/go-spew/spew" + cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" "github.com/mitchellh/mapstructure" - //"reflect" + "reflect" ) //http://www.commonwl.org/v1.0/Workflow.html#CommandLineBinding type CommandLineBinding struct { - LoadContents bool `yaml:"loadContents"` - Position int `yaml:"position"` - Prefix string `yaml:"prefix"` - Separate string `yaml:"separate"` - ItemSeparator string `yaml:"itemSeparator"` - ValueFrom string `yaml:"valueFrom"` - ShellQuote bool `yaml:"shellQuote"` + LoadContents bool `yaml:"loadContents,omitempty" bson:"loadContents,omitempty" json:"loadContents,omitempty"` + Position int `yaml:"position,omitempty" bson:"position,omitempty" json:"position,omitempty"` + Prefix string `yaml:"prefix,omitempty" bson:"prefix,omitempty" json:"prefix,omitempty"` + Separate bool `yaml:"separate,omitempty" bson:"separate,omitempty" json:"separate,omitempty"` + ItemSeparator string `yaml:"itemSeparator,omitempty" bson:"itemSeparator,omitempty" json:"itemSeparator,omitempty"` + ValueFrom *cwl_types.Expression `yaml:"valueFrom,omitempty" bson:"valueFrom,omitempty" json:"valueFrom,omitempty"` + ShellQuote bool `yaml:"shellQuote,omitempty" bson:"shellQuote,omitempty" json:"shellQuote,omitempty"` } func NewCommandLineBinding(original interface{}) (clb *CommandLineBinding, err error) { var commandlinebinding CommandLineBinding + original, err = makeStringMap(original) + if err != nil { + return + } + switch original.(type) { - case map[interface{}]interface{}: + case map[string]interface{}: + original_map, ok := original.(map[string]interface{}) + if !ok { + err = fmt.Errorf("(NewCommandLineBinding) type assertion error map[string]interface{}") + return + } + + value_from, ok := original_map["valueFrom"] + if ok { + exp, xerr := cwl_types.NewExpression(value_from) + if xerr != nil { + err = fmt.Errorf("(NewCommandLineBinding) NewExpression failed: %s", xerr.Error()) + return + } + original_map["valueFrom"] = *exp + } err = mapstructure.Decode(original, &commandlinebinding) if err != nil { - err = fmt.Errorf("(NewCommandLineBinding) %s", err.Error()) + err = fmt.Errorf("(NewCommandLineBinding) mapstructure: %s", err.Error()) return } clb = &commandlinebinding @@ -36,10 +57,13 @@ func NewCommandLineBinding(original interface{}) (clb *CommandLineBinding, err e org_str, _ := original.(string) - commandlinebinding.ValueFrom = org_str + commandlinebinding.ValueFrom, err = cwl_types.NewExpression(org_str) + if err != nil { + return + } default: - err = fmt.Errorf("(NewCommandLineBinding) type unknown") + err = fmt.Errorf("(NewCommandLineBinding) type %s unknown", reflect.TypeOf(original)) return } diff --git a/lib/core/cwl/commandLineTool.go b/lib/core/cwl/commandLineTool.go index 011f5b81..270805bf 100644 --- a/lib/core/cwl/commandLineTool.go +++ b/lib/core/cwl/commandLineTool.go @@ -8,20 +8,21 @@ import ( ) type CommandLineTool struct { - Id string `yaml:"id"` - BaseCommand []string `yaml:"baseCommand"` // TODO also allow []string - Inputs []CommandInputParameter `yaml:"inputs"` - Outputs []CommandOutputParameter `yaml:"outputs"` - Hints []Requirement `yaml:"hints"` // TODO Any - Label string `yaml:"label"` - Description string `yaml:"description"` - CwlVersion CWLVersion `yaml:"cwlVersion"` - Arguments []CommandLineBinding `yaml:"arguments"` // TODO support CommandLineBinding - Stdin string `yaml:"stdin"` // TODO support Expression - Stdout string `yaml:"stdout"` // TODO support Expression - SuccessCodes []int `yaml:"successCodes"` - TemporaryFailCodes []int `yaml:"temporaryFailCodes"` - PermanentFailCodes []int `yaml:"permanentFailCodes"` + Id string `yaml:"id,omitempty" bson:"id,omitempty" json:"id,omitempty"` + Class string `yaml:"class,omitempty" bson:"class,omitempty" json:"class,omitempty"` + BaseCommand []string `yaml:"baseCommand,omitempty" bson:"baseCommand,omitempty" json:"baseCommand,omitempty"` // TODO also allow []string + Inputs []CommandInputParameter `yaml:"inputs,omitempty" bson:"inputs,omitempty" json:"inputs,omitempty"` + Outputs []CommandOutputParameter `yaml:"outputs,omitempty" bson:"outputs,omitempty" json:"outputs,omitempty"` + Hints []Requirement `yaml:"hints,omitempty" bson:"hints,omitempty" json:"hints,omitempty"` // TODO Any + Label string `yaml:"label,omitempty" bson:"label,omitempty" json:"label,omitempty"` + Description string `yaml:"description,omitempty" bson:"description,omitempty" json:"description,omitempty"` + CwlVersion CWLVersion `yaml:"cwlVersion,omitempty" bson:"cwlVersion,omitempty" json:"cwlVersion,omitempty"` + Arguments []CommandLineBinding `yaml:"arguments,omitempty" bson:"arguments,omitempty" json:"arguments,omitempty"` // TODO support CommandLineBinding + Stdin string `yaml:"stdin,omitempty" bson:"stdin,omitempty" json:"stdin,omitempty"` // TODO support Expression + Stdout string `yaml:"stdout,omitempty" bson:"stdout,omitempty" json:"stdout,omitempty"` // TODO support Expression + SuccessCodes []int `yaml:"successCodes,omitempty" bson:"successCodes,omitempty" json:"successCodes,omitempty"` + TemporaryFailCodes []int `yaml:"temporaryFailCodes,omitempty" bson:"temporaryFailCodes,omitempty" json:"temporaryFailCodes,omitempty"` + PermanentFailCodes []int `yaml:"permanentFailCodes,omitempty" bson:"permanentFailCodes,omitempty" json:"permanentFailCodes,omitempty"` } func (c *CommandLineTool) GetClass() string { return "CommandLineTool" } @@ -32,12 +33,20 @@ func (c *CommandLineTool) Is_process() {} // keyname will be converted into 'Id'-field -func NewCommandLineTool(object CWL_object_generic) (commandLineTool *CommandLineTool, err error) { +//func NewCommandLineTool(object CWL_object_generic) (commandLineTool *CommandLineTool, err error) { +func NewCommandLineTool(generic interface{}) (commandLineTool *CommandLineTool, err error) { fmt.Println("NewCommandLineTool:") - spew.Dump(object) + spew.Dump(generic) - commandLineTool = &CommandLineTool{} + //switch type() + object, ok := generic.(map[string]interface{}) + if !ok { + err = fmt.Errorf("other types than map[string]interface{} not supported yet") + return + } + + commandLineTool = &CommandLineTool{Class: "CommandLineTool"} inputs, ok := object["inputs"] if ok { diff --git a/lib/core/cwl/commandOutputArraySchema.go b/lib/core/cwl/commandOutputArraySchema.go new file mode 100644 index 00000000..3ce4cf2a --- /dev/null +++ b/lib/core/cwl/commandOutputArraySchema.go @@ -0,0 +1,53 @@ +package cwl + +import ( + "fmt" + "github.com/mitchellh/mapstructure" + "reflect" +) + +type CommandOutputArraySchema struct { + Items []string `yaml:"items,omitempty" bson:"items,omitempty" json:"items,omitempty"` + Type string `yaml:"type,omitempty" bson:"type,omitempty" json:"type,omitempty"` // must be array + Label string `yaml:"label,omitempty" bson:"label,omitempty" json:"label,omitempty"` + OutputBinding *CommandOutputBinding `yaml:"outputBinding,omitempty" bson:"outputBinding,omitempty" json:"outputBinding,omitempty"` +} + +func (c *CommandOutputArraySchema) Is_CommandOutputParameterType() {} + +func NewCommandOutputArraySchema(original interface{}) (coas *CommandOutputArraySchema, err error) { + + original, err = makeStringMap(original) + if err != nil { + return + } + + coas = &CommandOutputArraySchema{} + coas.Type = "array" + switch original.(type) { + + case map[string]interface{}: + original_map, ok := original.(map[string]interface{}) + if !ok { + err = fmt.Errorf("(NewCommandOutputArraySchema) type error b") + return + } + + items, ok := original_map["items"] + if ok { + items_string, ok := items.(string) + if ok { + original_map["items"] = []string{items_string} + } + } + + err = mapstructure.Decode(original, coas) + if err != nil { + err = fmt.Errorf("(NewCommandOutputArraySchema) %s", err.Error()) + return + } + default: + err = fmt.Errorf("NewCommandOutputArraySchema, unknown type %s", reflect.TypeOf(original)) + } + return +} diff --git a/lib/core/cwl/commandOutputBinding.go b/lib/core/cwl/commandOutputBinding.go index 58e6a1cf..5adfbab2 100644 --- a/lib/core/cwl/commandOutputBinding.go +++ b/lib/core/cwl/commandOutputBinding.go @@ -6,25 +6,36 @@ import ( "github.com/mitchellh/mapstructure" ) +// http://www.commonwl.org/v1.0/CommandLineTool.html#CommandOutputBinding type CommandOutputBinding struct { - Glob []cwl_types.Expression `yaml:"glob"` - LoadContents bool `yaml:"loadContents"` - OutputEval cwl_types.Expression `yaml:"outputEval"` + Glob []cwl_types.Expression `yaml:"glob,omitempty" bson:"glob,omitempty" json:"glob,omitempty"` + LoadContents bool `yaml:"loadContents,omitempty" bson:"loadContents,omitempty" json:"loadContents,omitempty"` + OutputEval cwl_types.Expression `yaml:"outputEval,omitempty" bson:"outputEval,omitempty" json:"outputEval,omitempty"` } func NewCommandOutputBinding(original interface{}) (commandOutputBinding *CommandOutputBinding, err error) { - switch original.(type) { + original, err = makeStringMap(original) + if err != nil { + return + } - case map[interface{}]interface{}: - original_map := original.(map[interface{}]interface{}) + switch original.(type) { + case map[string]interface{}: + original_map, ok := original.(map[string]interface{}) + if !ok { + err = fmt.Errorf("(NewCommandOutputBinding) type error b") + return + } glob, ok := original_map["glob"] if ok { - original_map["glob"], err = cwl_types.NewExpressionArray(glob) - if err != nil { + glob_object, xerr := cwl_types.NewExpressionArray(glob) + if xerr != nil { + err = fmt.Errorf("(NewCommandOutputBinding/glob) NewExpressionArray returned: %s", xerr.Error()) return } + original_map["glob"] = glob_object } outputEval, ok := original_map["outputEval"] if ok { @@ -34,14 +45,14 @@ func NewCommandOutputBinding(original interface{}) (commandOutputBinding *Comman } } default: - err = fmt.Errorf("NewCommandOutputBinding: type unknown") + err = fmt.Errorf("(NewCommandOutputBinding) type unknown") return } commandOutputBinding = &CommandOutputBinding{} err = mapstructure.Decode(original, &commandOutputBinding) if err != nil { - err = fmt.Errorf("(NewCommandOutputBinding) %s", err.Error()) + err = fmt.Errorf("(NewCommandOutputBinding) mapstructure: %s", err.Error()) return } //output_parameter.OutputBinding = outputBinding diff --git a/lib/core/cwl/commandOutputEnumSchema.go b/lib/core/cwl/commandOutputEnumSchema.go new file mode 100644 index 00000000..5a3a7d4f --- /dev/null +++ b/lib/core/cwl/commandOutputEnumSchema.go @@ -0,0 +1,28 @@ +package cwl + +import ( + "fmt" + "github.com/mitchellh/mapstructure" +) + +//http://www.commonwl.org/v1.0/CommandLineTool.html#CommandOutputEnumSchema +type CommandOutputEnumSchema struct { + Symbols []string `yaml:"symbols,omitempty" bson:"symbols,omitempty" json:"symbols,omitempty"` + Type string `yaml:"type,omitempty" bson:"type,omitempty" json:"type,omitempty"` // must be enum + Label string `yaml:"label,omitempty" bson:"label,omitempty" json:"label,omitempty"` + OutputBinding *CommandOutputBinding `yaml:"outputbinding,omitempty" bson:"outputbinding,omitempty" json:"outputbinding,omitempty"` +} + +func (c *CommandOutputEnumSchema) Is_CommandOutputParameterType() {} + +func NewCommandOutputEnumSchema(v map[string]interface{}) (schema *CommandOutputEnumSchema, err error) { + + schema = &CommandOutputEnumSchema{} + err = mapstructure.Decode(v, schema) + if err != nil { + err = fmt.Errorf("(NewCommandOutputEnumSchema) decode error: %s", err.Error()) + return + } + + return +} diff --git a/lib/core/cwl/commandOutputParameter.go b/lib/core/cwl/commandOutputParameter.go index 8b14a491..bacbe595 100644 --- a/lib/core/cwl/commandOutputParameter.go +++ b/lib/core/cwl/commandOutputParameter.go @@ -3,51 +3,65 @@ package cwl import ( "fmt" cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" + "github.com/davecgh/go-spew/spew" "github.com/mitchellh/mapstructure" + "reflect" ) type CommandOutputParameter struct { - Id string `yaml:"id"` - SecondaryFiles []cwl_types.Expression `yaml:"secondaryFiles"` // TODO string | Expression | array - Format string `yaml:"format"` - Streamable bool `yaml:"streamable"` - Type []CommandOutputParameterType `yaml:"type"` // TODO CWLType | CommandInputRecordSchema | CommandInputEnumSchema | CommandInputArraySchema | string | array - Label string `yaml:"label"` - Description string `yaml:"description"` - OutputBinding CommandOutputBinding `yaml:"outputBinding"` + Id string `yaml:"id,omitempty" bson:"id,omitempty" json:"id,omitempty"` + SecondaryFiles []cwl_types.Expression `yaml:"secondaryFiles,omitempty" bson:"secondaryFiles,omitempty" json:"secondaryFiles,omitempty"` // TODO string | Expression | array + Format string `yaml:"format,omitempty" bson:"format,omitempty" json:"format,omitempty"` + Streamable bool `yaml:"streamable,omitempty" bson:"streamable,omitempty" json:"streamable,omitempty"` + Type interface{} `yaml:"type,omitempty" bson:"type,omitempty" json:"type,omitempty"` // []CommandOutputParameterType TODO CWLType | CommandInputRecordSchema | CommandInputEnumSchema | CommandInputArraySchema | string | array + Label string `yaml:"label,omitempty" bson:"label,omitempty" json:"label,omitempty"` + Description string `yaml:"description,omitempty" bson:"description,omitempty" json:"description,omitempty"` + OutputBinding CommandOutputBinding `yaml:"outputBinding,omitempty" bson:"outputBinding,omitempty" json:"outputBinding,omitempty"` } func NewCommandOutputParameter(original interface{}) (output_parameter *CommandOutputParameter, err error) { + + original, err = makeStringMap(original) + if err != nil { + return + } + switch original.(type) { - case map[interface{}]interface{}: - original_map := original.(map[interface{}]interface{}) + case map[string]interface{}: + + original_map, ok := original.(map[string]interface{}) + if !ok { + err = fmt.Errorf("(NewCommandOutputParameter) type error") + return + } outputBinding, ok := original_map["outputBinding"] if ok { original_map["outputBinding"], err = NewCommandOutputBinding(outputBinding) if err != nil { + err = fmt.Errorf("(NewCommandOutputParameter) NewCommandOutputBinding returns %s", err.Error()) return } } - COPtype, ok := original_map["type"] - if ok { - original_map["type"], err = NewCommandOutputParameterTypeArray(COPtype) - if err != nil { - return - } - } + //COPtype, ok := original_map["type"] + //if ok { + // original_map["type"], err = NewCommandOutputParameterTypeArray(COPtype) + // if err != nil { + // return + // } + //} output_parameter = &CommandOutputParameter{} err = mapstructure.Decode(original, output_parameter) if err != nil { - err = fmt.Errorf("(NewCommandOutputParameter) %s", err.Error()) + err = fmt.Errorf("(NewCommandOutputParameter) mapstructure returs: %s", err.Error()) return } default: - //spew.Dump(original) - err = fmt.Errorf("NewCommandOutputParameter, unknown type") + spew.Dump(original) + err = fmt.Errorf("NewCommandOutputParameter, unknown type %s", reflect.TypeOf(original)) } //spew.Dump(new_array) return @@ -59,7 +73,7 @@ func NewCommandOutputParameterArray(original interface{}) (copa *[]CommandOutput case map[interface{}]interface{}: cop, xerr := NewCommandOutputParameter(original) if xerr != nil { - err = xerr + err = fmt.Errorf("(NewCommandOutputParameterArray) a NewCommandOutputParameter returns: %s", xerr) return } copa = &[]CommandOutputParameter{*cop} @@ -71,7 +85,7 @@ func NewCommandOutputParameterArray(original interface{}) (copa *[]CommandOutput for _, element := range original_array { cop, xerr := NewCommandOutputParameter(element) if xerr != nil { - err = xerr + err = fmt.Errorf("(NewCommandOutputParameterArray) b NewCommandOutputParameter returns: %s", xerr) return } copa_nptr = append(copa_nptr, *cop) diff --git a/lib/core/cwl/commandOutputParameterType.go b/lib/core/cwl/commandOutputParameterType.go index 697649b7..548162d3 100644 --- a/lib/core/cwl/commandOutputParameterType.go +++ b/lib/core/cwl/commandOutputParameterType.go @@ -1,167 +1,144 @@ package cwl import ( - "fmt" - "github.com/MG-RAST/AWE/lib/logger" - "github.com/davecgh/go-spew/spew" - "github.com/mitchellh/mapstructure" - //"strings" - cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" +//"fmt" +//"github.com/MG-RAST/AWE/lib/logger" +//"github.com/davecgh/go-spew/spew" +//"github.com/mitchellh/mapstructure" +//"strings" +//cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" +//"reflect" ) //type of http://www.commonwl.org/v1.0/CommandLineTool.html#CommandOutputParameter -type CommandOutputParameterType struct { - Type string - CommandOutputArraySchema *CommandOutputArraySchema - CommandOutputRecordSchema *CommandOutputRecordSchema -} +// CWLType | stdout | stderr | CommandOutputRecordSchema | CommandOutputEnumSchema | CommandOutputArraySchema | string | +// array -type CommandOutputRecordSchema struct { - Type string // Must be record - Fields []CommandOutputRecordField - Label string +type CommandOutputParameterType interface { + Is_CommandOutputParameterType() } -type CommandOutputRecordField struct{} - -//http://www.commonwl.org/v1.0/CommandLineTool.html#CommandOutputEnumSchema -type CommandOutputEnumSchema struct { - Symbols []string - Type string // must be enum - Label string - OutputBinding *CommandOutputBinding +type CommandOutputParameterTypeImpl struct { + Type string } -type CommandOutputArraySchema struct { - Items []string `yaml:"items"` - Type string `yaml:"type"` // must be array - Label string `yaml:"label"` - OutputBinding *CommandOutputBinding `yaml:"outputBinding"` -} - -func NewCommandOutputArraySchema(original map[interface{}]interface{}) (coas *CommandOutputArraySchema, err error) { - coas = &CommandOutputArraySchema{} - - items, ok := original["items"] - if ok { - items_string, ok := items.(string) - if ok { - original["items"] = []string{items_string} - } - } - - err = mapstructure.Decode(original, coas) - if err != nil { - err = fmt.Errorf("(NewCommandOutputArraySchema) %s", err.Error()) - return - } - return -} - -func NewCommandOutputParameterType(original interface{}) (copt_ptr *CommandOutputParameterType, err error) { - - // Try CWL_Type - var copt CommandOutputParameterType - - switch original.(type) { - - case string: - original_str := original.(string) - - //original_str_lower := strings.ToLower(original_str) - - _, is_valid := cwl_types.Valid_cwltypes[original_str] - - if !is_valid { - err = fmt.Errorf("(NewCommandOutputParameterType) type %s is unknown", original_str) - return - } - - copt.Type = original_str - copt_ptr = &copt - return - case map[interface{}]interface{}: - // CommandOutputArraySchema www.commonwl.org/v1.0/CommandLineTool.html#CommandOutputArraySchema - original_map := original.(map[interface{}]interface{}) - - output_type, ok := original_map["type"] - - if !ok { - fmt.Printf("unknown type") - spew.Dump(original) - err = fmt.Errorf("(NewCommandOutputParameterType) Map-Type unknown") - } - - switch output_type { - case "array": - copt.CommandOutputArraySchema, err = NewCommandOutputArraySchema(original_map) - copt_ptr = &copt - return - default: - fmt.Printf("unknown type %s:", output_type) - spew.Dump(original) - err = fmt.Errorf("(NewCommandOutputParameterType) Map-Type %s unknown", output_type) - return - } - - } - - fmt.Printf("unknown type") - spew.Dump(original) - err = fmt.Errorf("(NewCommandOutputParameterType) Type unknown") - - return - -} - -func NewCommandOutputParameterTypeArray(original interface{}) (copta *[]CommandOutputParameterType, err error) { - - switch original.(type) { - case map[interface{}]interface{}: - logger.Debug(3, "[found map]") - - copt, xerr := NewCommandOutputParameterType(original) - if xerr != nil { - err = xerr - return - } - copta = &[]CommandOutputParameterType{*copt} - case []interface{}: - logger.Debug(3, "[found array]") - - copta_nptr := []CommandOutputParameterType{} - - original_array := original.([]interface{}) - - for _, element := range original_array { - - spew.Dump(original) - copt, xerr := NewCommandOutputParameterType(element) - if xerr != nil { - err = xerr - return - } - copta_nptr = append(copta_nptr, *copt) - } - - copta = &copta_nptr - case string: - copta_nptr := []CommandOutputParameterType{} - - copt, xerr := NewCommandOutputParameterType(original) - if xerr != nil { - err = xerr - return - } - copta_nptr = append(copta_nptr, *copt) - - copta = &copta_nptr - default: - fmt.Printf("unknown type") - spew.Dump(original) - err = fmt.Errorf("(NewCommandOutputParameterTypeArray) unknown type") - } - return - -} +func (c *CommandOutputParameterTypeImpl) Is_CommandOutputParameterType() {} + +// func NewCommandOutputParameterType(original interface{}) (copt_ptr *CommandOutputParameterType, err error) { +// +// // Try CWL_Type +// var copt CommandOutputParameterTypeImpl +// +// switch original.(type) { +// +// case string: +// original_str := original.(string) +// +// //original_str_lower := strings.ToLower(original_str) +// +// _, is_valid := cwl_types.Valid_cwltypes[original_str] +// +// if !is_valid { +// err = fmt.Errorf("(NewCommandOutputParameterType) type %s is unknown", original_str) +// return +// } +// +// copt.Type = original_str +// copt_ptr = &copt +// return +// +// case map[interface{}]interface{}: +// original_map, ok := original.(map[string]interface{}) +// if !ok { +// err = fmt.Errorf("type error") +// return +// } +// return NewCommandOutputParameterType(original_map) +// case map[string]interface{}: +// // CommandOutputArraySchema www.commonwl.org/v1.0/CommandLineTool.html#CommandOutputArraySchema +// original_map, ok := original.(map[string]interface{}) +// if !ok { +// err = fmt.Errorf("type error") +// return +// } +// output_type, ok := original_map["type"] +// +// if !ok { +// fmt.Printf("unknown type") +// spew.Dump(original) +// err = fmt.Errorf("(NewCommandOutputParameterType) map[string]interface{} has no type field") +// return +// } +// +// switch output_type { +// case "array": +// copt.CommandOutputArraySchema, err = NewCommandOutputArraySchema(original_map) +// copt_ptr = &copt +// return +// default: +// fmt.Printf("unknown type %s:", output_type) +// spew.Dump(original) +// err = fmt.Errorf("(NewCommandOutputParameterType) Map-Type %s unknown", reflect.TypeOf(output_type)) +// return +// } +// +// } +// +// spew.Dump(original) +// err = fmt.Errorf("(NewCommandOutputParameterType) Type %s unknown", reflect.TypeOf(original)) +// +// return +// +// } +// +// func NewCommandOutputParameterTypeArray(original interface{}) (copta *[]CommandOutputParameterType, err error) { +// +// switch original.(type) { +// case map[interface{}]interface{}: +// logger.Debug(3, "[found map]") +// +// copt, xerr := NewCommandOutputParameterType(original) +// if xerr != nil { +// err = xerr +// return +// } +// copta = &[]CommandOutputParameterType{*copt} +// case []interface{}: +// logger.Debug(3, "[found array]") +// +// copta_nptr := []CommandOutputParameterType{} +// +// original_array := original.([]interface{}) +// +// for _, element := range original_array { +// +// spew.Dump(original) +// copt, xerr := NewCommandOutputParameterType(element) +// if xerr != nil { +// err = xerr +// return +// } +// copta_nptr = append(copta_nptr, *copt) +// } +// +// copta = &copta_nptr +// case string: +// copta_nptr := []CommandOutputParameterType{} +// +// copt, xerr := NewCommandOutputParameterType(original) +// if xerr != nil { +// err = xerr +// return +// } +// copta_nptr = append(copta_nptr, *copt) +// +// copta = &copta_nptr +// default: +// fmt.Printf("unknown type") +// spew.Dump(original) +// err = fmt.Errorf("(NewCommandOutputParameterTypeArray) unknown type") +// } +// return +// +// } diff --git a/lib/core/cwl/commandOutputRecordSchema.go b/lib/core/cwl/commandOutputRecordSchema.go new file mode 100644 index 00000000..3be8fb46 --- /dev/null +++ b/lib/core/cwl/commandOutputRecordSchema.go @@ -0,0 +1,28 @@ +package cwl + +import ( + "fmt" + "github.com/mitchellh/mapstructure" +) + +type CommandOutputRecordSchema struct { + Type string `yaml:"type,omitempty" bson:"type,omitempty" json:"type,omitempty"` // Must be record + Fields []CommandOutputRecordField `yaml:"fields,omitempty" bson:"fields,omitempty" json:"fields,omitempty"` + Label string `yaml:"label,omitempty" bson:"label,omitempty" json:"label,omitempty"` +} + +func (c *CommandOutputRecordSchema) Is_CommandOutputParameterType() {} + +type CommandOutputRecordField struct{} + +func NewCommandOutputRecordSchema(v interface{}) (schema *CommandOutputRecordSchema, err error) { + + schema = &CommandOutputRecordSchema{} + err = mapstructure.Decode(v, schema) + if err != nil { + err = fmt.Errorf("(NewCommandOutputRecordSchema) decode error: %s", err.Error()) + return + } + + return +} diff --git a/lib/core/cwl/cwl.go b/lib/core/cwl/cwl.go index c3f22111..742e2026 100644 --- a/lib/core/cwl/cwl.go +++ b/lib/core/cwl/cwl.go @@ -7,60 +7,50 @@ import ( cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" "github.com/MG-RAST/AWE/lib/logger" "github.com/davecgh/go-spew/spew" - "github.com/mitchellh/mapstructure" + //"github.com/mitchellh/mapstructure" "gopkg.in/yaml.v2" //"io/ioutil" //"os" - _ "reflect" + "reflect" "strings" ) // this is used by YAML or JSON library for inital parsing type CWL_document_generic struct { - CwlVersion string `yaml:"cwlVersion"` - Graph []CWL_object_generic `yaml:"graph"` + CwlVersion CWLVersion `yaml:"cwlVersion"` + Graph []interface{} `yaml:"graph"` + //Graph []CWL_object_generic `yaml:"graph"` } type CWL_object_generic map[string]interface{} -type CWLVersion interface{} // TODO +type CWLVersion string type LinkMergeMethod string // merge_nested or merge_flattened -func Parse_cwl_document(collection *CWL_collection, yaml_str string) (err error) { +func New_CWL_object(original interface{}, cwl_version CWLVersion) (obj cwl_types.CWL_object, err error) { + fmt.Println("New_CWL_object") - // TODO check cwlVersion - // TODO screen for "$import": // this might break the YAML parser ! - - // this yaml parser (gopkg.in/yaml.v2) has problems with the CWL yaml format. We skip the header aand jump directly to "$graph" because of that. - graph_pos := strings.Index(yaml_str, "$graph") - - if graph_pos == -1 { - err = errors.New("yaml parisng error. keyword $graph missing: ") + if original == nil { + err = fmt.Errorf("(New_CWL_object) original is nil!") return } - yaml_str = strings.Replace(yaml_str, "$graph", "graph", -1) // remove dollar sign - - cwl_gen := CWL_document_generic{} - - err = Unmarshal([]byte(yaml_str), &cwl_gen) + original, err = makeStringMap(original) if err != nil { - logger.Debug(1, "CWL unmarshal error") - logger.Error("error: " + err.Error()) + return } - - fmt.Println("-------------- raw CWL") - spew.Dump(cwl_gen) - fmt.Println("-------------- Start real parsing") - - // iterated over Graph - for _, elem := range cwl_gen.Graph { - + fmt.Println("New_CWL_object 2") + switch original.(type) { + case map[string]interface{}: + elem := original.(map[string]interface{}) + fmt.Println("New_CWL_object B") cwl_object_type, ok := elem["class"].(string) if !ok { - err = errors.New("object has no member class") + fmt.Println("------------------") + spew.Dump(original) + err = errors.New("object has no field \"class\"") return } @@ -69,114 +59,162 @@ func Parse_cwl_document(collection *CWL_collection, yaml_str string) (err error) err = errors.New("object has no member id") return } - _ = cwl_object_id - //switch elem["hints"].(type) { - //case map[interface{}]interface{}: - //logger.Debug(1, "hints is map[interface{}]interface{}") - // Convert map of outputs into array of outputs - //err, elem["hints"] = CreateRequirementArray(elem["hints"]) - //if err != nil { - // return - //} - //case interface{}[]: - // logger.Debug(1, "hints is interface{}[]") - //default: - // logger.Debug(1, "hints is something else") - //} - + fmt.Println("New_CWL_object C") switch cwl_object_type { case "CommandLineTool": + fmt.Println("New_CWL_object CommandLineTool") logger.Debug(1, "parse CommandLineTool") result, xerr := NewCommandLineTool(elem) if xerr != nil { err = fmt.Errorf("NewCommandLineTool returned: %s", xerr) return } - //*** check if "inputs"" is an array or a map" - //collection.CommandLineTools[result.Id] = result - err = collection.Add(result) - if err != nil { - return - } - //collection = append(collection, result) + result.CwlVersion = cwl_version + + obj = result + return case "Workflow": + fmt.Println("New_CWL_object Workflow") logger.Debug(1, "parse Workflow") - workflow, xerr := NewWorkflow(elem, collection) + workflow, xerr := NewWorkflow(elem) if xerr != nil { err = xerr return } + obj = workflow + return + } // end switch - // some checks and add inputs to collection - for _, input := range workflow.Inputs { - // input is InputParameter - - if input.Id == "" { - err = fmt.Errorf("input has no ID") - return - } - - //if !strings.HasPrefix(input.Id, "#") { - // if !strings.HasPrefix(input.Id, "inputs.") { - // input.Id = "inputs." + input.Id - // } - //} - err = collection.Add(input) - if err != nil { - return - } - } + cwl_type, xerr := cwl_types.NewCWLType(cwl_object_id, elem) + if xerr != nil { + err = xerr + return + } + obj = cwl_type + default: + fmt.Printf("(New_CWL_object), unknown type %s\n", reflect.TypeOf(original)) + err = fmt.Errorf("(New_CWL_object), unknown type %s", reflect.TypeOf(original)) + return + } + fmt.Println("New_CWL_object leaving") + return +} - //fmt.Println("WORKFLOW") - //spew.Dump(workflow) - err = collection.Add(&workflow) - if err != nil { - return - } +func NewCWL_object_array(original interface{}) (array cwl_types.CWL_object_array, err error) { - //collection.Workflows = append(collection.Workflows, workflow) - //collection = append(collection, result) - case "File": - logger.Debug(1, "parse File") - var cwl_file cwl_types.File - err = mapstructure.Decode(elem, &cwl_file) - if err != nil { - err = fmt.Errorf("(Parse_cwl_document/File) %s", err.Error()) - return - } - if cwl_file.Id == "" { - cwl_file.Id = cwl_object_id - } - //collection.Files[cwl_file.Id] = cwl_file - err = collection.Add(&cwl_file) - if err != nil { + //original, err = makeStringMap(original) + //if err != nil { + // return + //} + + array = cwl_types.CWL_object_array{} + + switch original.(type) { + + case []interface{}: + + org_a := original.([]interface{}) + + for _, element := range org_a { + + cwl_object, xerr := New_CWL_object(element, "") + if xerr != nil { + err = fmt.Errorf("(NewCWL_object_array) New_CWL_object returned %s", xerr.Error()) return } - default: - err = errors.New("object unknown") - return - } // end switch - fmt.Printf("----------------------------------------------\n") + array = append(array, cwl_object) + } + + return + + default: + err = fmt.Errorf("(NewCWL_object_array), unknown type %s", reflect.TypeOf(original)) + } + return + +} + +func Parse_cwl_document(yaml_str string) (object_array cwl_types.CWL_object_array, cwl_version CWLVersion, err error) { + + graph_pos := strings.Index(yaml_str, "$graph") + + if graph_pos == -1 { + err = errors.New("yaml parisng error. keyword $graph missing: ") + return + } + + yaml_str = strings.Replace(yaml_str, "$graph", "graph", -1) // remove dollar sign + + cwl_gen := CWL_document_generic{} + + yaml_byte := []byte(yaml_str) + err = Unmarshal(&yaml_byte, &cwl_gen) + if err != nil { + logger.Debug(1, "CWL unmarshal error") + logger.Error("error: " + err.Error()) + } + + fmt.Println("-------------- raw CWL") + spew.Dump(cwl_gen) + fmt.Println("-------------- Start parsing") + + cwl_version = cwl_gen.CwlVersion + + // iterated over Graph + + fmt.Println("-------------- A Parse_cwl_document") + for count, elem := range cwl_gen.Graph { + fmt.Println("-------------- B Parse_cwl_document") + object, xerr := New_CWL_object(elem, cwl_version) + if xerr != nil { + err = fmt.Errorf("(Parse_cwl_document) New_CWL_object returns %s", xerr.Error()) + return + } + fmt.Println("-------------- C Parse_cwl_document") + object_array = append(object_array, object) + logger.Debug(3, "Added %d cwl objects...", count) + fmt.Println("-------------- loop Parse_cwl_document") } // end for + fmt.Println("-------------- finished Parse_cwl_document") + return +} + +func Add_to_collection(collection *CWL_collection, object_array cwl_types.CWL_object_array) (err error) { + + for _, object := range object_array { + err = collection.Add(object) + if err != nil { + return + } + } + return } -func Unmarshal(data []byte, v interface{}) (err error) { - err_yaml := yaml.Unmarshal(data, v) - if err_yaml != nil { - logger.Debug(1, "CWL YAML unmarshal error, (try json...) : "+err_yaml.Error()) +func Unmarshal(data_ptr *[]byte, v interface{}) (err error) { + + data := *data_ptr + + if data[0] == '{' { + err_json := json.Unmarshal(data, v) if err_json != nil { - logger.Debug(1, "CWL JSON unmarshal error: "+err_json.Error()) + logger.Debug(1, "CWL json unmarshal error: "+err_json.Error()) + err = err_json + return + } + } else { + err_yaml := yaml.Unmarshal(data, v) + if err_yaml != nil { + logger.Debug(1, "CWL yaml unmarshal error: "+err_yaml.Error()) + err = err_yaml + return } - } - if err != nil { - err = errors.New("Could not parse document as JSON or YAML") } return diff --git a/lib/core/cwl/inputParameter.go b/lib/core/cwl/inputParameter.go index 1349a3cf..35b436b4 100644 --- a/lib/core/cwl/inputParameter.go +++ b/lib/core/cwl/inputParameter.go @@ -5,19 +5,21 @@ import ( cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" "github.com/davecgh/go-spew/spew" "github.com/mitchellh/mapstructure" + + "reflect" "strings" ) type InputParameter struct { - Id string `yaml:"id"` - Label string `yaml:"label"` - SecondaryFiles []string `yaml:"secondaryFiles"` // TODO string | Expression | array - Format string `yaml:"format"` - Streamable bool `yaml:"streamable"` - Doc string `yaml:"doc"` - InputBinding CommandLineBinding `yaml:"inputBinding"` //TODO - Default cwl_types.Any `yaml:"default"` - Type *[]InputParameterType `yaml:"type"` // TODO CWLType | InputRecordSchema | InputEnumSchema | InputArraySchema | string | array + Id string `yaml:"id,omitempty" bson:"id,omitempty" json:"id,omitempty"` + Label string `yaml:"label,omitempty" bson:"label,omitempty" json:"label,omitempty"` + SecondaryFiles []string `yaml:"secondaryFiles,omitempty" bson:"secondaryFiles,omitempty" json:"secondaryFiles,omitempty"` // TODO string | Expression | array + Format string `yaml:"format,omitempty" bson:"format,omitempty" json:"format,omitempty"` + Streamable bool `yaml:"streamable,omitempty" bson:"streamable,omitempty" json:"streamable,omitempty"` + Doc string `yaml:"doc,omitempty" bson:"doc,omitempty" json:"doc,omitempty"` + InputBinding CommandLineBinding `yaml:"inputBinding,omitempty" bson:"inputBinding,omitempty" json:"inputBinding,omitempty"` //TODO + Default cwl_types.Any `yaml:"default,omitempty" bson:"default,omitempty" json:"default,omitempty"` + Type interface{} `yaml:"type,omitempty" bson:"type,omitempty" json:"type,omitempty"` // TODO CWLType | InputRecordSchema | InputEnumSchema | InputArraySchema | string | array } func (i InputParameter) GetClass() string { return "InputParameter" } @@ -27,6 +29,14 @@ func (i InputParameter) Is_CWL_minimal() {} func NewInputParameter(original interface{}) (input_parameter *InputParameter, err error) { + fmt.Println("---- NewInputParameter ----") + spew.Dump(original) + original, err = makeStringMap(original) + if err != nil { + return + } + spew.Dump(original) + input_parameter = &InputParameter{} switch original.(type) { @@ -60,9 +70,10 @@ func NewInputParameter(original interface{}) (input_parameter *InputParameter, e //} //input_parameter.Type = input_parameter_type - case map[interface{}]interface{}: - original_map := original.(map[interface{}]interface{}) + case map[string]interface{}: + + original_map := original.(map[string]interface{}) input_parameter_default, ok := original_map["default"] if ok { @@ -86,7 +97,8 @@ func NewInputParameter(original interface{}) (input_parameter *InputParameter, e return } default: - err = fmt.Errorf("(NewInputParameter) cannot parse input") + spew.Dump(original) + err = fmt.Errorf("(NewInputParameter) cannot parse input type %s", reflect.TypeOf(original)) return } diff --git a/lib/core/cwl/inputParameterType.go b/lib/core/cwl/inputParameterType.go index e8f942e5..ad062ca8 100644 --- a/lib/core/cwl/inputParameterType.go +++ b/lib/core/cwl/inputParameterType.go @@ -3,15 +3,21 @@ package cwl import ( "fmt" cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" + "github.com/davecgh/go-spew/spew" + "reflect" "strings" ) -type InputParameterType struct { - Type string -} +//type InputParameterType struct { +// Type string +//} + +type InputParameterType string func NewInputParameterType(original interface{}) (ipt_ptr *InputParameterType, err error) { + fmt.Println("---- NewInputParameterType ----") + spew.Dump(original) var ipt InputParameterType switch original.(type) { @@ -29,24 +35,28 @@ func NewInputParameterType(original interface{}) (ipt_ptr *InputParameterType, e case cwl_types.CWL_float: case cwl_types.CWL_double: case cwl_types.CWL_string: - case cwl_types.CWL_File: - case cwl_types.CWL_Directory: + case strings.ToLower(cwl_types.CWL_File): + case strings.ToLower(cwl_types.CWL_Directory): default: err = fmt.Errorf("(NewInputParameterType) type %s is unknown", original_str_lower) return } - ipt.Type = original_str + ipt = InputParameterType(original_str) + ipt_ptr = &ipt return default: + err = fmt.Errorf("(NewInputParameterType) type is not string: %s", reflect.TypeOf(original)) + return } return } func NewInputParameterTypeArray(original interface{}) (array_ptr *[]InputParameterType, err error) { - + fmt.Println("---- NewInputParameterTypeArray ----") + spew.Dump(original) array := []InputParameterType{} switch original.(type) { @@ -79,11 +89,59 @@ func NewInputParameterTypeArray(original interface{}) (array_ptr *[]InputParamet } } -func HasInputParameterType(array *[]InputParameterType, search_type string) (ok bool) { - for _, v := range *array { - if v.Type == search_type { - return true +func HasInputParameterType(array interface{}, search_type string) (ok bool, err error) { + + var array_of []interface{} + + switch array.(type) { + case *[]InputParameterType: + array_ipt_ptr, a_ok := array.(*[]InputParameterType) + if !a_ok { + err = fmt.Errorf("(HasInputParameterType) expected array A") + return + } + array_ipt := *array_ipt_ptr + for _, v := range array_ipt { + v_str := string(v) + + if v_str == search_type { + ok = true + return + } + } + + case *[]interface{}: + array_of_ptr, a_ok := array.(*[]interface{}) + if !a_ok { + err = fmt.Errorf("(HasInputParameterType) expected array A") + return + } + array_of = *array_of_ptr + case []interface{}: + var a_ok bool + array_of, a_ok = array.([]interface{}) + if !a_ok { + err = fmt.Errorf("(HasInputParameterType) expected array A") + return + } + + default: + err = fmt.Errorf("(HasInputParameterType) expected array B, got %s", reflect.TypeOf(array)) + return + } + + for _, v := range array_of { + v_str, s_ok := v.(string) + if !s_ok { + err = fmt.Errorf("(HasInputParameterType) array element is not string") + return + } + if v_str == search_type { + ok = true + return } } - return false + + ok = false + return } diff --git a/lib/core/cwl/job.go b/lib/core/cwl/job.go deleted file mode 100644 index 1cc6b359..00000000 --- a/lib/core/cwl/job.go +++ /dev/null @@ -1,118 +0,0 @@ -package cwl - -import ( - //"errors" - "fmt" - cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" - "github.com/davecgh/go-spew/spew" - "github.com/mitchellh/mapstructure" - "io/ioutil" - //"os" - //"strings" -) - -//type Job_document map[string]interface{} - -type Job_document map[string]cwl_types.CWLType - -func NewJob_document(original interface{}) (job *Job_document, err error) { - - job_nptr := Job_document{} - - job = &job_nptr - - switch original.(type) { - case map[interface{}]interface{}: - original_map := original.(map[interface{}]interface{}) - - keys := []string{} - for key, _ := range original_map { - key_str, ok := key.(string) - if !ok { - err = fmt.Errorf("key is not string") - return - } - keys = append(keys, key_str) - - } - - for _, key := range keys { - value := original_map[key] - cwl_obj, xerr := cwl_types.NewCWLType(key, value) - if xerr != nil { - err = xerr - return - } - original_map[key] = cwl_obj - } - - err = mapstructure.Decode(original, job) - if err != nil { - err = fmt.Errorf("(NewJob_document) %s", err.Error()) - return - } - case map[string]interface{}: - - original_map := original.(map[string]interface{}) - - keys := []string{} - for key_str, _ := range original_map { - - keys = append(keys, key_str) - - } - - for _, key := range keys { - value := original_map[key] - cwl_obj, xerr := cwl_types.NewCWLType(key, value) - if xerr != nil { - err = xerr - return - } - job_nptr[key] = cwl_obj - } - - //err = mapstructure.Decode(original, job) - //if err != nil { - // err = fmt.Errorf("(NewJob_document) %s", err.Error()) - // return - //} - return - - case []interface{}: - err = fmt.Errorf("(NewJob_document) type array not supported yet") - return - default: - spew.Dump(original) - err = fmt.Errorf("(NewJob_document) type unknown") - } - return -} - -func ParseJob(job_file string) (job_input *Job_document, err error) { - - job_stream, err := ioutil.ReadFile(job_file) - if err != nil { - return - } - - job_gen := map[interface{}]interface{}{} - err = Unmarshal(job_stream, &job_gen) - if err != nil { - return - } - - job_input, err = NewJob_document(job_gen) - - if err != nil { - return - } - - //fmt.Printf("-------MyCollection") - //spew.Dump(collection.All) - - //fmt.Printf("-------") - //os.Exit(0) - - return -} diff --git a/lib/core/cwl/jobinput.go b/lib/core/cwl/jobinput.go new file mode 100644 index 00000000..c0d5f351 --- /dev/null +++ b/lib/core/cwl/jobinput.go @@ -0,0 +1,155 @@ +package cwl + +import ( + //"errors" + "fmt" + cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" + "github.com/davecgh/go-spew/spew" + //"github.com/mitchellh/mapstructure" + "io/ioutil" + //"os" + //"strings" + "github.com/MG-RAST/AWE/lib/logger" + "gopkg.in/yaml.v2" + "reflect" + //"github.com/MG-RAST/AWE/lib/logger/event" +) + +//type Job_document map[string]interface{} + +type Job_document []cwl_types.CWLType + +func (job_input *Job_document) GetMap() (job_input_map map[string]cwl_types.CWLType) { + job_input_map = make(map[string]cwl_types.CWLType) + + for _, value := range *job_input { + id := value.GetId() + job_input_map[id] = value + } + return +} + +func NewJob_document(original interface{}) (job *Job_document, err error) { + + logger.Debug(3, "(NewJob_document) starting") + + job_nptr := Job_document{} + + job = &job_nptr + + switch original.(type) { + case map[interface{}]interface{}: + original_map := original.(map[interface{}]interface{}) + + for key, value := range original_map { + key_str, ok := key.(string) + if !ok { + err = fmt.Errorf("key is not string") + return + } + cwl_obj, xerr := cwl_types.NewCWLType(key_str, value) + if xerr != nil { + err = xerr + return + } + job_nptr = append(job_nptr, cwl_obj) + + } + return + case map[string]interface{}: + + original_map := original.(map[string]interface{}) + + for key_str, value := range original_map { + + cwl_obj, xerr := cwl_types.NewCWLType(key_str, value) + if xerr != nil { + err = xerr + return + } + job_nptr = append(job_nptr, cwl_obj) + + } + return + + case []interface{}: + original_array := original.([]interface{}) + + for _, value := range original_array { + + cwl_obj, xerr := cwl_types.NewCWLType("", value) + if xerr != nil { + err = xerr + return + } + job_nptr = append(job_nptr, cwl_obj) + + } + + return + default: + spew.Dump(original) + err = fmt.Errorf("(NewJob_document) type %s unknown", reflect.TypeOf(original)) + } + return +} + +func ParseJobFile(job_file string) (job_input *Job_document, err error) { + + job_stream, err := ioutil.ReadFile(job_file) + if err != nil { + return + } + + job_gen := map[interface{}]interface{}{} + err = Unmarshal(&job_stream, &job_gen) + if err != nil { + return + } + + job_input, err = NewJob_document(job_gen) + + if err != nil { + return + } + + return +} + +func ParseJob(job_byte_ptr *[]byte) (job_input *Job_document, err error) { + + // since Unmarshal (json and yaml) cannot unmarshal into interface{}, we try array and map + + job_byte := *job_byte_ptr + if job_byte[0] == '-' { + fmt.Println("yaml list") + // I guess this is a yaml list + //job_array := []cwl_types.CWLType{} + //job_array := []map[string]interface{}{} + job_array := []interface{}{} + err = yaml.Unmarshal(job_byte, &job_array) + + if err != nil { + err = fmt.Errorf("Could, not parse job input as yaml list: %s", err.Error()) + return + } + job_input, err = NewJob_document(job_array) + } else { + fmt.Println("yaml map") + + job_map := make(map[string]cwl_types.CWLType) + + err = yaml.Unmarshal(job_byte, &job_map) + if err != nil { + err = fmt.Errorf("Could, not parse job input as yaml map: %s", err.Error()) + return + } + job_input, err = NewJob_document(job_map) + } + + if err != nil { + return + } + + return +} diff --git a/lib/core/cwl/process.go b/lib/core/cwl/process.go index 39b9256e..0afd22cf 100644 --- a/lib/core/cwl/process.go +++ b/lib/core/cwl/process.go @@ -3,6 +3,9 @@ package cwl import ( "fmt" cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" + "github.com/davecgh/go-spew/spew" + "github.com/mitchellh/mapstructure" + "reflect" ) // needed for run in http://www.commonwl.org/v1.0/Workflow.html#WorkflowStep @@ -26,8 +29,34 @@ func (p *ProcessPointer) GetId() string { return p.Id } func (p *ProcessPointer) SetId(string) {} func (p *ProcessPointer) Is_CWL_minimal() {} +func NewProcessPointer(original interface{}) (pp *ProcessPointer, err error) { + + switch original.(type) { + case map[string]interface{}: + //original_map, ok := original.(map[string]interface{}) + + pp = &ProcessPointer{} + + err = mapstructure.Decode(original, pp) + if err != nil { + err = fmt.Errorf("(NewCommandInputParameter) decode error: %s", err.Error()) + return + } + return + default: + spew.Dump(original) + err = fmt.Errorf("(NewProcess) type %s unknown", reflect.TypeOf(original)) + } + return +} + // returns CommandLineTool, ExpressionTool or Workflow -func NewProcess(original interface{}, collection *CWL_collection) (process Process, err error) { +func NewProcess(original interface{}) (process interface{}, err error) { + + original, err = makeStringMap(original) + if err != nil { + return + } switch original.(type) { case string: @@ -37,9 +66,32 @@ func NewProcess(original interface{}, collection *CWL_collection) (process Proce process = pp return + case map[string]interface{}: + original_map, ok := original.(map[string]interface{}) + if !ok { + err = fmt.Errorf("(NewProcess) failed") + return + } + + var class string + class, err = cwl_types.GetClass(original_map) + if err != nil { + class = "" + } + + switch class { + case "": + return NewProcessPointer(original) + case "Workflow": + return NewWorkflow(original) + case "Expression": + return cwl_types.NewExpression(original) + + } default: - err = fmt.Errorf("(NewProcess) type unknown") + spew.Dump(original) + err = fmt.Errorf("(NewProcess) type %s unknown", reflect.TypeOf(original)) } diff --git a/lib/core/cwl/requirements.go b/lib/core/cwl/requirements.go index ff019e80..533204fc 100644 --- a/lib/core/cwl/requirements.go +++ b/lib/core/cwl/requirements.go @@ -19,6 +19,9 @@ func NewRequirement(class string, obj interface{}) (r Requirement, err error) { case "DockerRequirement": r, err = requirements.NewDockerRequirement(obj) return + case "InlineJavascriptRequirement": + r, err = requirements.NewInlineJavascriptRequirement(obj) + return case "EnvVarRequirement": r, err = requirements.NewEnvVarRequirement(obj) return @@ -31,6 +34,12 @@ func NewRequirement(class string, obj interface{}) (r Requirement, err error) { case "InitialWorkDirRequirement": r, err = requirements.NewInitialWorkDirRequirement(obj) return + case "ScatterFeatureRequirement": + r, err = requirements.NewScatterFeatureRequirement(obj) + return + case "MultipleInputFeatureRequirement": + r, err = requirements.NewMultipleInputFeatureRequirement(obj) + return default: err = errors.New("Requirement class not supported " + class) diff --git a/lib/core/cwl/requirements/baseRequirement.go b/lib/core/cwl/requirements/baseRequirement.go new file mode 100644 index 00000000..f1c770a9 --- /dev/null +++ b/lib/core/cwl/requirements/baseRequirement.go @@ -0,0 +1,7 @@ +package requirements + +type BaseRequirement struct { + Class string `yaml:"class,omitempty" json:"class,omitempty" bson:"class,omitempty"` +} + +func (c BaseRequirement) GetClass() string { return c.Class } diff --git a/lib/core/cwl/requirements/dockerRequirement.go b/lib/core/cwl/requirements/dockerRequirement.go index 1e597022..22410447 100644 --- a/lib/core/cwl/requirements/dockerRequirement.go +++ b/lib/core/cwl/requirements/dockerRequirement.go @@ -5,20 +5,21 @@ import ( ) type DockerRequirement struct { - //Class string `yaml:"class"` - DockerPull string `yaml:"dockerPull"` - DockerLoad string `yaml:"dockerLoad"` - DockerFile string `yaml:"dockerFile"` - DockerImport string `yaml:"dockerImport"` - DockerImageId string `yaml:"dockerImageId"` + BaseRequirement `bson:",inline" yaml:",inline" json:",inline"` + DockerPull string `yaml:"dockerPull,omitempty" bson:"dockerPull,omitempty" json:"dockerPull,omitempty"` + DockerLoad string `yaml:"dockerLoad,omitempty" bson:"dockerLoad,omitempty" json:"dockerLoad,omitempty"` + DockerFile string `yaml:"dockerFile,omitempty" bson:"dockerFile,omitempty" json:"dockerFile,omitempty"` + DockerImport string `yaml:"dockerImport,omitempty" bson:"dockerImport,omitempty" json:"dockerImport,omitempty"` + DockerImageId string `yaml:"dockerImageId,omitempty" bson:"dockerImageId,omitempty" json:"dockerImageId,omitempty"` } -func (c DockerRequirement) GetClass() string { return "DockerRequirement" } -func (c DockerRequirement) GetId() string { return "None" } +func (c DockerRequirement) GetId() string { return "None" } func NewDockerRequirement(original interface{}) (r *DockerRequirement, err error) { var requirement DockerRequirement r = &requirement err = mapstructure.Decode(original, &requirement) + + requirement.Class = "DockerRequirement" return } diff --git a/lib/core/cwl/requirements/envVarRequirement.go b/lib/core/cwl/requirements/envVarRequirement.go index 712d6f9a..2838ad69 100644 --- a/lib/core/cwl/requirements/envVarRequirement.go +++ b/lib/core/cwl/requirements/envVarRequirement.go @@ -6,17 +6,19 @@ import ( ) type EnvVarRequirement struct { - //Class string `yaml:"class"` - enfDef []EnvironmentDef `yaml:"enfDef"` + BaseRequirement `bson:",inline" yaml:",inline" json:",inline"` + enfDef []EnvironmentDef `yaml:"enfDef,omitempty" bson:"enfDef,omitempty" json:"enfDef,omitempty"` } -func (c EnvVarRequirement) GetClass() string { return "EnvVarRequirement" } -func (c EnvVarRequirement) GetId() string { return "None" } +func (c EnvVarRequirement) GetId() string { return "None" } func NewEnvVarRequirement(original interface{}) (r *EnvVarRequirement, err error) { var requirement EnvVarRequirement r = &requirement err = mapstructure.Decode(original, &requirement) + + requirement.Class = "EnvVarRequirement" + return } diff --git a/lib/core/cwl/requirements/initialWorkDirRequirement.go b/lib/core/cwl/requirements/initialWorkDirRequirement.go index ac55fdd5..f3b04183 100644 --- a/lib/core/cwl/requirements/initialWorkDirRequirement.go +++ b/lib/core/cwl/requirements/initialWorkDirRequirement.go @@ -6,16 +6,18 @@ import ( // http://www.commonwl.org/v1.0/CommandLineTool.html#InitialWorkDirRequirement type InitialWorkDirRequirement struct { - //Class string `yaml:"class"` - listing string `yaml:"listing"` // TODO: array | string | Expression + BaseRequirement `bson:",inline" yaml:",inline" json:",inline"` + listing string `yaml:"listing,omitempty" bson:"listing,omitempty" json:"listing,omitempty"` // TODO: array | string | Expression } -func (c InitialWorkDirRequirement) GetClass() string { return "InitialWorkDirRequirement" } -func (c InitialWorkDirRequirement) GetId() string { return "None" } +func (c InitialWorkDirRequirement) GetId() string { return "None" } func NewInitialWorkDirRequirement(original interface{}) (r *InitialWorkDirRequirement, err error) { var requirement InitialWorkDirRequirement r = &requirement err = mapstructure.Decode(original, &requirement) + + requirement.Class = "InlineJavascriptRequirement" + return } diff --git a/lib/core/cwl/requirements/inlineJavascriptRequirement.go b/lib/core/cwl/requirements/inlineJavascriptRequirement.go new file mode 100644 index 00000000..920b90ff --- /dev/null +++ b/lib/core/cwl/requirements/inlineJavascriptRequirement.go @@ -0,0 +1,22 @@ +package requirements + +import ( + "github.com/mitchellh/mapstructure" +) + +type InlineJavascriptRequirement struct { + BaseRequirement `bson:",inline" yaml:",inline" json:",inline"` + ExpressionLib []string `yaml:"expressionLib,omitempty" bson:"expressionLib,omitempty" json:"expressionLib,omitempty"` +} + +func (c InlineJavascriptRequirement) GetId() string { return "None" } + +func NewInlineJavascriptRequirement(original interface{}) (r *InlineJavascriptRequirement, err error) { + var requirement InlineJavascriptRequirement + r = &requirement + err = mapstructure.Decode(original, &requirement) + + requirement.Class = "InlineJavascriptRequirement" + + return +} diff --git a/lib/core/cwl/requirements/multipleInputFeatureRequirement.go b/lib/core/cwl/requirements/multipleInputFeatureRequirement.go new file mode 100644 index 00000000..89e906d3 --- /dev/null +++ b/lib/core/cwl/requirements/multipleInputFeatureRequirement.go @@ -0,0 +1,22 @@ +package requirements + +import ( + "github.com/mitchellh/mapstructure" +) + +// Indicates that the workflow platform must support multiple inbound data links listed in the source field of WorkflowStepInput. +type MultipleInputFeatureRequirement struct { + BaseRequirement `bson:",inline" yaml:",inline" json:",inline"` +} + +func (c MultipleInputFeatureRequirement) GetId() string { return "None" } + +func NewMultipleInputFeatureRequirement(original interface{}) (r *MultipleInputFeatureRequirement, err error) { + var requirement MultipleInputFeatureRequirement + r = &requirement + err = mapstructure.Decode(original, &requirement) + + requirement.Class = "MultipleInputFeatureRequirement" + + return +} diff --git a/lib/core/cwl/requirements/scatterFeatureRequirement.go b/lib/core/cwl/requirements/scatterFeatureRequirement.go new file mode 100644 index 00000000..0d8c999c --- /dev/null +++ b/lib/core/cwl/requirements/scatterFeatureRequirement.go @@ -0,0 +1,22 @@ +package requirements + +import ( + "github.com/mitchellh/mapstructure" +) + +//Indicates that the workflow platform must support the scatter and scatterMethod fields of WorkflowStep. +type ScatterFeatureRequirement struct { + BaseRequirement `bson:",inline" yaml:",inline" json:",inline"` +} + +func (c ScatterFeatureRequirement) GetId() string { return "None" } + +func NewScatterFeatureRequirement(original interface{}) (r *ScatterFeatureRequirement, err error) { + var requirement ScatterFeatureRequirement + r = &requirement + err = mapstructure.Decode(original, &requirement) + + requirement.Class = "ScatterFeatureRequirement" + + return +} diff --git a/lib/core/cwl/requirements/shockRequirement.go b/lib/core/cwl/requirements/shockRequirement.go index e0ed2060..7a3e13d6 100644 --- a/lib/core/cwl/requirements/shockRequirement.go +++ b/lib/core/cwl/requirements/shockRequirement.go @@ -5,15 +5,18 @@ import ( ) type ShockRequirement struct { - Host string `yaml:"host"` + BaseRequirement `bson:",inline" yaml:",inline" json:",inline"` + Host string `yaml:"host,omitempty" bson:"host,omitempty" json:"host,omitempty"` } -func (s ShockRequirement) GetClass() string { return "ShockRequirement" } -func (s ShockRequirement) GetId() string { return "None" } +func (s ShockRequirement) GetId() string { return "None" } func NewShockRequirement(original interface{}) (r *ShockRequirement, err error) { var requirement ShockRequirement r = &requirement err = mapstructure.Decode(original, &requirement) + + requirement.Class = "ShockRequirement" + return } diff --git a/lib/core/cwl/requirements/stepInputExpressionRequirement.go b/lib/core/cwl/requirements/stepInputExpressionRequirement.go index c9093f31..1b2282ff 100644 --- a/lib/core/cwl/requirements/stepInputExpressionRequirement.go +++ b/lib/core/cwl/requirements/stepInputExpressionRequirement.go @@ -8,15 +8,18 @@ import ( //http://www.commonwl.org/v1.0/Workflow.html#StepInputExpressionRequirement type StepInputExpressionRequirement struct { - //Class string `yaml:"class"` + BaseRequirement `bson:",inline" yaml:",inline" json:",inline"` } -func (c StepInputExpressionRequirement) GetClass() string { return "StepInputExpressionRequirement" } -func (c StepInputExpressionRequirement) GetId() string { return "None" } +func (c StepInputExpressionRequirement) GetId() string { return "None" } func NewStepInputExpressionRequirement(original interface{}) (r *StepInputExpressionRequirement, err error) { + var requirement StepInputExpressionRequirement r = &requirement err = mapstructure.Decode(original, &requirement) + + requirement.Class = "StepInputExpressionRequirement" + return } diff --git a/lib/core/cwl/types/array.go b/lib/core/cwl/types/array.go index 8613f3d5..a17b0924 100644 --- a/lib/core/cwl/types/array.go +++ b/lib/core/cwl/types/array.go @@ -6,15 +6,12 @@ import ( ) type Array struct { - CWL_object - Id string - Items []CWLType - Items_Type string + CWLType_Impl `yaml:",inline" json:",inline" bson:",inline" mapstructure:",squash"` + Items []CWLType `yaml:"items,omitempty" json:"items,omitempty" bson:"items,omitempty"` + Items_Type string `yaml:"items_type,omitempty" json:"items_type,omitempty" bson:"items_type,omitempty"` } func (c *Array) GetClass() string { return CWL_array } -func (c *Array) GetId() string { return c.Id } -func (c *Array) SetId(id string) { c.Id = id } func (c *Array) Is_CWL_minimal() {} func (c *Array) Is_CWLType() {} @@ -23,7 +20,7 @@ func (c *Array) Is_CommandOutputParameterType() {} func NewArray(id string, native []interface{}) (array *Array, err error) { - array = &Array{} + array = &Array{CWLType_Impl: CWLType_Impl{Id: id}} if id != "" { array.Id = id @@ -45,3 +42,11 @@ func NewArray(id string, native []interface{}) (array *Array, err error) { return } + +func (c *Array) String() string { + return "an array (TODO implement this)" +} + +func (c *Array) Add(ct CWLType) { + c.Items = append(c.Items, ct) +} diff --git a/lib/core/cwl/types/boolean.go b/lib/core/cwl/types/boolean.go index 755911e1..3e8978ed 100644 --- a/lib/core/cwl/types/boolean.go +++ b/lib/core/cwl/types/boolean.go @@ -1,14 +1,11 @@ package cwl type Boolean struct { - CWLType_Impl - Id string `yaml:"id"` - Value bool `yaml:"value"` + CWLType_Impl `bson:",inline" json:",inline" mapstructure:",squash"` + Value bool `yaml:"value,omitempty" json:"value,omitempty" bson:"value,omitempty"` } func (s *Boolean) GetClass() string { return CWL_boolean } // for CWL_object -func (s *Boolean) GetId() string { return s.Id } // for CWL_object -func (s *Boolean) SetId(id string) { s.Id = id } func (s *Boolean) String() string { if s.Value { return "True" @@ -17,3 +14,7 @@ func (s *Boolean) String() string { } func (s *Boolean) Is_CommandInputParameterType() {} // for CommandInputParameterType + +func NewBoolean(id string, value bool) *Boolean { + return &Boolean{CWLType_Impl: CWLType_Impl{Id: id}, Value: value} +} diff --git a/lib/core/cwl/types/cwl_object.go b/lib/core/cwl/types/cwl_object.go index 9ca99c42..6cfc52e0 100644 --- a/lib/core/cwl/types/cwl_object.go +++ b/lib/core/cwl/types/cwl_object.go @@ -1,5 +1,10 @@ package cwl +import ( +//"fmt" +//"reflect" +) + type CWL_object interface { CWL_minimal_interface GetClass() string @@ -7,3 +12,13 @@ type CWL_object interface { SetId(string) //is_Any() } + +type CWL_object_Impl struct { + Id string `yaml:"id,omitempty" json:"id,omitempty" bson:"id,omitempty"` + Class string `yaml:"class,omitempty" json:"class,omitempty" bson:"class,omitempty"` +} + +func (c *CWL_object_Impl) GetId() string { return c.Id } +func (c *CWL_object_Impl) SetId(id string) { c.Id = id } + +type CWL_object_array []CWL_object diff --git a/lib/core/cwl/types/cwl_type.go b/lib/core/cwl/types/cwl_type.go index 7f7a5218..e70f6e4c 100644 --- a/lib/core/cwl/types/cwl_type.go +++ b/lib/core/cwl/types/cwl_type.go @@ -3,6 +3,7 @@ package cwl import ( "fmt" "github.com/davecgh/go-spew/spew" + "gopkg.in/mgo.v2/bson" "reflect" ) @@ -86,6 +87,10 @@ type CWL_location interface { GetLocation() string } +type CWLType_Type string + +func (s CWLType_Type) Is_CommandOutputParameterType() {} + // CWLType - CWL basic types: int, string, boolean, .. etc // http://www.commonwl.org/v1.0/CommandLineTool.html#CWLType // null, boolean, int, long, float, double, string, File, Directory @@ -94,12 +99,17 @@ type CWLType interface { Is_CommandInputParameterType() Is_CommandOutputParameterType() Is_CWLType() - + String() string //Is_Array() bool //Is_CWL_minimal() } -type CWLType_Impl struct{} +type CWLType_Impl struct { + Id string `yaml:"id,omitempty" json:"id,omitempty" bson:"id,omitempty"` +} + +func (c *CWLType_Impl) GetId() string { return c.Id } +func (c *CWLType_Impl) SetId(id string) { c.Id = id } func (c *CWLType_Impl) Is_CWL_minimal() {} func (c *CWLType_Impl) Is_CWLType() {} @@ -112,6 +122,11 @@ func NewCWLType(id string, native interface{}) (cwl_type CWLType, err error) { //var cwl_type CWLType + native, err = makeStringMap(native) + if err != nil { + return + } + switch native.(type) { case []interface{}: @@ -128,7 +143,7 @@ func NewCWLType(id string, native interface{}) (cwl_type CWLType, err error) { case int: native_int := native.(int) - cwl_type = &Int{Value: native_int} + cwl_type = NewInt(id, native_int) //cwl_type = int_type.(*CWLType) @@ -138,23 +153,26 @@ func NewCWLType(id string, native interface{}) (cwl_type CWLType, err error) { case string: native_str := native.(string) - cwl_type = NewString("", native_str) + cwl_type = NewString(id, native_str) case bool: native_bool := native.(bool) - cwl_type = &Boolean{Value: native_bool} - - case map[interface{}]interface{}: + cwl_type = NewBoolean(id, native_bool) + case map[string]interface{}: + //empty, xerr := NewEmpty(native) + //if xerr != nil { + // err = xerr + // return + //} class, xerr := GetClass(native) if xerr != nil { err = xerr return } - cwl_type, err = NewCWLTypeByClass(class, native) + cwl_type, err = NewCWLTypeByClass(class, id, native) return - - case map[string]interface{}: + case map[interface{}]interface{}: //empty, xerr := NewEmpty(native) //if xerr != nil { // err = xerr @@ -165,9 +183,8 @@ func NewCWLType(id string, native interface{}) (cwl_type CWLType, err error) { err = xerr return } - cwl_type, err = NewCWLTypeByClass(class, native) + cwl_type, err = NewCWLTypeByClass(class, id, native) return - default: spew.Dump(native) err = fmt.Errorf("(NewCWLType) Type unknown: \"%s\"", reflect.TypeOf(native)) @@ -179,17 +196,17 @@ func NewCWLType(id string, native interface{}) (cwl_type CWLType, err error) { } -func NewCWLTypeByClass(class string, native interface{}) (cwl_type CWLType, err error) { +func NewCWLTypeByClass(class string, id string, native interface{}) (cwl_type CWLType, err error) { switch class { case CWL_File: - file, yerr := NewFile(native) + file, yerr := NewFile(id, native) cwl_type = &file if yerr != nil { err = yerr return } case CWL_string: - mystring, yerr := NewStringFromInterface(native) + mystring, yerr := NewStringFromInterface(id, native) cwl_type = mystring if yerr != nil { err = yerr @@ -199,7 +216,7 @@ func NewCWLTypeByClass(class string, native interface{}) (cwl_type CWLType, err // Map type unknown, maybe a record spew.Dump(native) - record, xerr := NewRecord(native) + record, xerr := NewRecord(id, native) if xerr != nil { err = xerr return @@ -210,6 +227,44 @@ func NewCWLTypeByClass(class string, native interface{}) (cwl_type CWLType, err return } +func makeStringMap(v interface{}) (result interface{}, err error) { + + switch v.(type) { + case bson.M: + + original_map := v.(bson.M) + + new_map := make(map[string]interface{}) + + for key_str, value := range original_map { + + new_map[key_str] = value + } + + result = new_map + return + case map[interface{}]interface{}: + + v_map, ok := v.(map[interface{}]interface{}) + if !ok { + err = fmt.Errorf("casting problem (b)") + return + } + v_string_map := make(map[string]interface{}) + + for key, value := range v_map { + key_string := key.(string) + v_string_map[key_string] = value + } + + result = v_string_map + return + + } + result = v + return +} + func NewCWLTypeArray_deprecated(native interface{}) (cwl_array_ptr *[]CWLType, err error) { switch native.(type) { diff --git a/lib/core/cwl/types/directory.go b/lib/core/cwl/types/directory.go index 96767bb1..3190e185 100644 --- a/lib/core/cwl/types/directory.go +++ b/lib/core/cwl/types/directory.go @@ -1,14 +1,13 @@ package cwl type Directory struct { - Id string `yaml:"id"` - Location string `yaml:"location"` - Path string `yaml:"path"` - Basename string `yaml:"basename"` - Listing []CWL_location `yaml:"basename"` + CWLType_Impl `yaml:",inline" json:",inline" bson:",inline" mapstructure:",squash"` + Location string `yaml:"location,omitempty" json:"location,omitempty" bson:"location,omitempty"` + Path string `yaml:"path,omitempty" json:"path,omitempty" bson:"path,omitempty"` + Basename string `yaml:"basename,omitempty" json:"basename,omitempty" bson:"basename,omitempty"` + Listing []CWL_location `yaml:"listing,omitempty" json:"listing,omitempty" bson:"listing,omitempty"` } func (d Directory) GetClass() string { return "Directory" } -func (d Directory) GetId() string { return d.Id } func (d Directory) String() string { return d.Path } func (d Directory) GetLocation() string { return d.Location } // for CWL_location diff --git a/lib/core/cwl/types/empty.go b/lib/core/cwl/types/empty.go index af8f1dbe..07faaa64 100644 --- a/lib/core/cwl/types/empty.go +++ b/lib/core/cwl/types/empty.go @@ -8,15 +8,13 @@ import ( // this is a generic CWL_object. Its only purpose is to retrieve the value of "class" type Empty struct { - CWLType_Impl - Id string `yaml:"id"` - Class string `yaml:"class"` + CWLType_Impl `yaml:",inline" json:",inline" bson:",inline" mapstructure:",squash"` + Class string `yaml:"class,omitempty" json:"class,omitempty" bson:"class,omitempty"` } func (e Empty) GetClass() string { return e.Class } -func (e Empty) GetId() string { return e.Id } -func (e Empty) SetId(id string) { e.Id = id } -func (e Empty) String() string { return "Empty" } + +func (e Empty) String() string { return "Empty" } func NewEmpty(value interface{}) (obj_empty *Empty, err error) { obj_empty = &Empty{} @@ -38,7 +36,18 @@ func GetClass(native interface{}) (class string, err error) { } class = empty.GetClass() - if class == "" { + return +} + +func GetId(native interface{}) (id string, err error) { + empty, xerr := NewEmpty(native) + if xerr != nil { + err = xerr + return + } + id = empty.GetId() + + if id == "" { spew.Dump(native) panic("class name empty") } diff --git a/lib/core/cwl/types/enum.go b/lib/core/cwl/types/enum.go index 84c24457..733b0b26 100644 --- a/lib/core/cwl/types/enum.go +++ b/lib/core/cwl/types/enum.go @@ -3,11 +3,8 @@ package cwl import () type Enum struct { - CWLType_Impl - Id string - Symbols []string + CWLType_Impl `yaml:",inline" json:",inline" bson:",inline" mapstructure:",squash"` + Symbols []string `yaml:"symbols,omitempty" json:"symbols,omitempty" bson:"symbols,omitempty"` } func (e *Enum) GetClass() string { return CWL_enum } -func (e *Enum) GetId() string { return e.Id } -func (e *Enum) SetId(id string) { e.Id = id } diff --git a/lib/core/cwl/types/expression.go b/lib/core/cwl/types/expression.go index ae86f9b2..37ab1573 100644 --- a/lib/core/cwl/types/expression.go +++ b/lib/core/cwl/types/expression.go @@ -2,6 +2,8 @@ package cwl import ( "fmt" + "github.com/davecgh/go-spew/spew" + "reflect" ) type Expression string @@ -30,16 +32,51 @@ func NewExpressionArray(original interface{}) (expressions *[]Expression, err er case string: expression, xerr := NewExpression(original) if xerr != nil { - err = xerr + err = fmt.Errorf("(NewExpressionArray) NewExpression returns: %s", xerr.Error) return } expressions_nptr := []Expression{*expression} expressions = &expressions_nptr case []string: - expressions_nptr := original.([]Expression) - expressions = &expressions_nptr + original_array, ok := original.([]string) + if !ok { + err = fmt.Errorf("(NewExpressionArray) type assertion error") + return + } + + expression_array := []Expression{} + + for _, value := range original_array { + + exp, xerr := NewExpression(value) + if xerr != nil { + err = fmt.Errorf("(NewExpressionArray) NewExpression returns: %s", xerr.Error()) + return + } + + expression_array = append(expression_array, *exp) + } + + expressions = &expression_array + + case []interface{}: + + string_array := []string{} + original_array := original.([]interface{}) + for _, value := range original_array { + value_str, ok := value.(string) + if !ok { + err = fmt.Errorf("(NewExpressionArray) value was not string") + return + } + string_array = append(string_array, value_str) + } + + return NewExpressionArray(string_array) + default: - err = fmt.Errorf("cannot parse Expression array, unknown type") + spew.Dump(original) + err = fmt.Errorf("(NewExpressionArray) cannot parse Expression array, unknown type %s", reflect.TypeOf(original)) } return diff --git a/lib/core/cwl/types/file.go b/lib/core/cwl/types/file.go index 6c747b31..4035e47a 100644 --- a/lib/core/cwl/types/file.go +++ b/lib/core/cwl/types/file.go @@ -12,21 +12,20 @@ import ( // http://www.commonwl.org/v1.0/Workflow.html#File type File struct { - CWLType_Impl `yaml:"-"` - Id string `yaml:"id,omitempty" json:"id"` - Class string `yaml:"class,omitempty" json:"class"` - Location string `yaml:"location,omitempty" json:"location"` // An IRI that identifies the file resource. - Location_url *url.URL `yaml:"-"` // only for internal purposes - Path string `yaml:"path,omitempty" json:"path"` // dirname + '/' + basename == path This field must be set by the implementation. - Basename string `yaml:"basename,omitempty" json:"basename"` // dirname + '/' + basename == path // if not defined, take from location - Dirname string `yaml:"dirname,omitempty" json:"dirname"` // dirname + '/' + basename == path - Nameroot string `yaml:"nameroot,omitempty" json:"nameroot"` - Nameext string `yaml:"nameext,omitempty" json:"nameext"` - Checksum string `yaml:"checksum,omitempty" json:"checksum"` - Size int32 `yaml:"size,omitempty" json:"size"` - SecondaryFiles []CWL_location `yaml:"secondaryFiles,omitempty" json:"secondaryFiles"` - Format string `yaml:"format,omitempty" json:"format"` - Contents string `yaml:"contents,omitempty" json:"contents"` + CWLType_Impl `yaml:",inline" json:",inline" bson:",inline" mapstructure:",squash"` + Class string `yaml:"class,omitempty" json:"class,omitempty bson:"class,omitempty"` + Location string `yaml:"location,omitempty" json:"location,omitempty bson:"location,omitempty"` // An IRI that identifies the file resource. + Location_url *url.URL `yaml:"-" json:"-" bson:"-"` // only for internal purposes + Path string `yaml:"path,omitempty" json:"path,omitempty bson:"path,omitempty"` // dirname + '/' + basename == path This field must be set by the implementation. + Basename string `yaml:"basename,omitempty" json:"basename,omitempty bson:"basename,omitempty"` // dirname + '/' + basename == path // if not defined, take from location + Dirname string `yaml:"dirname,omitempty" json:"dirname,omitempty bson:"dirname,omitempty"` // dirname + '/' + basename == path + Nameroot string `yaml:"nameroot,omitempty" json:"nameroot,omitempty bson:"nameroot,omitempty"` + Nameext string `yaml:"nameext,omitempty" json:"nameext,omitempty bson:"nameext,omitempty"` + Checksum string `yaml:"checksum,omitempty" json:"checksum,omitempty bson:"checksum,omitempty"` + Size int32 `yaml:"size,omitempty" json:"size,omitempty bson:"size,omitempty"` + SecondaryFiles []CWL_location `yaml:"secondaryFiles,omitempty" json:"secondaryFiles,omitempty bson:"secondaryFiles,omitempty"` + Format string `yaml:"format,omitempty" json:"format,omitempty bson:"format,omitempty"` + Contents string `yaml:"contents,omitempty" json:"contents,omitempty bson:"contents,omitempty"` // Shock node Host string `yaml:"-"` Node string `yaml:"-"` @@ -34,19 +33,24 @@ type File struct { Token string `yaml:"-"` } -func (f *File) GetClass() string { return CWL_File } -func (f *File) GetId() string { return f.Id } -func (f *File) SetId(id string) { f.Id = id } +func (f *File) GetClass() string { return CWL_File } + +//func (f *File) GetId() string { return f.Id } +//func (f *File) SetId(id string) { f.Id = id } func (f *File) String() string { return f.Path } func (f *File) GetLocation() string { return f.Location } // for CWL_location //func (f *File) Is_Array() bool { return false } func (f *File) Is_CommandInputParameterType() {} // for CommandInputParameterType -func NewFile(obj interface{}) (file File, err error) { +func NewFile(id string, obj interface{}) (file File, err error) { file, err = MakeFile("", obj) + if id != "" { + file.Id = id + } + return } diff --git a/lib/core/cwl/types/int.go b/lib/core/cwl/types/int.go index 59e2dd69..82b4bc77 100644 --- a/lib/core/cwl/types/int.go +++ b/lib/core/cwl/types/int.go @@ -1,25 +1,19 @@ package cwl import ( - "fmt" + //"fmt" "strconv" ) type Int struct { - CWLType_Impl - Id string `yaml:"id"` - Value int `yaml:"value"` + CWLType_Impl `yaml:",inline" json:",inline" bson:",inline" mapstructure:",squash"` + Value int `yaml:"value,omitempty" json:"value,omitempty" bson:"value,omitempty"` } -func (i *Int) GetClass() string { return CWL_int } -func (i *Int) GetId() string { - fmt.Printf("GetId id=%s\n", i.Id) - return i.Id -} -func (i *Int) SetId(id string) { - fmt.Printf("SetId id=%s\n", id) - fmt.Println("Hello world") - i.Id = id - fmt.Printf("SetId i.Id=%s\n", i.Id) +func NewInt(id string, value int) *Int { + return &Int{CWLType_Impl: CWLType_Impl{Id: id}, Value: value} } + +func (i *Int) GetClass() string { return CWL_int } + func (i *Int) String() string { return strconv.Itoa(i.Value) } diff --git a/lib/core/cwl/types/record.go b/lib/core/cwl/types/record.go index 92c200e8..dbc0d049 100644 --- a/lib/core/cwl/types/record.go +++ b/lib/core/cwl/types/record.go @@ -6,24 +6,26 @@ import ( ) type Record struct { - CWLType_Impl - Id string - Fields []CWLType + CWLType_Impl `yaml:",inline" json:",inline" bson:",inline" mapstructure:",squash"` + Fields []CWLType `yaml:"fields,omitempty" json:"fields,omitempty" bson:"fields,omitempty"` } func (r *Record) GetClass() string { return CWL_record } -func (r *Record) GetId() string { return r.Id } -func (r *Record) SetId(id string) { r.Id = id } func (r *Record) Is_CWL_minimal() {} func (r *Record) Is_CWLType() {} func (r *Record) Is_CommandInputParameterType() {} func (r *Record) Is_CommandOutputParameterType() {} -func NewRecord(native interface{}) (record *Record, err error) { +func NewRecord(id string, native interface{}) (record *Record, err error) { record = &Record{} + if id != "" { + err = fmt.Errorf("not sure what to do with id here") + return + } + switch native.(type) { case map[interface{}]interface{}: native_map, _ := native.(map[interface{}]interface{}) @@ -53,3 +55,7 @@ func NewRecord(native interface{}) (record *Record, err error) { return } + +func (c *Record) String() string { + return "an record (TODO implement this)" +} diff --git a/lib/core/cwl/types/string.go b/lib/core/cwl/types/string.go index ea15e019..7518053a 100644 --- a/lib/core/cwl/types/string.go +++ b/lib/core/cwl/types/string.go @@ -6,22 +6,20 @@ import ( ) type String struct { - CWLType_Impl `yaml:"-"` - Id string `yaml:"id,omitempty"` - Class string `yaml:"class,omitempty" json:"class"` - Value string `yaml:"value,omitempty"` + CWLType_Impl `yaml:",inline" json:",inline" bson:",inline" mapstructure:",squash"` + Class string `yaml:"class,omitempty" json:"class,omitempty" bson:"class,omitempty"` + Value string `yaml:"value,omitempty" json:"value,omitempty" bson:"value,omitempty"` } func (s *String) GetClass() string { return CWL_string } // for CWL_object -func (s *String) GetId() string { return s.Id } // for CWL_object -func (s *String) SetId(id string) { s.Id = id } -func (s *String) String() string { return s.Value } + +func (s *String) String() string { return s.Value } func NewString(id string, value string) (s *String) { - return &String{Class: CWL_string, Id: id, Value: value} + return &String{Class: CWL_string, CWLType_Impl: CWLType_Impl{Id: id}, Value: value} } -func NewStringFromInterface(native interface{}) (s *String, err error) { +func NewStringFromInterface(id string, native interface{}) (s *String, err error) { s = &String{Class: CWL_string} err = mapstructure.Decode(native, s) @@ -29,5 +27,10 @@ func NewStringFromInterface(native interface{}) (s *String, err error) { err = fmt.Errorf("(NewStringFromInterface) Could not convert fo string object") return } + + if id != "" { + s.Id = id + } + return } diff --git a/lib/core/cwl/workflow.go b/lib/core/cwl/workflow.go index bbcbcdf2..070673f8 100644 --- a/lib/core/cwl/workflow.go +++ b/lib/core/cwl/workflow.go @@ -2,25 +2,26 @@ package cwl import ( "fmt" - //cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" + cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" "github.com/davecgh/go-spew/spew" "github.com/mitchellh/mapstructure" //"os" - //"reflect" + "reflect" //"strings" + //"gopkg.in/mgo.v2/bson" ) type Workflow struct { - Inputs []InputParameter `yaml:"inputs"` - Outputs []WorkflowOutputParameter `yaml:"outputs"` - Id string `yaml:"id"` - Steps []WorkflowStep `yaml:"steps"` - Requirements []Requirement `yaml:"requirements"` - Hints []Requirement `yaml:"hints"` // TODO Hints may contain non-requirement objects. Give warning in those cases. - Label string `yaml:"label"` - Doc string `yaml:"doc"` - CwlVersion CWLVersion `yaml:"cwlVersion"` - Metadata map[string]interface{} `yaml:"metadata"` + cwl_types.CWL_object_Impl `bson:",inline" json:",inline" mapstructure:",squash"` // provides Id and Class fields + Inputs []InputParameter `yaml:"inputs,omitempty" bson:"inputs,omitempty" json:"inputs,omitempty"` + Outputs []WorkflowOutputParameter `yaml:"outputs,omitempty" bson:"outputs,omitempty" json:"outputs,omitempty"` + Steps []WorkflowStep `yaml:"steps,omitempty" bson:"steps,omitempty" json:"steps,omitempty"` + Requirements []interface{} `yaml:"requirements,omitempty" bson:"requirements,omitempty" json:"requirements,omitempty"` //[]Requirement + Hints []interface{} `yaml:"hints,omitempty" bson:"hints,omitempty" json:"hints,omitempty"` // []Requirement TODO Hints may contain non-requirement objects. Give warning in those cases. + Label string `yaml:"label,omitempty" bson:"label,omitempty" json:"label,omitempty"` + Doc string `yaml:"doc,omitempty" bson:"doc,omitempty" json:"doc,omitempty"` + CwlVersion CWLVersion `yaml:"cwlVersion,omitempty" bson:"cwlVersion,omitempty" json:"cwlVersion,omitempty"` + Metadata map[string]interface{} `yaml:"metadata,omitempty" bson:"metadata,omitempty" json:"metadata,omitempty"` } func (w *Workflow) GetClass() string { return "Workflow" } @@ -45,43 +46,77 @@ func GetMapElement(m map[interface{}]interface{}, key string) (value interface{} return } -func NewWorkflow(object CWL_object_generic, collection *CWL_collection) (workflow Workflow, err error) { +func NewWorkflow(original interface{}) (workflow_ptr *Workflow, err error) { // convert input map into input array - inputs, ok := object["inputs"] - if ok { - err, object["inputs"] = NewInputParameterArray(inputs) - if err != nil { - return - } + original, err = makeStringMap(original) + if err != nil { + return } - outputs, ok := object["outputs"] - if ok { - object["outputs"], err = NewWorkflowOutputParameterArray(outputs) - if err != nil { - return + workflow := Workflow{} + workflow_ptr = &workflow + + switch original.(type) { + case map[string]interface{}: + object := original.(map[string]interface{}) + inputs, ok := object["inputs"] + if ok { + err, object["inputs"] = NewInputParameterArray(inputs) + if err != nil { + return + } } - } - // convert steps to array if it is a map - steps, ok := object["steps"] - if ok { - err, object["steps"] = CreateWorkflowStepsArray(steps, collection) - if err != nil { - return + outputs, ok := object["outputs"] + if ok { + object["outputs"], err = NewWorkflowOutputParameterArray(outputs) + if err != nil { + return + } } - } - requirements, ok := object["requirements"] - if ok { - object["requirements"], err = CreateRequirementArray(requirements) + // convert steps to array if it is a map + steps, ok := object["steps"] + if ok { + err, object["steps"] = CreateWorkflowStepsArray(steps) + if err != nil { + return + } + } + + requirements, ok := object["requirements"] + if ok { + object["requirements"], err = CreateRequirementArray(requirements) + if err != nil { + return + } + } + + fmt.Printf("......WORKFLOW raw") + spew.Dump(object) + //fmt.Printf("-- Steps found ------------") // WorkflowStep + //for _, step := range elem["steps"].([]interface{}) { + + // spew.Dump(step) + + //} + + err = mapstructure.Decode(object, &workflow) if err != nil { + err = fmt.Errorf("error parsing workflow class: %s", err.Error()) return } - } + fmt.Printf(".....WORKFLOW") + spew.Dump(workflow) + return + default: + + err = fmt.Errorf("(NewWorkflow) Input type %s can not be parsed", reflect.TypeOf(original)) + return + } //switch object["requirements"].(type) { //case map[interface{}]interface{}: // Convert map of outputs into array of outputs @@ -129,21 +164,5 @@ func NewWorkflow(object CWL_object_generic, collection *CWL_collection) (workflo // //object["requirements"] = req_array //} - fmt.Printf("......WORKFLOW raw") - spew.Dump(object) - //fmt.Printf("-- Steps found ------------") // WorkflowStep - //for _, step := range elem["steps"].([]interface{}) { - - // spew.Dump(step) - - //} - - err = mapstructure.Decode(object, &workflow) - if err != nil { - err = fmt.Errorf("error parsing workflow class: %s", err.Error()) - return - } - fmt.Printf(".....WORKFLOW") - spew.Dump(workflow) return } diff --git a/lib/core/cwl/workflowOutputParameter.go b/lib/core/cwl/workflowOutputParameter.go index 38a66cea..2a437ffd 100644 --- a/lib/core/cwl/workflowOutputParameter.go +++ b/lib/core/cwl/workflowOutputParameter.go @@ -9,56 +9,70 @@ import ( ) type WorkflowOutputParameter struct { - Id string `yaml:"id"` - Label string `yaml:"label"` - SecondaryFiles []cwl_types.Expression `yaml:"secondaryFiles"` // TODO string | Expression | array - Format []cwl_types.Expression `yaml:"format"` - Streamable bool `yaml:"streamable"` - Doc string `yaml:"doc"` - OutputBinding CommandOutputBinding `yaml:"outputBinding"` //TODO - OutputSource []string `yaml:"outputSource"` - LinkMerge LinkMergeMethod `yaml:"linkMerge"` - Type []WorkflowOutputParameterType `yaml:"type"` // TODO CWLType | OutputRecordSchema | OutputEnumSchema | OutputArraySchema | string | array + Id string `yaml:"id,omitempty" bson:"id,omitempty" json:"id,omitempty"` + Label string `yaml:"label,omitempty" bson:"label,omitempty" json:"label,omitempty"` + SecondaryFiles []cwl_types.Expression `yaml:"secondaryFiles,omitempty" bson:"secondaryFiles,omitempty" json:"secondaryFiles,omitempty"` // TODO string | Expression | array + Format []cwl_types.Expression `yaml:"format,omitempty" bson:"format,omitempty" json:"format,omitempty"` + Streamable bool `yaml:"streamable,omitempty" bson:"streamable,omitempty" json:"streamable,omitempty"` + Doc string `yaml:"doc,omitempty" bson:"doc,omitempty" json:"doc,omitempty"` + OutputBinding CommandOutputBinding `yaml:"outputBinding,omitempty" bson:"outputBinding,omitempty" json:"outputBinding,omitempty"` //TODO + OutputSource []string `yaml:"outputSource,omitempty" bson:"outputSource,omitempty" json:"outputSource,omitempty"` + LinkMerge LinkMergeMethod `yaml:"linkMerge,omitempty" bson:"linkMerge,omitempty" json:"linkMerge,omitempty"` + Type []interface{} `yaml:"type,omitempty" bson:"type,omitempty" json:"type,omitempty"` //WorkflowOutputParameterType TODO CWLType | OutputRecordSchema | OutputEnumSchema | OutputArraySchema | string | array } func NewWorkflowOutputParameter(original interface{}) (wop *WorkflowOutputParameter, err error) { var output_parameter WorkflowOutputParameter - original_map, ok := original.(map[interface{}]interface{}) - if !ok { - err = fmt.Errorf("(NewWorkflowOutputParameter) type unknown") + original, err = makeStringMap(original) + if err != nil { return } - outputSource, ok := original_map["outputSource"] - if ok { - outputSource_str, ok := outputSource.(string) + switch original.(type) { + + case map[string]interface{}: + original_map, ok := original.(map[string]interface{}) + if !ok { + err = fmt.Errorf("(NewWorkflowOutputParameter) type switch error %s", err.Error()) + return + } + + outputSource, ok := original_map["outputSource"] if ok { - original_map["outputSource"] = []string{outputSource_str} + outputSource_str, ok := outputSource.(string) + if ok { + original_map["outputSource"] = []string{outputSource_str} + } } - } - wop_type, ok := original_map["type"] - if ok { + wop_type, ok := original_map["type"] + if ok { - wop_type_array, xerr := NewWorkflowOutputParameterTypeArray(wop_type) - if xerr != nil { - err = fmt.Errorf("from NewWorkflowOutputParameterTypeArray: %s", xerr.Error()) - return - } - fmt.Println("wop_type_array: \n") - fmt.Println(reflect.TypeOf(wop_type_array)) + wop_type_array, xerr := NewWorkflowOutputParameterTypeArray(wop_type) + if xerr != nil { + err = fmt.Errorf("from NewWorkflowOutputParameterTypeArray: %s", xerr.Error()) + return + } + fmt.Println("wop_type_array: \n") + fmt.Println(reflect.TypeOf(wop_type_array)) - original_map["type"] = *wop_type_array + original_map["type"] = wop_type_array - } + } - err = mapstructure.Decode(original, &output_parameter) - if err != nil { - err = fmt.Errorf("(NewWorkflowOutputParameter) decode error: %s", err.Error()) + err = mapstructure.Decode(original, &output_parameter) + if err != nil { + err = fmt.Errorf("(NewWorkflowOutputParameter) decode error: %s", err.Error()) + return + } + wop = &output_parameter + default: + err = fmt.Errorf("(NewWorkflowOutputParameter) type unknown, %s", reflect.TypeOf(original)) return + } - wop = &output_parameter + return } @@ -105,7 +119,7 @@ func NewWorkflowOutputParameterArray(original interface{}) (new_array_ptr *[]Wor default: spew.Dump(new_array) - err = fmt.Errorf("(NewWorkflowOutputParameterArray) type unknown") + err = fmt.Errorf("(NewWorkflowOutputParameterArray) type %s unknown", reflect.TypeOf(original)) } //spew.Dump(new_array) return diff --git a/lib/core/cwl/workflowOutputParameterType.go b/lib/core/cwl/workflowOutputParameterType.go index ad4c2f4a..af348711 100644 --- a/lib/core/cwl/workflowOutputParameterType.go +++ b/lib/core/cwl/workflowOutputParameterType.go @@ -6,12 +6,12 @@ import ( "github.com/davecgh/go-spew/spew" ) -type WorkflowOutputParameterType struct { - Type string - OutputRecordSchema *OutputRecordSchema - OutputEnumSchema *OutputEnumSchema - OutputArraySchema *OutputArraySchema -} +//type WorkflowOutputParameterType struct { +// Type string +// OutputRecordSchema *OutputRecordSchema +// OutputEnumSchema *OutputEnumSchema +// OutputArraySchema *OutputArraySchema +//} type OutputRecordSchema struct{} @@ -19,14 +19,16 @@ type OutputEnumSchema struct{} type OutputArraySchema struct{} -func NewWorkflowOutputParameterType(original interface{}) (wopt_ptr *WorkflowOutputParameterType, err error) { - wopt := WorkflowOutputParameterType{} - wopt_ptr = &wopt +// CWLType | OutputRecordSchema | OutputEnumSchema | OutputArraySchema | string | array + +func NewWorkflowOutputParameterType(original interface{}) (result interface{}, err error) { + //wopt := WorkflowOutputParameterType{} + //wopt_ptr = &wopt switch original.(type) { case string: - wopt.Type = original.(string) + result = original.(string) return case map[interface{}]interface{}: @@ -42,11 +44,14 @@ func NewWorkflowOutputParameterType(original interface{}) (wopt_ptr *WorkflowOut switch output_type { case "record": - wopt.OutputRecordSchema = &OutputRecordSchema{} + result = OutputRecordSchema{} + return case "enum": - wopt.OutputEnumSchema = &OutputEnumSchema{} + result = OutputEnumSchema{} + return case "array": - wopt.OutputArraySchema = &OutputArraySchema{} + result = OutputArraySchema{} + return default: err = fmt.Errorf("(NewWorkflowOutputParameterType) type %s is unknown", output_type) @@ -60,8 +65,10 @@ func NewWorkflowOutputParameterType(original interface{}) (wopt_ptr *WorkflowOut return } -func NewWorkflowOutputParameterTypeArray(original interface{}) (wopta_ptr *[]WorkflowOutputParameterType, err error) { - wopta := []WorkflowOutputParameterType{} +func NewWorkflowOutputParameterTypeArray(original interface{}) (result interface{}, err error) { + + wopta := []interface{}{} + switch original.(type) { case map[interface{}]interface{}: @@ -70,8 +77,8 @@ func NewWorkflowOutputParameterTypeArray(original interface{}) (wopta_ptr *[]Wor err = xerr return } - wopta = append(wopta, *wopt) - wopta_ptr = &wopta + wopta = append(wopta, wopt) + result = wopta return case []interface{}: logger.Debug(3, "[found array]") @@ -86,10 +93,10 @@ func NewWorkflowOutputParameterTypeArray(original interface{}) (wopta_ptr *[]Wor err = xerr return } - wopta = append(wopta, *wopt) + wopta = append(wopta, wopt) } - wopta_ptr = &wopta + result = wopta return case string: @@ -98,9 +105,9 @@ func NewWorkflowOutputParameterTypeArray(original interface{}) (wopta_ptr *[]Wor err = xerr return } - wopta = append(wopta, *wopt) + wopta = append(wopta, wopt) - wopta_ptr = &wopta + result = wopta return default: fmt.Printf("unknown type") diff --git a/lib/core/cwl/workflowStep.go b/lib/core/cwl/workflowStep.go index 51a94c02..aa648f1a 100644 --- a/lib/core/cwl/workflowStep.go +++ b/lib/core/cwl/workflowStep.go @@ -8,24 +8,30 @@ import ( ) type WorkflowStep struct { - Id string `yaml:"id"` - In []WorkflowStepInput `yaml:"in"` // array | map | map - Out []WorkflowStepOutput `yaml:"out"` - Run *Process `yaml:"run"` // Specification unclear: string | CommandLineTool | ExpressionTool | Workflow - Requirements []Requirement `yaml:"requirements"` - Hints []Requirement `yaml:"hints"` - Label string `yaml:"label"` - Doc string `yaml:"doc"` - Scatter string `yaml:"scatter"` // ScatterFeatureRequirement - ScatterMethod string `yaml:"scatterMethod"` // ScatterFeatureRequirement + Id string `yaml:"id,omitempty" bson:"id,omitempty" json:"id,omitempty"` + In []WorkflowStepInput `yaml:"in,omitempty" bson:"in,omitempty" json:",omitemptyin"` // array | map | map + Out []WorkflowStepOutput `yaml:"out,omitempty" bson:"out,omitempty" json:"out,omitempty"` + Run interface{} `yaml:"run,omitempty" bson:"run,omitempty" json:"run,omitempty"` // (*Process) Specification unclear: string | CommandLineTool | ExpressionTool | Workflow + Requirements []interface{} `yaml:"requirements,omitempty" bson:"requirements,omitempty" json:"requirements,omitempty"` //[]Requirement + Hints []interface{} `yaml:"hints,omitempty" bson:"hints,omitempty" json:"hints,omitempty"` //[]Requirement + Label string `yaml:"label,omitempty" bson:"label,omitempty" json:"label,omitempty"` + Doc string `yaml:"doc,omitempty" bson:"doc,omitempty" json:"doc,omitempty"` + Scatter string `yaml:"scatter,omitempty" bson:"scatter,omitempty" json:"scatter,omitempty"` // ScatterFeatureRequirement + ScatterMethod string `yaml:"scatterMethod,omitempty" bson:"scatterMethod,omitempty" json:"scatterMethod,omitempty"` // ScatterFeatureRequirement } -func NewWorkflowStep(original interface{}, collection *CWL_collection) (w *WorkflowStep, err error) { +func NewWorkflowStep(original interface{}) (w *WorkflowStep, err error) { var step WorkflowStep + original, err = makeStringMap(original) + if err != nil { + return + } + switch original.(type) { - case map[interface{}]interface{}: - v_map := original.(map[interface{}]interface{}) + + case map[string]interface{}: + v_map := original.(map[string]interface{}) //spew.Dump(v_map) step_in, ok := v_map["in"] @@ -47,7 +53,7 @@ func NewWorkflowStep(original interface{}, collection *CWL_collection) (w *Workf run, ok := v_map["run"] if ok { - v_map["run"], err = NewProcess(run, collection) + v_map["run"], err = NewProcess(run) if err != nil { err = fmt.Errorf("(NewWorkflowStep) run %s", err.Error()) return @@ -71,16 +77,19 @@ func NewWorkflowStep(original interface{}, collection *CWL_collection) (w *Workf return } } - + spew.Dump(v_map["run"]) err = mapstructure.Decode(original, &step) if err != nil { err = fmt.Errorf("(NewWorkflowStep) %s", err.Error()) return } w = &step + spew.Dump(w.Run) + + fmt.Println("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx") return default: - err = fmt.Errorf("(NewWorkflowStep) type unknown") + err = fmt.Errorf("(NewWorkflowStep) type %s unknown", reflect.TypeOf(original)) return } @@ -100,7 +109,7 @@ func (w WorkflowStep) GetOutput(id string) (output *WorkflowStepOutput, err erro } // CreateWorkflowStepsArray -func CreateWorkflowStepsArray(original interface{}, collection *CWL_collection) (err error, array_ptr *[]WorkflowStep) { +func CreateWorkflowStepsArray(original interface{}) (err error, array_ptr *[]WorkflowStep) { array := []WorkflowStep{} @@ -116,7 +125,7 @@ func CreateWorkflowStepsArray(original interface{}, collection *CWL_collection) fmt.Println("type: ") fmt.Println(reflect.TypeOf(v)) - step, xerr := NewWorkflowStep(v, collection) + step, xerr := NewWorkflowStep(v) if xerr != nil { err = fmt.Errorf("(CreateWorkflowStepsArray) NewWorkflowStep failed: %s", xerr.Error()) return @@ -144,7 +153,7 @@ func CreateWorkflowStepsArray(original interface{}, collection *CWL_collection) fmt.Println("type: ") fmt.Println(reflect.TypeOf(v)) - step, xerr := NewWorkflowStep(v, collection) + step, xerr := NewWorkflowStep(v) if xerr != nil { err = fmt.Errorf("(CreateWorkflowStepsArray) NewWorkflowStep failed: %s", xerr.Error()) return diff --git a/lib/core/cwl/workflowStepInput.go b/lib/core/cwl/workflowStepInput.go index f9629ccd..2dc2e938 100644 --- a/lib/core/cwl/workflowStepInput.go +++ b/lib/core/cwl/workflowStepInput.go @@ -5,15 +5,17 @@ import ( //"github.com/davecgh/go-spew/spew" cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" "github.com/mitchellh/mapstructure" + "reflect" ) //http://www.commonwl.org/v1.0/Workflow.html#WorkflowStepInput type WorkflowStepInput struct { - Id string `yaml:"id"` - Source []string `yaml:"source"` // MultipleInputFeatureRequirement - LinkMerge LinkMergeMethod `yaml:"linkMerge"` - Default cwl_types.Any `yaml:"default"` // type Any does not make sense - ValueFrom cwl_types.Expression `yaml:"valueFrom"` // StepInputExpressionRequirement + Id string `yaml:"id,omitempty" bson:"id,omitempty" json:"id,omitempty"` + Source []string `yaml:"source,omitempty" bson:"source,omitempty" json:"source,omitempty"` // MultipleInputFeatureRequirement + LinkMerge *LinkMergeMethod `yaml:"linkMerge,omitempty" bson:"linkMerge,omitempty" json:"linkMerge,omitempty"` + Default cwl_types.Any `yaml:"default,omitempty" bson:"default,omitempty" json:"default,omitempty"` // type Any does not make sense + ValueFrom cwl_types.Expression `yaml:"valueFrom,omitempty" bson:"valueFrom,omitempty" json:"valueFrom,omitempty"` // StepInputExpressionRequirement + Ready bool `yaml:"-" bson:"-" json:"-"` } func (w WorkflowStepInput) GetClass() string { return "WorkflowStepInput" } @@ -40,6 +42,11 @@ func NewWorkflowStepInput(original interface{}) (input_parameter_ptr *WorkflowSt input_parameter := WorkflowStepInput{} input_parameter_ptr = &input_parameter + original, err = makeStringMap(original) + if err != nil { + return + } + switch original.(type) { case string: @@ -49,13 +56,14 @@ func NewWorkflowStepInput(original interface{}) (input_parameter_ptr *WorkflowSt case int: fmt.Println(cwl_types.CWL_int) - input_parameter.Default = &cwl_types.Int{Id: input_parameter.Id, Value: original.(int)} + original_int := original.(int) + input_parameter.Default = cwl_types.NewInt(input_parameter.Id, original_int) return - case map[interface{}]interface{}: - fmt.Println("case map[interface{}]interface{}") + case map[string]interface{}: + fmt.Println("case map[string]interface{}") - original_map := original.(map[interface{}]interface{}) + original_map := original.(map[string]interface{}) source, ok := original_map["source"] @@ -75,22 +83,24 @@ func NewWorkflowStepInput(original interface{}) (input_parameter_ptr *WorkflowSt // TODO would it be better to do it later? // set Default field - default_value, errx := GetMapElement(original.(map[interface{}]interface{}), "default") - if errx == nil { + default_value, ok := original_map["default"] + //default_value, ok := , errx := GetMapElement(original_map, "default") + if ok { switch default_value.(type) { case string: - input_parameter.Default = &cwl_types.String{Id: input_parameter.Id, Value: default_value.(string)} + input_parameter.Default = cwl_types.NewString(input_parameter.Id, default_value.(string)) case int: - input_parameter.Default = &cwl_types.Int{Id: input_parameter.Id, Value: default_value.(int)} + input_parameter.Default = cwl_types.NewInt(input_parameter.Id, default_value.(int)) default: - err = fmt.Errorf("(NewWorkflowStepInput) string or int expected for key \"default\"") + err = fmt.Errorf("(NewWorkflowStepInput) string or int expected for key \"default\", got %s ", reflect.TypeOf(default_value)) return } } // set ValueFrom field - valueFrom_if, errx := GetMapElement(original.(map[interface{}]interface{}), "valueFrom") - if errx == nil { + //valueFrom_if, errx := GetMapElement(original.(map[interface{}]interface{}), "valueFrom") + valueFrom_if, ok := original_map["valueFrom"] + if ok { valueFrom_str, ok := valueFrom_if.(string) if !ok { err = fmt.Errorf("(NewWorkflowStepInput) cannot convert valueFrom") @@ -102,7 +112,7 @@ func NewWorkflowStepInput(original interface{}) (input_parameter_ptr *WorkflowSt default: - err = fmt.Errorf("(NewWorkflowStepInput) Input type for %s can not be parsed", input_parameter.Id) + err = fmt.Errorf("(NewWorkflowStepInput) Input type %s can not be parsed", reflect.TypeOf(original)) return } diff --git a/lib/core/cwl/workflowStepOutput.go b/lib/core/cwl/workflowStepOutput.go index ab408d7f..11eab3ff 100644 --- a/lib/core/cwl/workflowStepOutput.go +++ b/lib/core/cwl/workflowStepOutput.go @@ -7,7 +7,18 @@ import ( ) type WorkflowStepOutput struct { - Id string `yaml:"id"` + Id string `yaml:"id" bson:"id" json:"id"` +} + +func NewWorkflowStepOutput(original interface{}) (wso_ptr *WorkflowStepOutput, err error) { + + var wso WorkflowStepOutput + err = mapstructure.Decode(original, &wso) + if err != nil { + err = fmt.Errorf("(CreateWorkflowStepOutputArray) %s", err.Error()) + return + } + return } func CreateWorkflowStepOutputArray(original interface{}) (new_array []WorkflowStepOutput, err error) { @@ -18,16 +29,17 @@ func CreateWorkflowStepOutputArray(original interface{}) (new_array []WorkflowSt for k, v := range original.(map[interface{}]interface{}) { //fmt.Printf("A") - var output_parameter WorkflowStepOutput - err = mapstructure.Decode(v, &output_parameter) - if err != nil { - err = fmt.Errorf("(CreateWorkflowStepOutputArray) %s", err.Error()) + wso, xerr := NewWorkflowStepOutput(v) + //var output_parameter WorkflowStepOutput + //err = mapstructure.Decode(v, &output_parameter) + if xerr != nil { + err = fmt.Errorf("(CreateWorkflowStepOutputArray) %s", xerr.Error()) return } - output_parameter.Id = k.(string) + wso.Id = k.(string) //fmt.Printf("C") - new_array = append(new_array, output_parameter) + new_array = append(new_array, *wso) //fmt.Printf("D") } diff --git a/lib/core/cwl2awe.go b/lib/core/cwl2awe.go index 74ea25e3..451813cb 100644 --- a/lib/core/cwl2awe.go +++ b/lib/core/cwl2awe.go @@ -1,17 +1,19 @@ package core import ( - //"errors" + "errors" "fmt" - //"github.com/MG-RAST/AWE/lib/acl" + "github.com/MG-RAST/AWE/lib/acl" "github.com/MG-RAST/AWE/lib/core/cwl" - //"github.com/MG-RAST/AWE/lib/logger" - //"github.com/MG-RAST/AWE/lib/user" - //"github.com/davecgh/go-spew/spew" + cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" + "github.com/MG-RAST/AWE/lib/logger" + "github.com/MG-RAST/AWE/lib/user" + "github.com/davecgh/go-spew/spew" //aw_sequences"os" - //requirements "github.com/MG-RAST/AWE/lib/core/cwl/requirements" - //"path" + requirements "github.com/MG-RAST/AWE/lib/core/cwl/requirements" + "path" //"strconv" + //"regexp/syntax" "strings" ) @@ -49,881 +51,194 @@ func parseSourceString(source string, id string) (linked_step_name string, field return } -// func createAweTask(helper *Helper, cwl_tool *cwl.CommandLineTool, cwl_step *cwl.WorkflowStep, awe_task *Task) (err error) { -// -// // valueFrom is a StepInputExpressionRequirement -// // http://www.commonwl.org/v1.0/Workflow.html#StepInputExpressionRequirement -// -// collection := helper.collection -// //unprocessed_ws := helper.unprocessed_ws -// processed_ws := helper.processed_ws -// -// local_collection := cwl.NewCWL_collection() -// //workflowStepInputMap := make(map[string]cwl.WorkflowStepInput) -// -// linked_IO := make(map[string]*IO) -// -// spew.Dump(cwl_tool) -// spew.Dump(cwl_step) -// -// fmt.Println("=============================================== variables") -// // collect variables in local_collection -// for _, workflow_step_input := range cwl_step.In { -// // input is a cwl.WorkflowStepInput -// -// fmt.Println("workflow_step_input: ") -// spew.Dump(workflow_step_input) -// id := workflow_step_input.Id -// -// if id == "" { -// err = fmt.Errorf("(createAweTask) id is empty") -// return -// } -// -// fmt.Println("//////////////// step input: " + id) -// -// id_array := strings.Split(id, "/") -// id_base := id_array[len(id_array)-1] -// -// if len(workflow_step_input.Source) > 0 { -// logger.Debug(3, "len(workflow_step_input.Source): %d", len(workflow_step_input.Source)) -// if len(workflow_step_input.Source) == 1 { // other only via MultipleInputFeature requirement -// for _, source := range workflow_step_input.Source { // #fieldname or #stepname.fieldname -// logger.Debug(3, "working on source: %s id=%s", source, id) -// var linked_step_name string -// var fieldname string -// -// var linked_step cwl.WorkflowStep -// -// // source: # is absolute path, without its is stepname/outputname -// if strings.HasPrefix(source, "#") { -// -// // check if this is a -// source_array := strings.Split(source, "/") -// -// switch len(source_array) { -// case 1: -// err = fmt.Errorf("len(source_array) == 1 with source_array[0]=%s", source_array[0]) -// return -// case 2: -// fieldname = source_array[1] -// -// logger.Debug(3, "workflow input: %s", fieldname) -// // use workflow input as input -// -// var obj cwl.CWL_object -// job_input := *helper.collection.Job_input -// obj, ok := job_input[fieldname] -// if !ok { -// -// //for x, _ := range job_input { -// // logger.Debug(3, "Job_input key: %s", x) -// //} -// -// logger.Debug(3, "(createAweTask) %s not found in Job_input", fieldname) -// continue -// } -// -// //obj_local := *obj -// -// logger.Debug(3, "rename %s -> %s", obj.GetId(), id_base) -// -// obj.SetId(id_base) -// fmt.Printf("(createAweTask) ADDEDb %s\n", id_base) -// fmt.Printf("(createAweTask) ADDED2b %s\n", obj.GetId()) -// err = local_collection.Add(obj) -// if err != nil { -// return -// } -// -// case 3: -// linked_step_name = source_array[1] -// fieldname = source_array[2] -// logger.Debug(3, "workflowstep output: %s %s", source_array[1], source_array[2]) -// -// this_linked_step, ok := (*processed_ws)[linked_step_name] -// if !ok { -// -// // recurse into depending step -// err = cwl_step_2_awe_task(helper, linked_step_name) -// if err != nil { -// err = fmt.Errorf("(createAweTask) source empty (%s) (%s)", id, err.Error()) -// return -// } -// -// this2_linked_step, ok := (*processed_ws)[linked_step_name] -// if !ok { -// err = fmt.Errorf("(createAweTask) linked_step %s not found after processing", linked_step_name) -// return -// } -// linked_step = *this2_linked_step -// } else { -// linked_step = *this_linked_step -// } -// // linked_step is defined now. -// _ = linked_step -// -// // search output of linked_step -// -// linked_awe_task, ok := (*helper.AWE_tasks)[linked_step_name] -// if !ok { -// err = fmt.Errorf("(createAweTask) Could not find AWE task for linked step %s", linked_step_name) -// return -// } -// linked_task_id := linked_awe_task.Id -// -// linked_filename := "" -// for _, awe_output := range linked_awe_task.Outputs { -// // awe_output is an AWE IO{} object -// if awe_output.Name == fieldname { -// linked_filename = awe_output.FileName -// break -// } -// } -// -// if linked_filename == "" { -// err = fmt.Errorf("(createAweTask) Output %s not found", fieldname) -// return -// } -// //linked_output, err = linked_step.GetOutput(fieldname) -// //if err != nil { -// // return -// //} -// -// // linked_output.Id provides -// linked_IO[id] = &IO{Origin: linked_task_id, FileName: linked_filename, Name: id} -// -// default: -// err = fmt.Errorf("source too long: %s", source) -// return -// } -// -// //source_object, err := collection.Get(source) -// //if err != nil { -// // err = fmt.Errorf("Could not find source %s in collection", source) -// -// //} -// -// } else { -// err = fmt.Errorf("source without hash not implented yet: %s", source) -// return -// } -// -// //linked_step_name, fieldname, err = parseSourceString(source, id) -// //if err != nil { -// // return -// //} -// //logger.Debug(3, "source: \"%s\" linked_step_name: \"%s\" fieldname: \"%s\"", source, linked_step_name, fieldname) -// -// // find fieldname, may refer to other task, recurse in creation of that other task if necessary -// -// } -// -// } else { -// err = fmt.Errorf("MultipleInputFeatureRequirement not supported yet (id=%s)", id) -// return -// } -// -// } else { -// -// var obj *cwl.CWL_object -// obj, err = workflow_step_input.GetObject(helper.collection) // from ValueFrom or Default fields -// if err != nil { -// err = fmt.Errorf("(createAweTask) source for %s not found", id) -// return -// } -// obj_local := *obj -// -// //obj_int, ok := obj_local.(cwl.Int) -// //if ok { -// // fmt.Printf("obj_int.Id: %s\n", obj_int.Id) -// // obj_int.Id = "test" -// // fmt.Printf("obj_int.Id: %s\n", obj_int.Id) -// // fmt.Printf("obj_int.Value: %d\n", obj_int.Value) -// // spew.Dump(obj_int) -// // obj_int.SetId(id) -// // fmt.Printf("obj_int]]]]]]] %s\n", obj_int.GetId()) -// //} -// -// spew.Dump(obj_local) -// obj_local.SetId(id) -// spew.Dump(obj_local) -// fmt.Printf("ADDED %s\n", id) -// fmt.Printf("ADDED2 %s\n", obj_local.GetId()) -// err = local_collection.Add(obj_local) // add object with local input name -// if err != nil { -// return -// } -// } -// -// //workflowStepInputMap[id] = input -// // use content of input instead ": local_collection.Add(input) -// //fmt.Println("ADD local_collection.WorkflowStepInputs") -// //spew.Dump(local_collection.WorkflowStepInputs) -// } -// //fmt.Println("workflowStepInputMap:") -// //spew.Dump(workflowStepInputMap) -// -// // copy cwl_tool to AWE via step -// command_line := "" -// fmt.Println("=============================================== expected inputs") -// for _, expected_input := range cwl_tool.Inputs { -// // input is a cwl.CommandInputParameter -// fmt.Println("expected_input:") -// spew.Dump(expected_input) -// -// id := expected_input.Id -// _ = id -// expected_types := expected_input.Type -// //if len(expected_input.Type) > 1 { -// // err = fmt.Errorf("Not yet supported: len(expected_input.Type) > 1") -// // return -// //} -// -// //expected_input_parameter_type_0 := expected_input.Type[0] -// //_ = expected_input_parameter_type_0 -// //input_optional := strings.HasSuffix(expected_input_type_0, "?") TODO -// //expected_input_type := strings.ToLower(strings.TrimSuffix(expected_input_type_0, "?"))TODO -// //_ = input_optional -// //_ = expected_input_type -// -// // TODO lookup id in workflow step input -// -// //fmt.Println("local_collection.WorkflowStepInputs") -// //spew.Dump(local_collection.WorkflowStepInputs) -// //workflow_step_input, ok := workflowStepInputMap[id] -// //actual_input, xerr := local_collection.Get(id) -// //if xerr != nil { -// // err = fmt.Errorf("%s not found in workflowStepInputMap", id) -// // return -// //} -// //fmt.Println("workflow_step_input: ") -// //spew.Dump(workflow_step_input) -// -// id_array := strings.Split(id, "/") -// id_base := id_array[len(id_array)-1] -// -// input_string := "" -// found_input := false -// var obj *cwl.CWL_object -// -// logger.Debug(3, "try to find %s", id_base) -// for key, _ := range local_collection.All { -// logger.Debug(3, "local_collection key %s", key) -// } -// -// obj, err = local_collection.Get(id_base) -// -// if err == nil { -// found_input = true -// logger.Debug(3, "FOUND key %s in local_collection", id_base) -// } else { -// logger.Debug(3, "DID NOT FIND key %s", id_base) -// } -// -// if !found_input { -// // try linked_IO -// -// // TODO compare type -// io_struct, ok := linked_IO[id] -// if ok { -// awe_task.Inputs = append(awe_task.Inputs, io_struct) -// // TODO -// // io_struct.DataToken = .. -// found_input = true -// logger.Debug(3, "FOUND found_input %s in linked_IO", id) -// } -// } -// -// if !found_input { -// -// this_job_input, ok := (*collection.Job_input)[id_base] -// if ok { -// -// obj_nptr := (this_job_input).(cwl.CWL_object) -// -// obj = &obj_nptr -// found_input = true -// logger.Debug(3, "FOUND found_input %s in Job_input", id_base) -// } else { -// fmt.Println("was searching for: " + id_base) -// fmt.Println("collection.Job_input:") -// spew.Dump(collection.Job_input) -// } -// -// } -// -// if !found_input { -// // try the default value -// default_input := expected_input.Default -// if default_input != nil { -// -// // probably not a file, but some argument -// logger.Debug(3, "expected_input.Default found something") -// obj_nptr := (*default_input).(cwl.CWL_object) -// obj = &obj_nptr -// found_input = true -// logger.Debug(3, "FOUND found_input %s in expected_input.Default", id_base) -// } else { -// logger.Debug(3, "expected_input.Default found nothing") -// } -// } else { -// logger.Debug(3, "skipping expected_input.Default") -// } -// -// if !found_input { -// // check if argument is optional -// if !cwl.HasCommandInputParameterType(&expected_types, cwl.CWL_null) { -// err = fmt.Errorf("%s not found in local_collection or linked_IO, and no default found", id) -// -// fmt.Println("-------------------------------------------------") -// spew.Dump(local_collection) -// spew.Dump(linked_IO) -// return -// } -// // this argument is optional, continue.... -// continue -// -// } -// -// obj_class := (*obj).GetClass() -// switch obj_class { -// -// case cwl.CWL_File: -// -// var file_obj *cwl.File -// file_obj, ok := (*obj).(*cwl.File) //local_collection.GetFile(id_base) -// if !ok { -// err = fmt.Errorf("File %s not found in local_collection ", id_base) -// return -// } -// -// // TODO HERE HERE HERE HERE HERE HERE HERE HERE HERE HERE HERE HERE HERE -// input_string = file_obj.Basename -// -// awe_input := NewIO() -// awe_input.FileName = file_obj.Basename -// awe_input.Name = file_obj.Id -// awe_input.Host = file_obj.Host -// awe_input.Node = file_obj.Node -// //awe_input.Url=input_file. -// awe_input.DataToken = file_obj.Token -// -// awe_task.Inputs = append(awe_task.Inputs, awe_input) -// -// if input_string == "" { -// err = fmt.Errorf("input_string File is empty") -// return -// } -// -// case cwl.CWL_string: -// -// var string_obj *cwl.String -// string_obj, ok := (*obj).(*cwl.String) //local_collection.GetString(id_base) -// if !ok { -// err = fmt.Errorf("String %s not found in local_collection ", id_base) -// return -// } -// input_string = string_obj.String() -// -// if input_string == "" { -// err = fmt.Errorf("input_string String is empty") -// return -// } -// -// case cwl.CWL_int: -// -// var int_obj *cwl.Int -// int_obj, ok := (*obj).(*cwl.Int) //local_collection.GetInt(id_base) -// if !ok { -// err = fmt.Errorf("Int %s not found in local_collection ", id_base) -// return -// } -// input_string = int_obj.String() -// -// if input_string == "" { -// err = fmt.Errorf("input_string Int is empty") -// return -// } -// -// case cwl.CWL_boolean: -// input_string = "" -// -// default: -// -// err = fmt.Errorf("(expected inputs) not implemented yet (%s) id=%s", obj_class, id_base) -// return -// } -// -// input_binding := expected_input.InputBinding -// -// //if input_binding != nil { -// prefix := input_binding.Prefix -// if prefix != "" { -// -// if command_line != "" { -// command_line = command_line + " " -// } -// command_line = command_line + prefix -// -// if input_string != "" { -// command_line = command_line + " " + input_string -// } -// } -// -// //} -// // Id: #fieldname or #stepname.fieldname -// //switch input.Source { -// //case -// -// //} -// fmt.Println("command_line: ", command_line) -// } -// -// awe_task.Cmd.Name = cwl_tool.BaseCommand[0] -// if len(cwl_tool.BaseCommand) > 1 { -// awe_task.Cmd.Args = strings.Join(cwl_tool.BaseCommand[1:], " ") -// } -// awe_task.Cmd.Args += local_collection.Evaluate(command_line) -// -// for _, requirement := range cwl_tool.Hints { -// class := requirement.GetClass() -// switch class { -// case "DockerRequirement": -// -// //req_nptr := *requirement -// -// dr := requirement.(*requirements.DockerRequirement) -// //if !ok { -// // err = fmt.Errorf("Could not cast into DockerRequirement") -// // return -// //} -// if dr.DockerPull != "" { -// awe_task.Cmd.DockerPull = dr.DockerPull -// } -// -// if dr.DockerLoad != "" { -// err = fmt.Errorf("DockerRequirement.DockerLoad not supported yet") -// return -// } -// if dr.DockerFile != "" { -// err = fmt.Errorf("DockerRequirement.DockerFile not supported yet") -// return -// } -// if dr.DockerImport != "" { -// err = fmt.Errorf("DockerRequirement.DockerImport not supported yet") -// return -// } -// if dr.DockerImageId != "" { -// err = fmt.Errorf("DockerRequirement.DockerImageId not supported yet") -// return -// } -// default: -// err = fmt.Errorf("Requirement \"%s\" not supported", class) -// return -// } -// } -// -// // collect tool outputs -// tool_outputs := make(map[string]*cwl.CommandOutputParameter) -// for _, output := range cwl_tool.Outputs { -// tool_outputs[output.Id] = &output -// } -// -// process := cwl_step.Run -// process_class := (*process).GetClass() -// -// process_prefix := "" -// -// switch process_class { -// case "ProcessPointer": -// process_nptr := *process -// pp := process_nptr.(*cwl.ProcessPointer) -// -// process_prefix = pp.Value -// -// default: -// err = fmt.Errorf("Process type %s not supported yet", process_class) -// return -// } -// -// for _, output := range cwl_step.Out { -// // output is a WorkflowStepOutput, example: "#main/qc/assembly" convert to run_id + basename(output.Id) -// output_id := output.Id -// -// output_id_array := strings.Split(output_id, "/") -// output_id_base := output_id_array[len(output_id_array)-1] -// -// output_source_id := process_prefix + "/" + output_id_base -// -// // lookup glob of CommandLine tool -// cmd_out_param, ok := tool_outputs[output_source_id] -// if !ok { -// -// for key, _ := range tool_outputs { -// logger.Debug(3, "tool_outputs key: %s", key) -// } -// -// err = fmt.Errorf("output_id %s not found in tool_outputs", output_id) -// return -// } -// -// awe_output := NewIO() -// awe_output.Name = output_id -// -// // need to switch, but I do no know what type to expect. CommandOutputBinding.Type is []CommandOutputParameterType -// //switch cmd_out_param -// -// glob_array := cmd_out_param.OutputBinding.Glob -// switch len(glob_array) { -// case 0: -// err = fmt.Errorf("output_id %s glob empty", output_id) -// return -// case 1: -// glob := glob_array[0] -// -// fmt.Println("READ local_collection.WorkflowStepInputs") -// spew.Dump(local_collection.WorkflowStepInputs) -// -// glob_evaluated := local_collection.Evaluate(glob.String()) -// fmt.Printf("glob: %s -> %s", glob.String(), glob_evaluated) -// // TODO evalue glob from workflowStepInputMap -// //step_input, xerr := local_collection.Get(id) -// //if xerr != nil { -// // err = xerr -// // return -// //} -// //blubb := step_input.Default.String() -// -// awe_output.FileName = glob_evaluated -// default: -// err = fmt.Errorf("output_id %s too many globs, not supported yet", output_id) -// return -// } -// -// if awe_output.Host == "" { -// awe_output.Host = helper.job.ShockHost -// } -// awe_task.Outputs = append(awe_task.Outputs, awe_output) -// } -// -// //fmt.Println("=============================================== expected inputs") -// -// return -// } -// -// func CommandLineTool2awe_task(helper *Helper, step *cwl.WorkflowStep, cmdlinetool *cwl.CommandLineTool) (awe_task *Task, err error) { -// -// //processed_ws := helper.processed_ws -// //collection := helper.collection -// job := helper.job -// -// // TODO detect identifier URI (http://www.commonwl.org/v1.0/SchemaSalad.html#Identifier_resolution) -// // TODO: strings.Contains(step.Run, ":") or use split -// -// if len(cmdlinetool.BaseCommand) == 0 { -// err = errors.New("(cwl_step_2_awe_task) Run string empty") -// return -// } -// -// pos := len(*helper.processed_ws) -// logger.Debug(1, "pos: %d", pos) -// -// pos_str := strconv.Itoa(pos) -// logger.Debug(1, "pos_str: %s", pos_str) -// -// awe_task, err = NewTask(job, pos_str) -// if err != nil { -// err = fmt.Errorf("(cwl_step_2_awe_task) Task creation failed: %v", err) -// return -// } -// awe_task.Init(job) -// logger.Debug(1, "(cwl_step_2_awe_task) Task created: %s", awe_task.Id) -// -// (*helper.AWE_tasks)[job.Id] = awe_task -// awe_task.JobId = job.Id -// -// err = createAweTask(helper, cmdlinetool, step, awe_task) -// if err != nil { -// return -// } -// -// return -// } - -// func cwl_step_2_awe_task(helper *Helper, step_id string) (err error) { -// logger.Debug(1, "(cwl_step_2_awe_task) step_id: "+step_id) -// -// processed_ws := helper.processed_ws -// collection := helper.collection -// job := helper.job -// -// step, ok := (*helper.unprocessed_ws)[step_id] -// if !ok { -// err = fmt.Errorf("Step %s not found in unprocessed_ws", step_id) -// return -// } -// delete(*helper.unprocessed_ws, step_id) // will be added at the end of this function to processed_ws -// -// spew.Dump(step) -// process_ptr := step.Run -// process := *process_ptr -// //fmt.Println("run: " + step.Run) -// -// process_type := process.GetClass() -// -// switch process_type { -// case "CommandLineTool": -// -// cmdlinetool, ok := process.(*cwl.CommandLineTool) -// if !ok { -// err = fmt.Errorf("process type is not a CommandLineTool: %s", err.Error()) -// return -// } -// -// awe_task, xerr := CommandLineTool2awe_task(helper, step, cmdlinetool) -// if xerr != nil { -// err = xerr -// return -// } -// job.Tasks = append(job.Tasks, awe_task) -// -// (*processed_ws)[step.Id] = step -// logger.Debug(1, "(cwl_step_2_awe_task) LEAVING , step_id: "+step_id) -// case "ProcessPointer": -// -// pp, ok := process.(*cwl.ProcessPointer) -// if !ok { -// err = fmt.Errorf("ProcessPointer error") -// return -// } -// -// pp_value := pp.Value -// -// the_process_ptr, xerr := collection.Get(pp_value) -// if xerr != nil { -// err = xerr -// return -// } -// the_process := *the_process_ptr -// -// the_process_class := the_process.GetClass() -// -// // CommandLineTool | ExpressionTool | Workflow -// switch the_process_class { -// case "CommandLineTool": -// logger.Debug(1, "(cwl_step_2_awe_task) got CommandLineTool") -// -// cmdlinetool, ok := the_process.(*cwl.CommandLineTool) -// -// if !ok { -// err = fmt.Errorf("(cwl_step_2_awe_task) casting error") -// return -// } -// -// awe_task, xerr := CommandLineTool2awe_task(helper, step, cmdlinetool) -// if xerr != nil { -// err = xerr -// return -// } -// job.Tasks = append(job.Tasks, awe_task) -// -// (*processed_ws)[step.Id] = step -// -// //case cwl.ExpressionTool: -// //logger.Debug(1, "(cwl_step_2_awe_task) got ExpressionTool") -// case "Workflow": -// logger.Debug(1, "(cwl_step_2_awe_task) got Workflow") -// default: -// err = fmt.Errorf("ProcessPointer error") -// return -// } -// -// default: -// err = fmt.Errorf("process type %s unknown", process_type) -// return -// -// } -// -// //if strings.HasPrefix(step.Run, "#") { -// // '#' it is a URI relative fragment identifier -// // cmdlinetool_str = strings.TrimPrefix(step.Run, "#") -// -// //} -// -// return -// } - -// func CWL2AWE(_user *user.User, files FormFiles, cwl_workflow *cwl.Workflow, collection *cwl.CWL_collection) (job *Job, err error) { -// -// //CommandLineTools := collection.CommandLineTools -// -// // check that all expected workflow inputs exist and that they have the correct type -// logger.Debug(1, "CWL2AWE starting") -// for _, input := range cwl_workflow.Inputs { -// // input is a cwl.InputParameter object -// -// spew.Dump(input) -// -// id := input.Id -// logger.Debug(3, "Parsing workflow input %s", id) -// //if strings.HasPrefix(id, "#") { -// // workflow_prefix := cwl_workflow.Id + "/" -// // if strings.HasPrefix(id, workflow_prefix) { -// // id = strings.TrimPrefix(id, workflow_prefix) -// // logger.Debug(3, "using new id : %s", id) -// // } else { -// // err = fmt.Errorf("prefix of id %s does not match workflow id %s with workflow_prefix: %s", id, cwl_workflow.Id, workflow_prefix) -// // return -// // } -// //} -// -// id_base := path.Base(id) -// expected_types := input.Type -// -// job_input := *(collection.Job_input) -// obj_ref, ok := job_input[id_base] -// if !ok { -// -// if cwl.HasInputParameterType(expected_types, cwl.CWL_null) { // null type means parameter is optional -// continue -// } -// -// fmt.Printf("-------Collection") -// spew.Dump(collection.All) -// -// fmt.Printf("-------") -// err = fmt.Errorf("value for workflow input \"%s\" not found", id) -// return -// -// } -// -// //obj := *obj_ref -// obj_type := obj_ref.GetClass() -// logger.Debug(1, "obj_type: %s", obj_type) -// found_type := cwl.HasInputParameterType(expected_types, obj_type) -// -// if !found_type { -// //if strings.ToLower(obj_type) != strings.ToLower(expected_types) { -// fmt.Printf("object found: ") -// spew.Dump(obj_ref) -// expected_types_str := "" -// -// for _, elem := range *expected_types { -// expected_types_str += "," + elem.Type -// } -// fmt.Printf("cwl_workflow.Inputs") -// spew.Dump(cwl_workflow.Inputs) -// err = fmt.Errorf("Input %s expects types %s, but got %s)", id, expected_types_str, obj_type) -// return -// } -// -// } -// //os.Exit(0) -// job = NewJob() -// logger.Debug(1, "Job created") -// -// found_ShockRequirement := false -// for _, r := range cwl_workflow.Requirements { // TODO put ShockRequirement in Hints -// switch r.GetClass() { -// case "ShockRequirement": -// sr, ok := r.(requirements.ShockRequirement) -// if !ok { -// err = fmt.Errorf("Could not assert ShockRequirement") -// return -// } -// -// job.ShockHost = sr.Host -// found_ShockRequirement = true -// -// } -// } -// -// if !found_ShockRequirement { -// //err = fmt.Errorf("ShockRequirement has to be provided in the workflow object") -// //return -// job.ShockHost = "http://shock:7445" -// -// } -// logger.Debug(1, "Requirements checked") -// -// // Once, job has been created, set job owner and add owner to all ACL's -// job.Acl.SetOwner(_user.Uuid) -// job.Acl.Set(_user.Uuid, acl.Rights{"read": true, "write": true, "delete": true}) -// -// // TODO first check that all resources are available: local files and remote links -// -// helper := Helper{} -// -// processed_ws := make(map[string]*cwl.WorkflowStep) -// unprocessed_ws := make(map[string]*cwl.WorkflowStep) -// awe_tasks := make(map[string]*Task) -// helper.processed_ws = &processed_ws -// helper.unprocessed_ws = &unprocessed_ws -// helper.collection = collection -// helper.job = job -// helper.AWE_tasks = &awe_tasks -// -// for _, step := range cwl_workflow.Steps { -// unprocessed_ws[step.Id] = &step -// } -// -// for { -// -// // find any step to start with -// step_id := "" -// for step_id, _ = range unprocessed_ws { -// break -// } -// -// err = cwl_step_2_awe_task(&helper, step_id) -// if err != nil { -// return -// } -// //job.Tasks = append(job.Tasks, awe_tasks) -// -// if len(unprocessed_ws) == 0 { -// break -// } -// -// } -// logger.Debug(1, "cwl_step_2_awe_task done") -// // loop until all steps have been converted -// -// //err = job.InitTasks() -// //if err != nil { -// // err = fmt.Errorf("job.InitTasks() failed: %v", err) -// // return -// //} -// //logger.Debug(1, "job.InitTasks done") -// -// _, err = job.Init() -// if err != nil { -// err = fmt.Errorf("job.Init() failed: %v", err) -// return -// } -// logger.Debug(1, "Init called") -// -// err = job.Mkdir() -// if err != nil { -// err = errors.New("(CWL2AWE) error creating job directory, error=" + err.Error()) -// return -// } -// -// err = job.UpdateFile(files, "cwl") // TODO that may not make sense. Check if I should store AWE job. -// if err != nil { -// err = errors.New("error in UpdateFile, error=" + err.Error()) -// return -// } -// -// spew.Dump(job) -// -// logger.Debug(1, "job.Id: %s", job.Id) -// err = job.Save() -// if err != nil { -// err = errors.New("error in job.Save(), error=" + err.Error()) -// return -// } -// return -// -// } +func CWL_input_check(job_input *cwl.Job_document, cwl_workflow *cwl.Workflow) (err error) { + + //job_input := *(collection.Job_input) + + job_input_map := job_input.GetMap() + + for _, input := range cwl_workflow.Inputs { + // input is a cwl.InputParameter object + + spew.Dump(input) + + id := input.Id + logger.Debug(3, "Parsing workflow input %s", id) + //if strings.HasPrefix(id, "#") { + // workflow_prefix := cwl_workflow.Id + "/" + // if strings.HasPrefix(id, workflow_prefix) { + // id = strings.TrimPrefix(id, workflow_prefix) + // logger.Debug(3, "using new id : %s", id) + // } else { + // err = fmt.Errorf("prefix of id %s does not match workflow id %s with workflow_prefix: %s", id, cwl_workflow.Id, workflow_prefix) + // return + // } + //} + + id_base := path.Base(id) + expected_types := input.Type + + obj_ref, ok := job_input_map[id_base] + if !ok { + + has_type, xerr := cwl.HasInputParameterType(expected_types, cwl_types.CWL_null) + if xerr != nil { + err = xerr + return + } + if has_type { // null type means parameter is optional + continue + } + + fmt.Printf("-------job_input:") + //spew.Dump(job_input) + + fmt.Printf("-------job_input_map:") + spew.Dump(job_input_map) + panic("uhoh") + err = fmt.Errorf("value for workflow input \"%s\" not found", id) + return + + } + + //obj := *obj_ref + obj_type := obj_ref.GetClass() + logger.Debug(1, "obj_type: %s", obj_type) + has_type, xerr := cwl.HasInputParameterType(expected_types, obj_type) + if xerr != nil { + err = xerr + return + } + if !has_type { + //if strings.ToLower(obj_type) != strings.ToLower(expected_types) { + fmt.Printf("object found: ") + spew.Dump(obj_ref) + + //for _, elem := range *expected_types { + // expected_types_str += "," + string(elem) + //} + fmt.Printf("cwl_workflow.Inputs") + spew.Dump(cwl_workflow.Inputs) + err = fmt.Errorf("Input %s got %s)", id, obj_type) + return + } + + } + return +} + +func CWL2AWE(_user *user.User, files FormFiles, job_input *cwl.Job_document, cwl_workflow *cwl.Workflow, collection *cwl.CWL_collection) (job *Job, err error) { + + //CommandLineTools := collection.CommandLineTools + + // check that all expected workflow inputs exist and that they have the correct type + logger.Debug(1, "CWL2AWE starting") + + err = CWL_input_check(job_input, cwl_workflow) + if err != nil { + return + } + + //os.Exit(0) + job = NewJob() + //job.CWL_workflow = cwl_workflow + + logger.Debug(1, "Job created") + + found_ShockRequirement := false + for _, r := range cwl_workflow.Requirements { // TODO put ShockRequirement in Hints + req, ok := r.(cwl.Requirement) + if !ok { + err = fmt.Errorf("not a requirement") + return + } + switch req.GetClass() { + case "ShockRequirement": + sr, ok := req.(requirements.ShockRequirement) + if !ok { + err = fmt.Errorf("Could not assert ShockRequirement") + return + } + + job.ShockHost = sr.Host + found_ShockRequirement = true + + } + } + + if !found_ShockRequirement { + //err = fmt.Errorf("ShockRequirement has to be provided in the workflow object") + //return + job.ShockHost = "http://shock:7445" + + } + logger.Debug(1, "Requirements checked") + + // Once, job has been created, set job owner and add owner to all ACL's + job.Acl.SetOwner(_user.Uuid) + job.Acl.Set(_user.Uuid, acl.Rights{"read": true, "write": true, "delete": true}) + + // TODO first check that all resources are available: local files and remote links + + //helper := Helper{} + + //processed_ws := make(map[string]*cwl.WorkflowStep) + //unprocessed_ws := make(map[string]*cwl.WorkflowStep) + //awe_tasks := make(map[string]*Task) + //helper.processed_ws = &processed_ws + //helper.unprocessed_ws = &unprocessed_ws + //helper.collection = collection + //helper.job = job + //helper.AWE_tasks = &awe_tasks + + for _, step := range cwl_workflow.Steps { + //task_name := strings.Map( + // func(r rune) rune { + // if syntax.IsWordChar(r) || r == '/' || r == '-' { // word char: [0-9A-Za-z_] + // return r + // } + // return -1 + // }, + // step.Id) + + if !strings.HasPrefix(step.Id, "#") { + err = fmt.Errorf("Workflow step name does not start with a #: %s", step.Id) + return + } + + task_name := step.Id + awe_task := NewTask(job, task_name) + awe_task.WorkflowStep = &step + job.Tasks = append(job.Tasks, awe_task) + } + + _, err = job.Init() + if err != nil { + err = fmt.Errorf("job.Init() failed: %s", err.Error()) + return + } + logger.Debug(1, "Init called") + + err = job.Mkdir() + if err != nil { + err = errors.New("(CWL2AWE) error creating job directory, error=" + err.Error()) + return + } + + err = job.UpdateFile(files, "cwl") // TODO that may not make sense. Check if I should store AWE job. + if err != nil { + err = errors.New("error in UpdateFile, error=" + err.Error()) + return + } + + spew.Dump(job) + + logger.Debug(1, "job.Id: %s", job.Id) + err = job.Save() + if err != nil { + err = errors.New("error in job.Save(), error=" + err.Error()) + return + } + return + +} diff --git a/lib/core/cwl_workunit.go b/lib/core/cwl_workunit.go new file mode 100644 index 00000000..66d9ea0c --- /dev/null +++ b/lib/core/cwl_workunit.go @@ -0,0 +1,90 @@ +package core + +import ( + "github.com/MG-RAST/AWE/lib/core/cwl" + //cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" + //"github.com/davecgh/go-spew/spew" + "fmt" +) + +type CWL_workunit struct { + Job_input *cwl.Job_document `bson:"job_input,omitempty" json:"job_input,omitempty"` + Job_input_filename string `bson:"job_input_filename,omitempty" json:"job_input_filename,omitempty"` + CWL_tool *cwl.CommandLineTool `bson:"cwl_tool,omitempty" json:"cwl_tool,omitempty"` + CWL_tool_filename string `bson:"cwl_tool_filename,omitempty" json:"cwl_tool_filename,omitempty"` + Tool_results *cwl.Job_document `bson:"tool_results,omitempty" json:"tool_results,omitempty"` + OutputsExpected *cwl.WorkflowStepOutput `bson:"outputs_expected,omitempty" json:"outputs_expected,omitempty"` // this is the subset of outputs that are needed by the workflow +} + +func NewCWL_workunit() *CWL_workunit { + return &CWL_workunit{ + Job_input: nil, + CWL_tool: nil, + Tool_results: nil, + OutputsExpected: nil, + } + +} + +func NewCWL_workunit_from_interface(native interface{}) (workunit *CWL_workunit, err error) { + + workunit = &CWL_workunit{} + + switch native.(type) { + + case map[string]interface{}: + + native_map, ok := native.(map[string]interface{}) + if !ok { + err = fmt.Errorf("(NewCWL_workunit_from_interface) type error") + return + } + + job_input_generic, has_job_input_generic := native_map["job_input"] + if has_job_input_generic { + + job_input, xerr := cwl.NewJob_document(job_input_generic) + if xerr != nil { + err = fmt.Errorf("(NewCWL_workunit_from_interface) NewJob_document failed: %s", xerr.Error()) + return + } + workunit.Job_input = job_input + + } + + workunit.Job_input_filename, _ = native_map["Job_input_filename"].(string) + workunit.CWL_tool_filename, _ = native_map["CWL_tool_filename"].(string) + + outputs_expected_generic, has_outputs_expected := native_map["outputs_expected"] + if has_outputs_expected { + + outputs_expected, xerr := cwl.NewWorkflowStepOutput(outputs_expected_generic) + if xerr != nil { + err = fmt.Errorf("(NewCWL_workunit_from_interface) NewWorkflowStepOutput failed: %s", xerr.Error()) + return + } + + workunit.OutputsExpected = outputs_expected + } + + cwl_tool_generic, has_cwl_tool_generic := native_map["cwl_tool"] + if has_cwl_tool_generic { + + cwl_tool, xerr := cwl.NewCommandLineTool(cwl_tool_generic) + if xerr != nil { + err = fmt.Errorf("(NewCWL_workunit_from_interface) NewCommandLineTool failed: %s", xerr.Error()) + return + } + workunit.CWL_tool = cwl_tool + + } + + default: + err = fmt.Errorf("(NewCWL_workunit_from_interface) wrong type, map expected") + return + + } + + return + +} diff --git a/lib/core/db.go b/lib/core/db.go index fd9a1077..f3f6abd6 100644 --- a/lib/core/db.go +++ b/lib/core/db.go @@ -286,7 +286,7 @@ func dbGetJobTaskField(job_id string, task_id string, fieldname string, result * return } - logger.Debug(3, "GOT: %v", temp_result) + //logger.Debug(3, "GOT: %v", temp_result) tasks_unknown := temp_result["tasks"] tasks, ok := tasks_unknown.([]interface{}) @@ -309,7 +309,7 @@ func dbGetJobTaskField(job_id string, task_id string, fieldname string, result * } result.Data = test_result - logger.Debug(3, "GOT2: %v", result) + //logger.Debug(3, "GOT2: %v", result) logger.Debug(3, "(dbGetJobTaskField) %s got something", fieldname) return } @@ -362,6 +362,7 @@ func dbGetJobFieldInt(job_id string, fieldname string) (result int, err error) { func dbGetJobFieldString(job_id string, fieldname string) (result string, err error) { err = dbGetJobField(job_id, fieldname, &result) + logger.Debug(3, "(dbGetJobFieldString) result: %s", result) if err != nil { return } @@ -491,7 +492,12 @@ func dbUpdateJobTaskInt(job_id string, task_id string, fieldname string, value i func dbUpdateJobTaskString(job_id string, task_id string, fieldname string, value string) (err error) { update_value := bson.M{"tasks.$." + fieldname: value} - return dbUpdateJobTaskFields(job_id, task_id, update_value) + err = dbUpdateJobTaskFields(job_id, task_id, update_value) + + if err != nil { + err = fmt.Errorf(" job_id=%s, task_id=%s, fieldname=%s, value=%s error=%s", job_id, task_id, fieldname, value, err.Error()) + } + return } func dbUpdateJobTaskTime(job_id string, task_id string, fieldname string, value time.Time) (err error) { @@ -558,12 +564,13 @@ func LoadJob(id string) (job *Job, err error) { err = c.Find(bson.M{"id": id}).One(&job) if err != nil { job = nil + err = fmt.Errorf("(LoadJob) c.Find failed: %s", err.Error()) return } changed, xerr := job.Init() if xerr != nil { - err = xerr + err = fmt.Errorf("(LoadJob) job.Init failed: %s", xerr.Error()) return } diff --git a/lib/core/info.go b/lib/core/info.go index 1754e705..8d136421 100644 --- a/lib/core/info.go +++ b/lib/core/info.go @@ -7,23 +7,24 @@ import ( //job info type Info struct { - Name string `bson:"name" json:"name"` - Xref string `bson:"xref" json:"xref"` - Service string `bson:"service" json:"service"` - Project string `bson:"project" json:"project"` - User string `bson:"user" json:"user"` - Pipeline string `bson:"pipeline" json:"pipeline"` - ClientGroups string `bson:"clientgroups" json:"clientgroups"` - SubmitTime time.Time `bson:"submittime" json:"submittime"` - StartedTme time.Time `bson:"startedtime" json:"startedtime"` - CompletedTime time.Time `bson:"completedtime" json:"completedtime"` - Priority int `bson:"priority" json:"priority"` - Auth bool `bson:"auth" json:"auth"` - DataToken string `bson:"datatoken" json:"-"` - NoRetry bool `bson:"noretry" json:"noretry"` - UserAttr map[string]string `bson:"userattr" json:"userattr"` - Description string `bson:"description" json:"description"` - Tracking bool `bson:"tracking" json:"tracking"` + Name string `bson:"name" json:"name" mapstructure:"name"` + Xref string `bson:"xref" json:"xref" mapstructure:"xref"` + Service string `bson:"service" json:"service" mapstructure:"service"` + Project string `bson:"project" json:"project" mapstructure:"project"` + User string `bson:"user" json:"user" mapstructure:"user"` + Pipeline string `bson:"pipeline" json:"pipeline" mapstructure:"pipeline"` // or workflow + ClientGroups string `bson:"clientgroups" json:"clientgroups" mapstructure:"clientgroups"` + SubmitTime time.Time `bson:"submittime" json:"submittime" mapstructure:"submittime"` + StartedTime time.Time `bson:"startedtime" json:"startedtime" mapstructure:"startedtime"` + CompletedTime time.Time `bson:"completedtime" json:"completedtime" mapstructure:"completedtime"` + Priority int `bson:"priority" json:"priority" mapstructure:"priority"` + Auth bool `bson:"auth" json:"auth" mapstructure:"auth"` + DataToken string `bson:"datatoken" json:"-" mapstructure:"-"` + NoRetry bool `bson:"noretry" json:"noretry" mapstructure:"noretry"` + UserAttr map[string]string `bson:"userattr" json:"userattr" mapstructure:"userattr"` + Description string `bson:"description" json:"description" mapstructure:"description"` + Tracking bool `bson:"tracking" json:"tracking" mapstructure:"tracking"` + StartAt time.Time `bson:"start_at" json:"start_at" mapstructure:"start_at"` // will start tasks at this timepoint or shortly after } func NewInfo() *Info { @@ -32,3 +33,13 @@ func NewInfo() *Info { Priority: conf.BasePriority, } } + +func (this *Info) SetStartedTime(jobid string, t time.Time) (err error) { + + err = DbUpdateJobField(jobid, "info.startedtime", t) + if err != nil { + return + } + this.StartedTime = t + return +} diff --git a/lib/core/io.go b/lib/core/io.go index 1929c931..27bf83da 100644 --- a/lib/core/io.go +++ b/lib/core/io.go @@ -11,41 +11,41 @@ import ( ) type IO struct { - FileName string `bson:"filename" json:"filename"` - Name string `bson:"name" json:"name"` // specifies abstract name of output as defined by the app - AppPosition int `bson:"appposition" json:"-"` // specifies position in app output array - Directory string `bson:"directory" json:"directory"` - Host string `bson:"host" json:"host"` - Node string `bson:"node" json:"node"` - Url string `bson:"url" json:"url"` // can be shock or any other url - Size int64 `bson:"size" json:"size"` - MD5 string `bson:"md5" json:"-"` - Cache bool `bson:"cache" json:"cache"` // indicates that this files is "predata"" that needs to be cached - Origin string `bson:"origin" json:"origin"` - Path string `bson:"path" json:"-"` - Optional bool `bson:"optional" json:"-"` - Nonzero bool `bson:"nonzero" json:"nonzero"` - DataToken string `bson:"datatoken" json:"-"` - Intermediate bool `bson:"Intermediate" json:"-"` - Temporary bool `bson:"temporary" json:"temporary"` - ShockFilename string `bson:"shockfilename" json:"shockfilename"` - ShockIndex string `bson:"shockindex" json:"shockindex"` // on input it indicates that Shock node has to be indexed by AWE server - AttrFile string `bson:"attrfile" json:"attrfile"` - NoFile bool `bson:"nofile" json:"nofile"` - Delete bool `bson:"delete" json:"delete"` // speficies that this is a temorary node, to be deleted from shock on job completion - Type string `bson:"type" json:"type"` - NodeAttr map[string]interface{} `bson:"nodeattr" json:"nodeattr"` // specifies attribute data to be stored in shock node (output only) - FormOptions map[string]string `bson:"formoptions" json:"formoptions"` - Uncompress string `bson:"uncompress" json:"uncompress"` // tells AWE client to uncompress this file, e.g. "gzip" - Indexes map[string]shock.IdxInfo `bson:"-" json:"-"` // copy of shock node.Indexes + FileName string `bson:"filename" json:"filename" mapstructure:"filename"` + Name string `bson:"name" json:"name" mapstructure:"name"` // specifies abstract name of output as defined by the app + AppPosition int `bson:"appposition" json:"-" mapstructure:"-"` // specifies position in app output array + Directory string `bson:"directory" json:"directory" mapstructure:"directory"` + Host string `bson:"host" json:"host" mapstructure:"host"` + Node string `bson:"node" json:"node" mapstructure:"node"` + Url string `bson:"url" json:"url" mapstructure:"url"` // can be shock or any other url + Size int64 `bson:"size" json:"size" mapstructure:"size"` + MD5 string `bson:"md5" json:"-" mapstructure:"-"` + Cache bool `bson:"cache" json:"cache" mapstructure:"cache"` // indicates that this files is "predata"" that needs to be cached + Origin string `bson:"origin" json:"origin" mapstructure:"origin"` + Path string `bson:"path" json:"-" mapstructure:"-"` + Optional bool `bson:"optional" json:"-" mapstructure:"-"` + Nonzero bool `bson:"nonzero" json:"nonzero" mapstructure:"nonzero"` + DataToken string `bson:"datatoken" json:"-" mapstructure:"-"` + Intermediate bool `bson:"Intermediate" json:"-" mapstructure:"-"` + Temporary bool `bson:"temporary" json:"temporary" mapstructure:"temporary"` + ShockFilename string `bson:"shockfilename" json:"shockfilename" mapstructure:"shockfilename"` + ShockIndex string `bson:"shockindex" json:"shockindex" mapstructure:"shockindex"` // on input it indicates that Shock node has to be indexed by AWE server + AttrFile string `bson:"attrfile" json:"attrfile" mapstructure:"attrfile"` + NoFile bool `bson:"nofile" json:"nofile" mapstructure:"nofile"` + Delete bool `bson:"delete" json:"delete" mapstructure:"delete"` // speficies that this is a temorary node, to be deleted from shock on job completion + Type string `bson:"type" json:"type" mapstructure:"type"` + NodeAttr map[string]interface{} `bson:"nodeattr" json:"nodeattr" mapstructure:"nodeattr"` // specifies attribute data to be stored in shock node (output only) + FormOptions map[string]string `bson:"formoptions" json:"formoptions" mapstructure:"formoptions"` + Uncompress string `bson:"uncompress" json:"uncompress" mapstructure:"uncompress"` // tells AWE client to uncompress this file, e.g. "gzip" + Indexes map[string]shock.IdxInfo `bson:"-" json:"-" mapstructure:"-"` // copy of shock node.Indexes } type PartInfo struct { - Input string `bson:"input" json:"input"` - Index string `bson:"index" json:"index"` - TotalIndex int `bson:"totalindex" json:"totalindex"` - MaxPartSizeMB int `bson:"maxpartsize_mb" json:"maxpartsize_mb"` - Options string `bson:"options" json:"-"` + Input string `bson:"input" json:"input" mapstructure:"input"` + Index string `bson:"index" json:"index" mapstructure:"index"` + TotalIndex int `bson:"totalindex" json:"totalindex" mapstructure:"totalindex"` + MaxPartSizeMB int `bson:"maxpartsize_mb" json:"maxpartsize_mb" mapstructure:"maxpartsize_mb"` + Options string `bson:"options" json:"-" mapstructure:"-"` } // Deprecated JobDep struct uses deprecated TaskDep struct which uses the deprecated IOmap. Maintained for backwards compatibility. @@ -84,22 +84,31 @@ func NewIO() *IO { return &IO{} } +func (io *IO) Url2Shock() (err error) { + u, _ := url.Parse(io.Url) + if (u.Scheme == "") || (u.Host == "") || (u.Path == "") { + err = fmt.Errorf("Not a valid url: %s", io.Url) + return + } + // get shock info from url + if (io.Host == "") || (io.Node == "") || (io.Node == "-") { + trimPath := strings.Trim(u.Path, "/") + cleanUuid := strings.Trim(strings.TrimPrefix(trimPath, "node"), "/") + // appears to be a shock url + if (cleanUuid != trimPath) && (uuid.Parse(cleanUuid) != nil) { + io.Host = u.Scheme + "://" + u.Host + io.Node = cleanUuid + } + } + return +} + func (io *IO) DataUrl() (dataurl string, err error) { if io.Url != "" { // parse and test url - u, _ := url.Parse(io.Url) - if (u.Scheme == "") || (u.Host == "") || (u.Path == "") { - return "", errors.New("Not a valid url: " + io.Url) - } - // get shock info from url - if (io.Host == "") || (io.Node == "") || (io.Node == "-") { - trimPath := strings.Trim(u.Path, "/") - cleanUuid := strings.Trim(strings.TrimPrefix(trimPath, "node"), "/") - // appears to be a shock url - if (cleanUuid != trimPath) && (uuid.Parse(cleanUuid) != nil) { - io.Host = u.Scheme + "://" + u.Host - io.Node = cleanUuid - } + err = io.Url2Shock() + if err != nil { + return } return io.Url, nil } else if (io.Host != "") && (io.Node != "") && (io.Node != "-") { diff --git a/lib/core/job.go b/lib/core/job.go index 4b8aede8..2bc6764a 100644 --- a/lib/core/job.go +++ b/lib/core/job.go @@ -1,27 +1,33 @@ package core import ( + b64 "encoding/base64" + "encoding/json" "errors" "fmt" "github.com/MG-RAST/AWE/lib/acl" "github.com/MG-RAST/AWE/lib/conf" + "github.com/MG-RAST/AWE/lib/core/cwl" + //cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" "github.com/MG-RAST/AWE/lib/core/uuid" "github.com/MG-RAST/AWE/lib/logger" "github.com/MG-RAST/AWE/lib/logger/event" + "github.com/davecgh/go-spew/spew" "gopkg.in/mgo.v2/bson" "io/ioutil" "os" "path" "path/filepath" "strconv" - //"strings" + "strings" "time" ) const ( - JOB_STAT_INIT = "init" - JOB_STAT_QUEUED = "queued" - JOB_STAT_INPROGRESS = "in-progress" + JOB_STAT_INIT = "init" // inital state + JOB_STAT_QUEUING = "queuing" // transition from "init" to "queued" + JOB_STAT_QUEUED = "queued" // all tasks have been added to taskmap + JOB_STAT_INPROGRESS = "in-progress" // a first task went into state in-progress JOB_STAT_COMPLETED = "completed" JOB_STAT_SUSPEND = "suspend" JOB_STAT_FAILED_PERMANENT = "failed-permanent" // this sepcific error state can be trigger by the workflow software @@ -29,7 +35,7 @@ const ( ) var JOB_STATS_ACTIVE = []string{JOB_STAT_QUEUED, JOB_STAT_INPROGRESS} -var JOB_STATS_REGISTERED = []string{JOB_STAT_QUEUED, JOB_STAT_INPROGRESS, JOB_STAT_SUSPEND} +var JOB_STATS_REGISTERED = []string{JOB_STAT_QUEUING, JOB_STAT_QUEUED, JOB_STAT_INPROGRESS, JOB_STAT_SUSPEND} var JOB_STATS_TO_RECOVER = []string{JOB_STAT_INIT, JOB_STAT_QUEUED, JOB_STAT_INPROGRESS, JOB_STAT_SUSPEND} type JobError struct { @@ -42,28 +48,35 @@ type JobError struct { Status string `bson:"status" json:"status"` } -type JobRaw struct { - RWMutex - Id string `bson:"id" json:"id"` - //Tasks []*Task `bson:"tasks" json:"tasks"` - Acl acl.Acl `bson:"acl" json:"-"` - Info *Info `bson:"info" json:"info"` - Script script `bson:"script" json:"-"` - State string `bson:"state" json:"state"` - Registered bool `bson:"registered" json:"registered"` - RemainTasks int `bson:"remaintasks" json:"remaintasks"` - Expiration time.Time `bson:"expiration" json:"expiration"` // 0 means no expiration - UpdateTime time.Time `bson:"updatetime" json:"updatetime"` - Error *JobError `bson:"error" json:"error"` // error struct exists when in suspended state - Resumed int `bson:"resumed" json:"resumed"` // number of times the job has been resumed from suspension - ShockHost string `bson:"shockhost" json:"shockhost"` // this is a fall-back default if not specified at a lower level -} - type Job struct { JobRaw `bson:",inline"` Tasks []*Task `bson:"tasks" json:"tasks"` } +type JobRaw struct { + RWMutex + Id string `bson:"id" json:"id"` // uuid + Acl acl.Acl `bson:"acl" json:"-"` + Info *Info `bson:"info" json:"info"` + Script script `bson:"script" json:"-"` + State string `bson:"state" json:"state"` + Registered bool `bson:"registered" json:"registered"` + RemainTasks int `bson:"remaintasks" json:"remaintasks"` + Expiration time.Time `bson:"expiration" json:"expiration"` // 0 means no expiration + UpdateTime time.Time `bson:"updatetime" json:"updatetime"` + Error *JobError `bson:"error" json:"error"` // error struct exists when in suspended state + Resumed int `bson:"resumed" json:"resumed"` // number of times the job has been resumed from suspension + ShockHost string `bson:"shockhost" json:"shockhost"` // this is a fall-back default if not specified at a lower level + IsCWL bool `bson:"is_cwl" json:"is_cwl` + CwlVersion cwl.CWLVersion `bson:"cwl_version" json:"cwl_version"` + CWL_objects interface{} `bson:"cwl_objects" json:"cwl_objects` + CWL_job_input interface{} `bson:"cwl_job_input" json:"cwl_job_input` + + CWL_collection *cwl.CWL_collection `bson:"-" json:"-" yaml:"-" mapstructure:"-"` + //CWL_job_input *cwl.Job_document `bson:"-" json:"-" yaml:"-" mapstructure:"-"` + CWL_workflow *cwl.Workflow `bson:"-" json:"-" yaml:"-" mapstructure:"-"` +} + // Deprecated JobDep struct uses deprecated TaskDep struct which uses the deprecated IOmap. Maintained for backwards compatibility. // Jobs that cannot be parsed into the Job struct, but can be parsed into the JobDep struct will be translated to the new Job struct. // (=deprecated=) @@ -155,7 +168,8 @@ func (job *Job) Init() (changed bool, err error) { if task.Id == "" { // suspend and create error logger.Error("(job.Init) task.Id empty, job %s broken?", job.Id) - task.Id = job.Id + "_" + uuid.New() + //task.Id = job.Id + "_" + uuid.New() + task.Id = uuid.New() job.State = JOB_STAT_SUSPEND job.Error = &JobError{ ServerNotes: "task.Id was empty", @@ -212,13 +226,115 @@ func (job *Job) Init() (changed bool, err error) { inputFileNames := make(map[string]bool) for _, io := range task.Inputs { if _, exists := inputFileNames[io.FileName]; exists { - err = errors.New("invalid inputs: task " + task.Id + " contains multiple inputs with filename=" + io.FileName) + err = errors.New("(job.Init) invalid inputs: task " + task.Id + " contains multiple inputs with filename=" + io.FileName) return } inputFileNames[io.FileName] = true } } + //var workflow *cwl.Workflow + + if job.IsCWL { + + //if job.CWL_workflow_interface != nil { + // workflow, err = cwl.NewWorkflow(job.CWL_workflow_interface) + // if err != nil { + // return + // } + // job.CWL_workflow = workflow + //} + + //var job_input *cwl.Job_document + //if job.CWL_job_input != nil { + // job_input, err = cwl.NewJob_document(job.CWL_job_input) + // if err != nil { + // return + // } + // job.CWL_job_input = job_input + //} + collection := cwl.NewCWL_collection() + + object_array, xerr := cwl.NewCWL_object_array(job.CWL_objects) + if xerr != nil { + err = fmt.Errorf("(job.Init) cannot type assert CWL_objects: %s", xerr.Error()) + return + } + //object_array, ok := job.CWL_objects.(cwl_types.CWL_object_array) + //if !ok { + // spew.Dump(job.CWL_objects) + // err = fmt.Errorf("(job.Init) cannot type assert CWL_objects") + // return + //} + err = cwl.Add_to_collection(&collection, object_array) + if err != nil { + return + } + + //job_input, ok := job.CWL_job_input.([]cwl_types.CWLType) + + job_input, xerr := cwl.NewJob_document(job.CWL_job_input) + //job_input, ok := job.CWL_job_input.(cwl.Job_document) + if xerr != nil { + err = fmt.Errorf("(job.Init) cannot create CWL_job_input: %s", xerr.Error) + return + } + + job_input_map := job_input.GetMap() + + collection.Job_input_map = &job_input_map + + cwl_workflow, ok := collection.Workflows["#main"] + if !ok { + err = fmt.Errorf("(job.Init) Workflow \"main\" not found") + return + } + + job.CWL_collection = &collection + job.CWL_workflow = cwl_workflow + + } + + // read from base64 string + // if job.CWL_collection == nil && job.CWL_job_input_b64 != "" { + // new_collection := cwl.NewCWL_collection() + // + // //1) parse job + // job_input_byte_array, xerr := b64.StdEncoding.DecodeString(job.CWL_job_input_b64) + // if xerr != nil { + // err = fmt.Errorf("(job.Init) error decoding CWL_job_input_b64: %s", xerr.Error()) + // return + // } + // + // job_input, xerr := cwl.ParseJob(&job_input_byte_array) + // if xerr != nil { + // err = fmt.Errorf("(job.Init) error in reading job yaml/json file: " + xerr.Error()) + // return + // } + // new_collection.Job_input = job_input + // + // // 2) parse workflow document + // + // workflow_byte_array, xerr := b64.StdEncoding.DecodeString(job.CWL_workflow_b64) + // + // err = cwl.Parse_cwl_document(&new_collection, string(workflow_byte_array[:])) + // if err != nil { + // err = fmt.Errorf("(job.Init) Parse_cwl_document error: " + err.Error()) + // + // return + // } + // + // cwl_workflow, ok := new_collection.Workflows["#main"] + // if !ok { + // + // err = fmt.Errorf("(job.Init) Workflow main not found") + // return + // } + // + // job.CWL_collection = &new_collection + // _ = cwl_workflow + // } + return } @@ -287,26 +403,69 @@ func (job *Job) SaveToDisk() (err error) { return } +func Deserialize_b64(encoding string, target interface{}) (err error) { + byte_array, err := b64.StdEncoding.DecodeString(encoding) + if err != nil { + err = fmt.Errorf("(Deserialize_b64) DecodeString error: %s", err.Error()) + return + } + + err = json.Unmarshal(byte_array, target) + if err != nil { + return + } + + return +} + +// takes a yaml string as input, may change that later +// func (job *Job) Set_CWL_workflow_b64(yaml_str string) { +// +// job.CWL_workflow_b64 = b64.StdEncoding.EncodeToString([]byte(yaml_str)) +// +// return +// } + +// func (job *Job) Set_CWL_job_input_b64(yaml_str string) { +// +// job.CWL_job_input_b64 = b64.StdEncoding.EncodeToString([]byte(yaml_str)) +// +// return +// } + +// func Serialize_b64(input interface{}) (serialized string, err error) { +// json_byte, xerr := json.Marshal(input) +// if xerr != nil { +// err = fmt.Errorf("(job.Save) json.Marshal(input) failed: %s", xerr.Error()) +// return +// } +// serialized = b64.StdEncoding.EncodeToString(json_byte) +// return +// } + func (job *Job) Save() (err error) { if job.Id == "" { - err = fmt.Errorf("job id empty") + err = fmt.Errorf("(job.Save()) job id empty") return } - logger.Debug(1, "Save() saving job: %s", job.Id) + logger.Debug(1, "(job.Save()) saving job: %s", job.Id) job.UpdateTime = time.Now() err = job.SaveToDisk() if err != nil { + err = fmt.Errorf("(job.Save()) SaveToDisk failed: %s", err.Error()) return } - logger.Debug(1, "Save() dbUpsert next: %s", job.Id) + logger.Debug(1, "(job.Save()) dbUpsert next: %s", job.Id) + //spew.Dump(job) + err = dbUpsert(job) if err != nil { - err = fmt.Errorf("error in dbUpsert in job.Save(), (job_id=%s) error=%v", job.Id, err) + err = fmt.Errorf("(job.Save()) dbUpsert failed (job_id=%s) error=%s", job.Id, err.Error()) return } - logger.Debug(1, "Save() job saved: %s", job.Id) + logger.Debug(1, "(job.Save()) job saved: %s", job.Id) return } @@ -418,17 +577,34 @@ func (job *Job) NumTask() int { //---Field update functions -func (job *Job) SetState(newState string) (err error) { +func (job *Job) SetState(newState string, oldstates []string) (err error) { err = job.LockNamed("SetState") if err != nil { return } defer job.Unlock() - if job.State == newState { + job_state := job.State + + if job_state == newState { return } + if len(oldstates) > 0 { + matched := false + for _, oldstate := range oldstates { + if oldstate == job_state { + matched = true + break + } + } + if !matched { + oldstates_str := strings.Join(oldstates, ",") + err = fmt.Errorf("(UpdateJobState) old state %s does not match one of the required ones (required: %s)", job_state, oldstates_str) + return + } + } + err = dbUpdateJobFieldString(job.Id, "state", newState) if err != nil { return @@ -436,13 +612,22 @@ func (job *Job) SetState(newState string) (err error) { job.State = newState // set time if completed - if newState == JOB_STAT_COMPLETED { + switch newState { + case JOB_STAT_COMPLETED: newTime := time.Now() err = dbUpdateJobFieldTime(job.Id, "info.completedtime", newTime) if err != nil { return } job.Info.CompletedTime = newTime + case JOB_STAT_INPROGRESS: + time_now := time.Now() + jobid := job.Id + err = job.Info.SetStartedTime(jobid, time_now) + if err != nil { + return + } + } // unset error if not suspended @@ -462,6 +647,7 @@ func (job *Job) SetError(newError *JobError) (err error) { return } defer job.Unlock() + spew.Dump(newError) update_value := bson.M{"error": newError} err = dbUpdateJobFields(job.Id, update_value) diff --git a/lib/core/perf.go b/lib/core/perf.go index d59e4c3d..0148b80e 100644 --- a/lib/core/perf.go +++ b/lib/core/perf.go @@ -61,7 +61,7 @@ func NewTaskPerf(id string) *TaskPerf { } } -func NewWorkPerf(id string) *WorkPerf { +func NewWorkPerf() *WorkPerf { return &WorkPerf{ Queued: time.Now().Unix(), } diff --git a/lib/core/proxymgr.go b/lib/core/proxymgr.go index 071af77a..f5c2678e 100644 --- a/lib/core/proxymgr.go +++ b/lib/core/proxymgr.go @@ -58,8 +58,8 @@ func (qm *ProxyMgr) ClientHandle() { coReq.response <- ack case notice := <-qm.feedback: logger.Debug(2, fmt.Sprintf("proxymgr: workunit feedback received, workid=%s, status=%s, clientid=%s\n", notice.WorkId, notice.Status, notice.ClientId)) - if err := qm.handleWorkStatusChange(notice); err != nil { - logger.Error("handleWorkStatusChange(): " + err.Error()) + if err := qm.handleNoticeWorkDelivered(notice); err != nil { + logger.Error("handleNoticeWorkDelivered(): " + err.Error()) } } } @@ -99,7 +99,7 @@ func (qm *ProxyMgr) GetTextStatus() string { // workunit methods //handle feedback from a client about the execution of a workunit -func (qm *ProxyMgr) handleWorkStatusChange(notice Notice) (err error) { +func (qm *ProxyMgr) handleNoticeWorkDelivered(notice Notice) (err error) { //relay the notice to the server perf := new(WorkPerf) workid := notice.WorkId @@ -110,19 +110,12 @@ func (qm *ProxyMgr) handleWorkStatusChange(notice Notice) (err error) { } if ok { //delete(client.Current_work, workid) - client.LockNamed("ProxyMgr/handleWorkStatusChange A2") - err = client.Current_work_delete(workid, false) + client.LockNamed("ProxyMgr/handleNoticeWorkDelivered A2") + err = client.Assigned_work.Delete(notice.WorkId, false) if err != nil { return } - cw_length, xerr := client.Current_work_length(false) - if xerr != nil { - err = xerr - return - } - if cw_length == 0 { - client.Status = CLIENT_STAT_ACTIVE_IDLE - } + qm.AddClient(client, true) client.Unlock() } @@ -131,7 +124,10 @@ func (qm *ProxyMgr) handleWorkStatusChange(notice Notice) (err error) { return } if ok { - work.SetState(notice.Status) + err = work.SetState(notice.Status, "") + if err != nil { + return + } if err = proxy_relay_workunit(work, perf); err != nil { return } @@ -153,7 +149,7 @@ func (qm *ProxyMgr) handleWorkStatusChange(notice Notice) (err error) { return } if ok { - client.LockNamed("ProxyMgr/handleWorkStatusChange B") + client.LockNamed("ProxyMgr/handleNoticeWorkDelivered B") err = client.Append_Skip_work(workid, false) if err != nil { return @@ -164,7 +160,7 @@ func (qm *ProxyMgr) handleWorkStatusChange(notice Notice) (err error) { } client.Last_failed += 1 //last consecutive failures if client.Last_failed == conf.MAX_CLIENT_FAILURE { - client.Status = CLIENT_STAT_SUSPEND + client.Suspend("MAX_CLIENT_FAILURE reached", false) } qm.AddClient(client, false) client.Unlock() @@ -175,7 +171,7 @@ func (qm *ProxyMgr) handleWorkStatusChange(notice Notice) (err error) { return } -func (qm *ProxyMgr) FetchDataToken(workid string, clientid string) (token string, err error) { +func (qm *ProxyMgr) FetchDataToken(workunit *Workunit, clientid string) (token string, err error) { return } @@ -206,20 +202,22 @@ func (qm *ProxyMgr) RegisterNewClient(files FormFiles, cg *ClientGroup) (client return nil, errors.New("Clientgroup name in token does not match that in the client configuration.") } qm.AddClient(client, true) - cw_length, err := client.Current_work_length(false) + cw_length, err := client.Current_work.Length(false) if err != nil { return } if cw_length > 0 { //re-registered client // move already checked-out workunit from waiting queue (workMap) to checked-out list (coWorkMap) - for workid, _ := range client.Current_work { + work_list, _ := client.Current_work.Get_list(false) + + for _, workid := range work_list { has_work, err := qm.workQueue.Has(workid) if err != nil { continue } if has_work { - qm.workQueue.StatusChange(workid, nil, WORK_STAT_CHECKOUT) + qm.workQueue.StatusChange(workid, nil, WORK_STAT_CHECKOUT, "") } } @@ -259,7 +257,7 @@ func (qm *ProxyMgr) ClientChecker() { } else { //now client must be gone as tag set to false 30 seconds ago and no heartbeat received thereafter - logger.Event(event.CLIENT_UNREGISTER, "clientid="+client.Id+";name="+client.Name) + logger.Event(event.CLIENT_UNREGISTER, "clientid="+client.Id) //qm.RemoveClient(client.Id) delete_clients = append(delete_clients, client.Id) diff --git a/lib/core/resmgr.go b/lib/core/resmgr.go index 31aaaf44..a209a2e8 100644 --- a/lib/core/resmgr.go +++ b/lib/core/resmgr.go @@ -6,37 +6,37 @@ import ( type ClientMgr interface { RegisterNewClient(FormFiles, *ClientGroup) (*Client, error) - ClientHeartBeat(string, *ClientGroup) (HBmsg, error) + ClientHeartBeat(string, *ClientGroup, WorkerState) (HeartbeatInstructions, error) GetClient(string, bool) (*Client, bool, error) GetClientByUser(string, *user.User) (*Client, error) //GetAllClients() []*Client GetClientMap() *ClientMap GetAllClientsByUser(*user.User) ([]*Client, error) - DeleteClient(*Client) error - DeleteClientById(string) error - DeleteClientByUser(string, *user.User) error - SuspendClient(string, *Client, bool) error - SuspendClientByUser(string, *user.User) error + //DeleteClient(*Client) error + //DeleteClientById(string) error + //DeleteClientByUser(string, *user.User) error + SuspendClient(string, *Client, string, bool) error + SuspendClientByUser(string, *user.User, string) error ResumeClient(string) error ResumeClientByUser(string, *user.User) error ResumeSuspendedClients() (int, error) ResumeSuspendedClientsByUser(*user.User) int - SuspendAllClients() (int, error) - SuspendAllClientsByUser(*user.User) int + SuspendAllClients(string) (int, error) + SuspendAllClientsByUser(*user.User, string) (int, error) ClientChecker() UpdateSubClients(string, int) error UpdateSubClientsByUser(string, int, *user.User) } type WorkMgr interface { - GetWorkById(string) (*Workunit, error) + GetWorkById(Workunit_Unique_Identifier) (*Workunit, error) ShowWorkunits(string) ([]*Workunit, error) ShowWorkunitsByUser(string, *user.User) []*Workunit CheckoutWorkunits(string, string, *Client, int64, int) ([]*Workunit, error) NotifyWorkStatus(Notice) EnqueueWorkunit(*Workunit) error - FetchDataToken(string, string) (string, error) - FetchPrivateEnv(string, string) (map[string]string, error) + FetchDataToken(Workunit_Unique_Identifier, string) (string, error) + FetchPrivateEnv(Workunit_Unique_Identifier, string) (map[string]string, error) } type JobMgr interface { @@ -53,9 +53,9 @@ type JobMgr interface { DeleteZombieJobsByUser(*user.User, bool) int RecoverJob(string) error RecoverJobs() error - FinalizeWorkPerf(string, string) error - SaveStdLog(string, string, string) error - GetReportMsg(string, string) (string, error) + FinalizeWorkPerf(Workunit_Unique_Identifier, string) error + SaveStdLog(Workunit_Unique_Identifier, string, string) error + GetReportMsg(Workunit_Unique_Identifier, string) (string, error) RecomputeJob(string, string) error UpdateQueueToken(*Job) error } diff --git a/lib/core/servermgr.go b/lib/core/servermgr.go index bc097bd5..e370d38d 100644 --- a/lib/core/servermgr.go +++ b/lib/core/servermgr.go @@ -10,11 +10,13 @@ import ( "github.com/MG-RAST/AWE/lib/logger/event" "github.com/MG-RAST/AWE/lib/shock" "github.com/MG-RAST/AWE/lib/user" + "github.com/davecgh/go-spew/spew" "gopkg.in/mgo.v2/bson" "io/ioutil" "os" + "path" "path/filepath" - //"strconv" + "strconv" "strings" "sync" "time" @@ -45,10 +47,10 @@ func NewServerMgr() *ServerMgr { workQueue: NewWorkQueue(), suspendQueue: false, - coReq: make(chan CoReq, conf.COREQ_LENGTH), // number of clients that wait in queue to get a workunit. If queue is full, other client will be rejected and have to come back later again - feedback: make(chan Notice), - coSem: make(chan int, 1), //non-blocking buffered channel - + coReq: make(chan CoReq, conf.COREQ_LENGTH), // number of clients that wait in queue to get a workunit. If queue is full, other client will be rejected and have to come back later again + feedback: make(chan Notice), + coSem: make(chan int, 1), //non-blocking buffered channel + }, lastUpdate: time.Now().Add(time.Second * -30), TaskMap: *NewTaskMap(), @@ -65,7 +67,7 @@ func (qm *ServerMgr) Unlock() {} func (qm *ServerMgr) RLock() {} func (qm *ServerMgr) RUnlock() {} -func (qm *ServerMgr) TaskHandle() { +func (qm *ServerMgr) TaskHandle() { // TODO DEPRECATED logger.Info("TaskHandle is starting") for { task := <-qm.taskIn @@ -73,11 +75,14 @@ func (qm *ServerMgr) TaskHandle() { task_id, err := task.GetId() if err != nil { logger.Error("(TaskHandle) %s", err.Error()) - task_id = "unknown" + continue } - logger.Debug(2, "ServerMgr/TaskHandle received task from channel taskIn, id=%s", task_id) - qm.addTask(task) + logger.Debug(2, "(ServerMgr/TaskHandle) received task from channel taskIn, id=%s", task_id) + err = qm.addTask(task, nil) + if err != nil { + logger.Error("(ServerMgr/TaskHandle) qm.addTask failed: %s", err.Error()) + } } } @@ -161,7 +166,7 @@ func (qm *ServerMgr) ClientHandle() { select { case coReq.response <- ack: - logger.Debug(3, "(ServerMgr ClientHandle %s) send workunit to client via response channel", coReq.fromclient) + logger.Debug(3, "(ServerMgr ClientHandle %s) send response (maybe workunit) to client via response channel", coReq.fromclient) case <-timer.C: elapsed_time := time.Since(start_time) logger.Error("(ServerMgr ClientHandle %s) timed out after %s ", coReq.fromclient, elapsed_time) @@ -169,7 +174,7 @@ func (qm *ServerMgr) ClientHandle() { } logger.Debug(3, "(ServerMgr ClientHandle %s) done", coReq.fromclient) - if count%10 == 0 { // use modulo to reduce number of log messages + if count%100 == 0 { // use modulo to reduce number of log messages request_time_elapsed := time.Since(request_start_time) logger.Info("(ServerMgr ClientHandle) Responding to work request took %s", request_time_elapsed) @@ -182,7 +187,7 @@ func (qm *ServerMgr) NoticeHandle() { for { notice := <-qm.feedback logger.Debug(3, "(ServerMgr NoticeHandle) got notice: workid=%s, status=%s, clientid=%s", notice.WorkId, notice.Status, notice.ClientId) - if err := qm.handleWorkStatusChange(notice); err != nil { + if err := qm.handleNoticeWorkDelivered(notice); err != nil { logger.Error("(NoticeHandle): " + err.Error()) } } @@ -346,26 +351,56 @@ func (qm *ServerMgr) updateQueue() (err error) { } logger.Debug(3, "(updateQueue) range tasks (%d)", len(tasks)) for _, task := range tasks { - task_id, err := task.GetId() - if err != nil { + task_id, yerr := task.GetId() + if yerr != nil { + err = yerr return err } + task_state, xerr := task.GetState() + if xerr != nil { + continue + } + + if !(task_state == TASK_STAT_INIT || task_state == TASK_STAT_PENDING) { // == TASK_STAT_INPROGRESS || task_state == TASK_STAT_COMPLETED + logger.Debug(3, "(updateQueue) skipping task %s , it has state %s", task_id, task_state) + continue + } + logger.Debug(3, "(updateQueue) task: %s", task_id) task_ready, err := qm.isTaskReady(task) if err != nil { - logger.Error("(updateQueue) isTaskReady=%s error: %s", task_id, err.Error()) + logger.Error("(updateQueue) %s isTaskReady returns error: %s", task_id, err.Error()) continue } if task_ready { logger.Debug(3, "(updateQueue) task ready: %s", task_id) - err = qm.taskEnQueue(task) + + job_id, err := task.GetJobId() + if err != nil { + continue + } + + job, err := LoadJob(job_id) + if err != nil { + continue + } + + err = qm.taskEnQueue(task, job) if err != nil { - _ = task.SetState(TASK_STAT_SUSPEND) - job_id, _ := GetJobIdByTaskId(task_id) + _ = task.SetState(TASK_STAT_SUSPEND, true) + var job_id string + job_id, xerr := task.GetJobId() + if xerr != nil { + err = xerr + return err + } + + task_string := task.String() + jerror := &JobError{ - TaskFailed: task_id, + TaskFailed: task_string, ServerNotes: "failed enqueuing task, err=" + err.Error(), Status: JOB_STAT_SUSPEND, } @@ -381,17 +416,11 @@ func (qm *ServerMgr) updateQueue() (err error) { } logger.Debug(3, "(updateQueue) range qm.workQueue.Clean()") - for _, id := range qm.workQueue.Clean() { - job_id, err := GetJobIdByWorkId(id) - if err != nil { - logger.Error("(updateQueue) workunit %s is nil, cannot get job id", id) - continue - } - task_id, err := GetTaskIdByWorkId(id) - if err != nil { - logger.Error("(updateQueue) workunit %s is nil, cannot get task id", id) - continue - } + for _, workunit := range qm.workQueue.Clean() { + id := workunit.Id + job_id := workunit.JobId + task_id := workunit.TaskId + jerror := &JobError{ WorkFailed: id, TaskFailed: task_id, @@ -409,45 +438,53 @@ func (qm *ServerMgr) updateQueue() (err error) { return } -func RemoveWorkFromClient(client *Client, clientid string, workid string) (err error) { - err = client.Current_work_delete(workid, true) +func RemoveWorkFromClient(client *Client, workid Workunit_Unique_Identifier) (err error) { + err = client.Assigned_work.Delete(workid, true) if err != nil { return } - work_length, err := client.Current_work_length(true) + work_length, err := client.Assigned_work.Length(true) if err != nil { return } if work_length > 0 { - logger.Error("(RemoveWorkFromClient) Client %s still has %d workunits, after delivering one workunit", clientid, work_length) - current_work_ids, err := client.Get_current_work(true) + clientid, _ := client.Get_Id(true) + + logger.Error("(RemoveWorkFromClient) Client %s still has %d workunits assigned, after delivering one workunit", clientid, work_length) + + assigned_work_ids, err := client.Assigned_work.Get_list(true) if err != nil { return err } - for _, work_id := range current_work_ids { - _ = client.Current_work_delete(work_id, true) + for _, work_id := range assigned_work_ids { + _ = client.Assigned_work.Delete(work_id, true) } - work_length, err = client.Current_work_length(true) + work_length, err = client.Assigned_work.Length(true) if err != nil { return err } if work_length > 0 { - logger.Error("(RemoveWorkFromClient) Client still has work, even after everything should have been deleted.") + logger.Error("(RemoveWorkFromClient) Client still has work assigned, even after everything should have been deleted.") return fmt.Errorf("(RemoveWorkFromClient) Client %s still has %d workunits", clientid, work_length) } } return } -func (qm *ServerMgr) handleWorkStatDone(client *Client, clientid string, task *Task, task_id string, workid string, computetime int) (err error) { +func (qm *ServerMgr) handleWorkStatDone(client *Client, clientid string, task *Task, workid Workunit_Unique_Identifier, computetime int) (err error) { //log event about work done (WD) - logger.Event(event.WORK_DONE, "workid="+workid+";clientid="+clientid) + + workid_string := workid.String() + + logger.Event(event.WORK_DONE, "workid="+workid_string+";clientid="+clientid) //update client status + task_id := task.Id + defer func() { //done, remove from the workQueue err = qm.workQueue.Delete(workid) @@ -486,7 +523,7 @@ func (qm *ServerMgr) handleWorkStatDone(client *Client, clientid string, task *T if xerr != nil { err = xerr logger.Error("task %s, err: %s", task_id, err.Error()) - yerr := task.SetState(TASK_STAT_SUSPEND) + yerr := task.SetState(TASK_STAT_SUSPEND, true) if yerr != nil { err = yerr return @@ -508,7 +545,7 @@ func (qm *ServerMgr) handleWorkStatDone(client *Client, clientid string, task *T } } - err = task.SetState(TASK_STAT_COMPLETED) + err = task.SetState(TASK_STAT_COMPLETED, true) if err != nil { return } @@ -541,24 +578,29 @@ func (qm *ServerMgr) handleWorkStatDone(client *Client, clientid string, task *T } //handle feedback from a client about the execution of a workunit -func (qm *ServerMgr) handleWorkStatusChange(notice Notice) (err error) { - workid := notice.WorkId +func (qm *ServerMgr) handleNoticeWorkDelivered(notice Notice) (err error) { + + id := notice.WorkId + task_id := id.GetTask() + + job_id := id.JobId + workid := id.String() status := notice.Status clientid := notice.ClientId computetime := notice.ComputeTime notes := notice.Notes - logger.Debug(3, "(handleWorkStatusChange) workid: %s status: %s client: %s", workid, status, clientid) + logger.Debug(3, "(handleNoticeWorkDelivered) workid: %s status: %s client: %s", workid, status, clientid) // we should not get here, but if we do than end if status == WORK_STAT_DISCARDED { - logger.Error("(handleWorkStatusChange) [warning] skip status change: workid=%s status=%s", workid, status) + logger.Error("(handleNoticeWorkDelivered) [warning] skip status change: workid=%s status=%s", workid, status) return } - parts := strings.Split(workid, "_") - task_id := fmt.Sprintf("%s_%s", parts[0], parts[1]) - job_id := parts[0] + //parts := strings.Split(workid, "_") + //task_id := fmt.Sprintf("%s_%s", parts[0], parts[1]) + //job_id := parts[0] // *** Get Client client, ok, err := qm.GetClient(clientid, true) @@ -566,9 +608,9 @@ func (qm *ServerMgr) handleWorkStatusChange(notice Notice) (err error) { return } if !ok { - return fmt.Errorf("(handleWorkStatusChange) client not found") + return fmt.Errorf("(handleNoticeWorkDelivered) client not found") } - defer RemoveWorkFromClient(client, clientid, workid) + defer RemoveWorkFromClient(client, id) // *** Get Task task, tok, err := qm.TaskMap.Get(task_id, true) @@ -578,28 +620,34 @@ func (qm *ServerMgr) handleWorkStatusChange(notice Notice) (err error) { if !tok { //task not existed, possible when job is deleted before the workunit done logger.Error("Task %s for workunit %s not found", task_id, workid) - qm.workQueue.Delete(workid) - return fmt.Errorf("(handleWorkStatusChange) task %s for workunit %s not found", task_id, workid) + qm.workQueue.Delete(id) + return fmt.Errorf("(handleNoticeWorkDelivered) task %s for workunit %s not found", task_id, workid) } // *** Get workunit - work, wok, err := qm.workQueue.Get(workid) + work, wok, err := qm.workQueue.Get(id) if err != nil { return err } if !wok { - return fmt.Errorf("(handleWorkStatusChange) workunit %s not found in workQueue", workid) + return fmt.Errorf("(handleNoticeWorkDelivered) workunit %s not found in workQueue", workid) } if work.State != WORK_STAT_CHECKOUT && work.State != WORK_STAT_RESERVED { - return fmt.Errorf("(handleWorkStatusChange) workunit %s did not have state WORK_STAT_CHECKOUT or WORK_STAT_RESERVED (state is %s)", workid, work.State) + return fmt.Errorf("(handleNoticeWorkDelivered) workunit %s did not have state WORK_STAT_CHECKOUT or WORK_STAT_RESERVED (state is %s)", workid, work.State) + } + + reason := "" + + if status == WORK_STAT_SUSPEND { + reason = "workunit suspended by worker" // TODO add more info from worker } // *** update state of workunit - if err = qm.workQueue.StatusChange("", work, status); err != nil { + if err = qm.workQueue.StatusChange(Workunit_Unique_Identifier{}, work, status, reason); err != nil { return err } - if err = task.LockNamed("handleWorkStatusChange/noretry"); err != nil { + if err = task.LockNamed("handleNoticeWorkDelivered/noretry"); err != nil { return err } noretry := task.Info.NoRetry @@ -621,66 +669,66 @@ func (qm *ServerMgr) handleWorkStatusChange(notice Notice) (err error) { // A work unit for this task failed before this one arrived. // User set Skip=2 so the task was just skipped. Any subsiquent // workunits are just deleted... - qm.workQueue.Delete(workid) - return fmt.Errorf("(handleWorkStatusChange) workunit %s failed due to skip", workid) + qm.workQueue.Delete(id) + return fmt.Errorf("(handleNoticeWorkDelivered) workunit %s failed due to skip", workid) } - logger.Debug(3, "(handleWorkStatusChange) handling status %s", status) + logger.Debug(3, "(handleNoticeWorkDelivered) handling status %s", status) if status == WORK_STAT_DONE { - if err = qm.handleWorkStatDone(client, clientid, task, task_id, workid, computetime); err != nil { + if err = qm.handleWorkStatDone(client, clientid, task, id, computetime); err != nil { return err } } else if status == WORK_STAT_FAILED_PERMANENT { // (special case !) failed and cannot be recovered logger.Event(event.WORK_FAILED, "workid="+workid+";clientid="+clientid) - logger.Debug(3, "(handleWorkStatusChange) work failed (status=%s) workid=%s clientid=%s", status, workid, clientid) + logger.Debug(3, "(handleNoticeWorkDelivered) work failed (status=%s) workid=%s clientid=%s", status, workid, clientid) work.Failed += 1 - qm.workQueue.StatusChange(workid, work, WORK_STAT_FAILED_PERMANENT) + qm.workQueue.StatusChange(Workunit_Unique_Identifier{}, work, WORK_STAT_FAILED_PERMANENT, "") - if err = task.SetState(TASK_STAT_FAILED_PERMANENT); err != nil { + if err = task.SetState(TASK_STAT_FAILED_PERMANENT, true); err != nil { return err } jerror := &JobError{ ClientFailed: clientid, WorkFailed: workid, - TaskFailed: task_id, + TaskFailed: id.TaskId, ServerNotes: "exit code 42 encountered", WorkNotes: notes, AppError: notice.Stderr, Status: JOB_STAT_FAILED_PERMANENT, } if err = qm.SuspendJob(job_id, jerror); err != nil { - logger.Error("(handleWorkStatusChange:SuspendJob) job_id=%s; err=%s", job_id, err.Error()) + logger.Error("(handleNoticeWorkDelivered:SuspendJob) job_id=%s; err=%s", job_id, err.Error()) } } else if status == WORK_STAT_ERROR { //workunit failed, requeue or put it to suspend list logger.Event(event.WORK_FAIL, "workid="+workid+";clientid="+clientid) - logger.Debug(3, "(handleWorkStatusChange) work failed (status=%s) workid=%s clientid=%s", status, workid, clientid) + logger.Debug(3, "(handleNoticeWorkDelivered) work failed (status=%s) workid=%s clientid=%s", status, workid, clientid) work.Failed += 1 if work.Failed < MAX_FAILURE { - qm.workQueue.StatusChange(workid, work, WORK_STAT_QUEUED) + qm.workQueue.StatusChange(Workunit_Unique_Identifier{}, work, WORK_STAT_QUEUED, "") logger.Event(event.WORK_REQUEUE, "workid="+workid) } else { //failure time exceeds limit, suspend workunit, task, job - qm.workQueue.StatusChange(workid, work, WORK_STAT_SUSPEND) + qm.workQueue.StatusChange(Workunit_Unique_Identifier{}, work, WORK_STAT_SUSPEND, "work.Failed >= MAX_FAILURE") logger.Event(event.WORK_SUSPEND, "workid="+workid) - if err = task.SetState(TASK_STAT_SUSPEND); err != nil { + if err = task.SetState(TASK_STAT_SUSPEND, true); err != nil { return err } jerror := &JobError{ ClientFailed: clientid, WorkFailed: workid, - TaskFailed: task_id, + TaskFailed: id.TaskId, ServerNotes: fmt.Sprintf("workunit failed %d time(s)", MAX_FAILURE), WorkNotes: notes, AppError: notice.Stderr, Status: JOB_STAT_SUSPEND, } if err = qm.SuspendJob(job_id, jerror); err != nil { - logger.Error("(handleWorkStatusChange:SuspendJob) job_id=%s; err=%s", job_id, err.Error()) + logger.Error("(handleNoticeWorkDelivered:SuspendJob) job_id=%s; err=%s", job_id, err.Error()) } } @@ -692,7 +740,7 @@ func (qm *ServerMgr) handleWorkStatusChange(notice Notice) (err error) { if !ok { return fmt.Errorf(e.ClientNotFound) } - if err = client.Append_Skip_work(workid, true); err != nil { + if err = client.Append_Skip_work(id, true); err != nil { return err } if err = client.Increment_total_failed(true); err != nil { @@ -703,7 +751,7 @@ func (qm *ServerMgr) handleWorkStatusChange(notice Notice) (err error) { return err } if last_failed >= conf.MAX_CLIENT_FAILURE { - qm.SuspendClient(clientid, client, true) + qm.SuspendClient(clientid, client, "MAX_CLIENT_FAILURE on client reached", true) } } else { return fmt.Errorf("No handler for workunit status '%s' implemented (allowd: %s, %s, %s)", status, WORK_STAT_DONE, WORK_STAT_FAILED_PERMANENT, WORK_STAT_ERROR) @@ -803,19 +851,17 @@ func (qm *ServerMgr) GetJsonStatus() (status map[string]map[string]int, err erro continue } - busy, _ := client.IsBusy(false) - status := client.Status - - client.RUnlockNamed(rlock) - - if status == CLIENT_STAT_SUSPEND { + if client.Suspended { suspend_client += 1 - } else if busy { + } + if client.Busy { busy_client += 1 } else { idle_client += 1 } + client.RUnlockNamed(rlock) + } elapsed = time.Since(start) logger.Debug(3, "time GetJsonStatus/client_list: %s", elapsed) @@ -883,7 +929,8 @@ func (qm *ServerMgr) GetTextStatus() string { //---end of mgr methods //--workunit methds (servermgr implementation) -func (qm *ServerMgr) FetchDataToken(workid string, clientid string) (token string, err error) { +func (qm *ServerMgr) FetchDataToken(work_id Workunit_Unique_Identifier, clientid string) (token string, err error) { + //precheck if the client is registered client, ok, err := qm.GetClient(clientid, true) if err != nil { @@ -892,72 +939,75 @@ func (qm *ServerMgr) FetchDataToken(workid string, clientid string) (token strin if !ok { return "", errors.New(e.ClientNotFound) } - client_status, err := client.Get_Status(true) - if err != nil { - return - } - if client_status == CLIENT_STAT_SUSPEND { - return "", errors.New(e.ClientSuspended) - } - jobid, err := GetJobIdByWorkId(workid) - if err != nil { - return "", err - } - job, err := GetJob(jobid) - if err != nil { - return "", err - } - token = job.GetDataToken() - if token == "" { - return token, errors.New("no data token set for workunit " + workid) - } - return token, nil -} -func (qm *ServerMgr) FetchPrivateEnvs(workid string, clientid string) (envs map[string]string, err error) { - //precheck if the client is registered - client, ok, err := qm.GetClient(clientid, true) + is_suspended, err := client.Get_Suspended(true) if err != nil { return } - if !ok { - return nil, errors.New(e.ClientNotFound) - } - client_status, err := client.Get_Status(true) - if err != nil { + + if is_suspended { + err = errors.New(e.ClientSuspended) return } - if client_status == CLIENT_STAT_SUSPEND { - return nil, errors.New(e.ClientSuspended) - } - jobid, err := GetJobIdByWorkId(workid) - if err != nil { - return nil, err - } + + jobid := work_id.JobId job, err := GetJob(jobid) if err != nil { - return nil, err - } - - taskid, _ := GetTaskIdByWorkId(workid) - - idx := -1 - for i, t := range job.Tasks { - if t.Id == taskid { - idx = i - break - } + return } - envs = job.Tasks[idx].Cmd.Environ.Private - if envs == nil { - return nil, errors.New("no private envs for workunit " + workid) + token = job.GetDataToken() + if token == "" { + err = errors.New("no data token set for workunit " + work_id.String()) + return } - return envs, nil + return } -func (qm *ServerMgr) SaveStdLog(workid string, logname string, tmppath string) (err error) { - savedpath, err := getStdLogPathByWorkId(workid, logname) +// func (qm *ServerMgr) FetchPrivateEnvs_deprecated(workid string, clientid string) (envs map[string]string, err error) { +// //precheck if the client is registered +// client, ok, err := qm.GetClient(clientid, true) +// if err != nil { +// return +// } +// if !ok { +// return nil, errors.New(e.ClientNotFound) +// } +// client_status, err := client.Get_Status(true) +// if err != nil { +// return +// } +// if client_status == CLIENT_STAT_SUSPEND { +// return nil, errors.New(e.ClientSuspended) +// } +// jobid, err := GetJobIdByWorkId(workid) +// if err != nil { +// return nil, err +// } +// +// job, err := GetJob(jobid) +// if err != nil { +// return nil, err +// } +// +// taskid, _ := GetTaskIdByWorkId(workid) +// +// idx := -1 +// for i, t := range job.Tasks { +// if t.Id == taskid { +// idx = i +// break +// } +// } +// envs = job.Tasks[idx].Cmd.Environ.Private +// if envs == nil { +// return nil, errors.New("no private envs for workunit " + workid) +// } +// return envs, nil +// } + +func (qm *ServerMgr) SaveStdLog(id Workunit_Unique_Identifier, logname string, tmppath string) (err error) { + savedpath, err := getStdLogPathByWorkId(id, logname) if err != nil { return err } @@ -965,8 +1015,8 @@ func (qm *ServerMgr) SaveStdLog(workid string, logname string, tmppath string) ( return } -func (qm *ServerMgr) GetReportMsg(workid string, logname string) (report string, err error) { - logpath, err := getStdLogPathByWorkId(workid, logname) +func (qm *ServerMgr) GetReportMsg(id Workunit_Unique_Identifier, logname string) (report string, err error) { + logpath, err := getStdLogPathByWorkId(id, logname) if err != nil { return "", err } @@ -981,10 +1031,14 @@ func (qm *ServerMgr) GetReportMsg(workid string, logname string) (report string, return string(content), err } -func deleteStdLogByTask(taskid string, logname string) (err error) { - jobid, err := GetJobIdByTaskId(taskid) +func deleteStdLogByTask(task *Task, logname string) (err error) { + jobid, err := task.GetJobId() if err != nil { - return err + return + } + taskid, err := task.GetId() + if err != nil { + return } var logdir string logdir, err = getPathByJobId(jobid) @@ -1004,82 +1058,152 @@ func deleteStdLogByTask(taskid string, logname string) (err error) { return } -func getStdLogPathByWorkId(workid string, logname string) (savedpath string, err error) { - jobid, err := GetJobIdByWorkId(workid) - if err != nil { - return "", err - } +func getStdLogPathByWorkId(id Workunit_Unique_Identifier, logname string) (savedpath string, err error) { + jobid := id.JobId + var logdir string logdir, err = getPathByJobId(jobid) if err != nil { return } + workid := id.String() savedpath = fmt.Sprintf("%s/%s.%s", logdir, workid, logname) return } //---task methods---- - +// this is invoked after a job is uploaded and saved in mongo func (qm *ServerMgr) EnqueueTasksByJobId(jobid string) (err error) { - + logger.Debug(3, "(EnqueueTasksByJobId) starting") job, err := GetJob(jobid) if err != nil { + err = fmt.Errorf("(EnqueueTasksByJobId) GetJob failed: %s", err.Error()) return } tasks, err := job.GetTasks() if err != nil { + err = fmt.Errorf("(EnqueueTasksByJobId) job.GetTasks failed: %s", err.Error()) + return + } + + task_len := len(tasks) + + logger.Debug(3, "(EnqueueTasksByJobId) got %d tasks", task_len) + + err = job.SetState(JOB_STAT_QUEUING, nil) + if err != nil { + err = fmt.Errorf("(qmgr.taskEnQueue) UpdateJobState: %s", err.Error()) return } for _, task := range tasks { - qm.taskIn <- task + //qm.taskIn <- task + + task_state, xerr := task.GetState() + if xerr != nil { + err = xerr + return + } + + if task_state == TASK_STAT_INPROGRESS || task_state == TASK_STAT_QUEUED { + task.SetState(TASK_STAT_READY, true) + } else if task_state == TASK_STAT_SUSPEND { + task.SetState(TASK_STAT_PENDING, true) + } + + qm.addTask(task, job) + } + + err = job.SetState(JOB_STAT_QUEUED, []string{JOB_STAT_INIT, JOB_STAT_SUSPEND, JOB_STAT_QUEUING}) + if err != nil { + return } + qm.CreateJobPerf(jobid) return } //---end of task methods -func (qm *ServerMgr) addTask(task *Task) (err error) { +func (qm *ServerMgr) addTask(task *Task, job *Job) (err error) { + logger.Debug(3, "(addTask) got task") + + task_id, err := task.GetId() + if err != nil { + return + } + + has_task, err := qm.TaskMap.Has(task_id, true) + if err != nil { + return + } + if has_task { + err = fmt.Errorf("(addTask) task %s is already in taskmap", task_id.String()) + return + } + task_state, err := task.GetState() if err != nil { return } + logger.Debug(3, "(addTask) state of task: %s", task_state) + + //if (task_state == TASK_STAT_COMPLETED) || (task_state == TASK_STAT_PASSED) { + // logger.Debug(3, "(addTask) already completed or passed") + // return + //} + //logger.Debug(3, "(addTask) NOT completed or passed") + + err = qm.TaskMap.Add(task) // makes it a pending task if init + if err != nil { + logger.Error("(qm.TaskMap.Add): %s", err.Error()) + } if (task_state == TASK_STAT_COMPLETED) || (task_state == TASK_STAT_PASSED) { - qm.TaskMap.Add(task) + // logger.Debug(3, "(addTask) already completed or passed") return } - defer qm.TaskMap.Add(task) - err = task.SetState(TASK_STAT_PENDING) + task_state, err = task.GetState() if err != nil { return } - task_ready, err := qm.isTaskReady(task) + task_ready, err := qm.isTaskReady(task) //makes the task ready if err != nil { return } - if task_ready { - err = qm.taskEnQueue(task) - if err != nil { - _ = task.SetState(TASK_STAT_SUSPEND) - task_id, _ := task.GetId() - job_id, _ := GetJobIdByTaskId(task_id) - jerror := &JobError{ - TaskFailed: task_id, - ServerNotes: "failed in enqueuing task, err=" + err.Error(), - Status: JOB_STAT_SUSPEND, - } - if serr := qm.SuspendJob(job_id, jerror); serr != nil { - logger.Error("(updateQueue:SuspendJob) job_id=%s; err=%s", job_id, serr.Error()) - } - return err + + if !task_ready { + return + } + + //task_id := task.String() + logger.Debug(3, "(addTask) task %s is ready (invoking taskEnQueue)", task_id) + err = qm.taskEnQueue(task, job) + if err != nil { + logger.Error("(addTask) taskEnQueue returned error: %s", err.Error()) + _ = task.SetState(TASK_STAT_SUSPEND, true) + + job_id, xerr := task.GetJobId() + if xerr != nil { + err = xerr + return + } + jerror := &JobError{ + TaskFailed: task_id.String(), + ServerNotes: "failed in enqueuing task, err=" + err.Error(), + Status: JOB_STAT_SUSPEND, } + if serr := qm.SuspendJob(job_id, jerror); serr != nil { + logger.Error("(updateQueue:SuspendJob) job_id=%s; err=%s", job_id, serr.Error()) + } + return err } + err = qm.updateJobTask(task) //task state INIT->PENDING + logger.Debug(3, "(addTask) leaving...") return } @@ -1097,7 +1221,15 @@ func (qm *ServerMgr) isTaskReady(task *Task) (ready bool, err error) { logger.Debug(3, "(isTaskReady) task state is %s", task_state) - if task_state != TASK_STAT_PENDING { + if task_state == TASK_STAT_READY { + ready = true + return + } + + if task_state == TASK_STAT_INIT || task_state == TASK_STAT_PENDING { + logger.Debug(3, "(isTaskReady) task state is %s", task_state) + } else { + err = fmt.Errorf("(isTaskReady) task has state %s, it does not make sense to test if it is ready", task_state) return } @@ -1105,7 +1237,12 @@ func (qm *ServerMgr) isTaskReady(task *Task) (ready bool, err error) { if err != nil { return } - logger.Debug(3, "(isTaskReady) task_id: %s", task_id) + logger.Debug(3, "(isTaskReady %s)", task_id) + + //defer func() { + // logger.Debug(3, "(isTaskReady %s) ready=%t", task_id, ready) + // //fmt.Printf("(isTaskReady %s) ready=%t\n", task_id, ready) + //}() //skip if the belonging job is suspended jobid, err := task.GetJobId() @@ -1117,41 +1254,127 @@ func (qm *ServerMgr) isTaskReady(task *Task) (ready bool, err error) { return } - logger.Debug(3, "(isTaskReady) GetDependsOn %s", task_id) - deps, xerr := task.GetDependsOn() - if xerr != nil { - err = xerr - return - } + if task.Info != nil { + info := task.Info + if !info.StartAt.IsZero() { - logger.Debug(3, "(isTaskReady) range deps %s (%d)", task_id, len(deps)) - for _, predecessor := range deps { - pretask, ok, yerr := qm.TaskMap.Get(predecessor, true) - if yerr != nil { - err = yerr + if info.StartAt.After(time.Now()) { + // too early + logger.Debug(3, "(isTaskReady %s) too early to execute (now: %s, StartAt: %s)", task_id, time.Now(), info.StartAt) + return + } else { + logger.Debug(3, "(isTaskReady %s) StartAt field is in the past, can execute now (now: %s, StartAt: %s)", task_id, time.Now(), info.StartAt) - return + } } + } - if !ok { - logger.Error("predecessor %s of task %s is unknown", predecessor, task_id) - ready = false + if task.WorkflowStep == nil { + + // check if AWE-style predecessors are all TASK_STAT_COMPLETED + + logger.Debug(3, "(isTaskReady %s) GetDependsOn", task_id) + deps, xerr := task.GetDependsOn() + if xerr != nil { + err = xerr return } - pretask_state, zerr := pretask.GetState() - if zerr != nil { - err = zerr + logger.Debug(3, "(isTaskReady %s) range deps (%d)", task_id, len(deps)) + for _, predecessor := range deps { + predecessor_id, xerr := New_Task_Unique_Identifier(predecessor) + if xerr != nil { + err = xerr + return + } + predecessor_task, ok, yerr := qm.TaskMap.Get(predecessor_id, true) + if yerr != nil { + err = yerr + return + } + if !ok { + logger.Error("(isTaskReady %s) predecessor %s is unknown", task_id, predecessor) + return + } + + predecessor_task_state, zerr := predecessor_task.GetState() + if zerr != nil { + err = zerr + return + } + + if predecessor_task_state != TASK_STAT_COMPLETED { + logger.Debug(3, "(isTaskReady %s) (AWE-style) not ready because predecessor is not ready", task_id) + return + } - return } + logger.Debug(3, "(isTaskReady %s) task seems to be ready", task_id) + } + + if task.WorkflowStep != nil { + // check if CWL-style predecessors are all TASK_STAT_COMPLETED + for _, wsi := range task.WorkflowStep.In { // WorkflowStepInput + + //job_input := *(job.CWL_collection.Job_input) + + for _, src := range wsi.Source { + fmt.Println("src: " + src) + + source_task := "" + source_name := "" + source_array := strings.Split(src, "/") + if len(source_array) == 0 { + err = fmt.Errorf("len(source_array) == 0") + return + } else if len(source_array) == 1 { + source_name = source_array[0] + } else if len(source_array) == 2 { + source_task = strings.TrimPrefix(source_array[0], "#") + source_name = source_array[1] + + } else { + err = fmt.Errorf("len(source_array) > 2 %s", src) + return + } + + fmt.Println("source_task: " + source_task) + fmt.Println("source_name: " + source_name) + + if source_task == "main" { + // wokflow input, can continue + continue + } + + predecessor_task_id := task.Task_Unique_Identifier + predecessor_task_id.Id = source_task + + predecessor_task, ok, yerr := qm.TaskMap.Get(predecessor_task_id, true) + if !ok { + err = fmt.Errorf("predecessor_task_id not found %s", predecessor_task_id.String()) + return + } + if yerr != nil { + err = yerr + return + } + + predecessor_task_state, zerr := predecessor_task.GetState() + if zerr != nil { + err = zerr + return + } + + if predecessor_task_state != TASK_STAT_COMPLETED { + logger.Debug(3, "(isTaskReady %s) (CWL-style) not ready because predecessor is not ready", task_id) + return + } + + } - if pretask_state != TASK_STAT_COMPLETED { - return } } - logger.Debug(3, "(isTaskReady) task %s is ready", task_id) modified := false for _, io := range task.Inputs { @@ -1161,14 +1384,15 @@ func (qm *ServerMgr) isTaskReady(task *Task) (ready bool, err error) { continue } - preId := fmt.Sprintf("%s_%s", jobid, io.Origin) + //preId := fmt.Sprintf("%s_%s", jobid, io.Origin) + preId := Task_Unique_Identifier{JobId: jobid, Id: io.Origin} preTask, ok, xerr := qm.TaskMap.Get(preId, true) if xerr != nil { err = xerr return } if !ok { - err = fmt.Errorf("Task %s not found", preId) + err = fmt.Errorf("(isTaskReady %s) Task %s not found", task_id, preId) return } @@ -1180,7 +1404,7 @@ func (qm *ServerMgr) isTaskReady(task *Task) (ready bool, err error) { } if pretask_state != TASK_STAT_COMPLETED { - err = fmt.Errorf("pretask_state != TASK_STAT_COMPLETED state: %s preId: %s", pretask_state, preId) + logger.Debug(3, "(isTaskReady %s) pretask_state != TASK_STAT_COMPLETED state: %s preId: %s", task_id, pretask_state, preId) return } @@ -1191,7 +1415,7 @@ func (qm *ServerMgr) isTaskReady(task *Task) (ready bool, err error) { return } - logger.Debug(3, "pretask_output size = %d, state = %s", pretask_output.Size, preTask.State) + logger.Debug(3, "(isTaskReady %s) pretask_output size = %d, state = %s", task_id, pretask_output.Size, preTask.State) if io.Size != pretask_output.Size { io.Size = pretask_output.Size @@ -1207,123 +1431,226 @@ func (qm *ServerMgr) isTaskReady(task *Task) (ready bool, err error) { } } - ready = true + if task_state == TASK_STAT_PENDING { + err = task.SetState(TASK_STAT_READY, true) + if err != nil { + return + } + } + ready = true + logger.Debug(3, "(isTaskReady %s) finished, task is ready", task_id) return } // happens when task is ready -func (qm *ServerMgr) taskEnQueue(task *Task) (err error) { +// prepares task and creates workunits +func (qm *ServerMgr) taskEnQueue(task *Task, job *Job) (err error) { task_id, err := task.GetId() if err != nil { + err = fmt.Errorf("(taskEnQueue) Could not get Id: %s", err.Error()) return } - job_id, err := task.GetJobId() + state, err := task.GetState() if err != nil { + err = fmt.Errorf("(taskEnQueue) Could not get State: %s", err.Error()) return } - logger.Debug(2, "qmgr.taskEnQueue trying to enqueue task %s", task_id) + if state != TASK_STAT_READY { + err = fmt.Errorf("(taskEnQueue) Task state should be TASK_STAT_READY, got state %s", state) + return + } - if err := qm.locateInputs(task); err != nil { - logger.Error("qmgr.taskEnQueue locateInputs:" + err.Error()) - return err + if task.WorkflowStep != nil { + logger.Debug(3, "(taskEnQueue) have WorkflowStep") + } else { + logger.Debug(3, "(taskEnQueue) DO NOT have WorkflowStep") + } + + if job.CWL_collection != nil { + logger.Debug(3, "(taskEnQueue) have job.CWL_collection") + } else { + logger.Debug(3, "(taskEnQueue) DO NOT have job.CWL_collection") + } + + logger.Debug(2, "(qmgr.taskEnQueue) trying to enqueue task %s", task_id) + + err = qm.locateInputs(task, job) + if err != nil { + err = fmt.Errorf("(qmgr.taskEnQueue) locateInputs: %s", err.Error()) + return } //create shock index on input nodes (if set in workflow document) - if err := task.CreateIndex(); err != nil { - logger.Error("qmgr.taskEnQueue CreateIndex:" + err.Error()) - return err + err = task.CreateIndex() + if err != nil { + err = fmt.Errorf("(qmgr.taskEnQueue) CreateIndex: %s", err.Error()) + return } //init partition - if err := task.InitPartIndex(); err != nil { - logger.Error("qmgr.taskEnQueue InitPartitionIndex:" + err.Error()) - return err + err = task.InitPartIndex() + if err != nil { + err = fmt.Errorf("(qmgr.taskEnQueue) InitPartitionIndex: %s", err.Error()) + return } - if err := qm.createOutputNode(task); err != nil { - logger.Error("qmgr.taskEnQueue createOutputNode:" + err.Error()) - return err + err = qm.createOutputNode(task) + if err != nil { + err = fmt.Errorf("(qmgr.taskEnQueue) createOutputNode: %s", err.Error()) + return } - if err := qm.CreateAndEnqueueWorkunits(task); err != nil { - logger.Error("qmgr.taskEnQueue CreateAndEnqueueWorkunits:" + err.Error()) - return err + err = qm.CreateAndEnqueueWorkunits(task, job) + if err != nil { + err = fmt.Errorf("(qmgr.taskEnQueue) CreateAndEnqueueWorkunits: %s", err.Error()) + return } - err = task.SetState(TASK_STAT_QUEUED) + err = task.SetState(TASK_STAT_QUEUED, true) if err != nil { return } - task.SetCreatedDate(time.Now()) - task.SetStartedDate(time.Now()) //TODO: will be changed to the time when the first workunit is checked out - qm.updateJobTask(task) //task status PENDING->QUEUED - + err = task.SetCreatedDate(time.Now()) + if err != nil { + return + } + err = task.SetStartedDate(time.Now()) //TODO: will be changed to the time when the first workunit is checked out + if err != nil { + return + } + err = qm.updateJobTask(task) //task status PENDING->QUEUED + if err != nil { + err = fmt.Errorf("(qmgr.taskEnQueue) qm.updateJobTask: %s", err.Error()) + return + } //log event about task enqueue (TQ) logger.Event(event.TASK_ENQUEUE, fmt.Sprintf("taskid=%s;totalwork=%d", task_id, task.TotalWork)) - qm.CreateTaskPerf(task_id) + qm.CreateTaskPerf(task) - if IsFirstTask(task_id) { - UpdateJobState(job_id, JOB_STAT_QUEUED, []string{JOB_STAT_INIT, JOB_STAT_SUSPEND}) - } + //job_id, err := task.GetJobId() + //if err != nil { + // err = fmt.Errorf("(taskEnQueue) Could not get JobId: %s", err.Error()) + // return + // } + + //if IsFirstTask(task_id) { + + //job, err := LoadJob(job_id) + //if err != nil { + // return + //} + + //} + + logger.Debug(2, "(qmgr.taskEnQueue) leaving (task=%s)", task_id) return } -func (qm *ServerMgr) locateInputs(task *Task) (err error) { - task_id, err := task.GetId() +// invoked by taskEnQueue +// main purpose is to copy output io struct of predecessor task to create the input io structs +func (qm *ServerMgr) locateInputs(task *Task, job *Job) (err error) { + + if task.WorkflowStep != nil && job.CWL_collection != nil { + + if job.CWL_collection.Job_input_map == nil { + err = fmt.Errorf("job.CWL_collection.Job_input_map is empty") + return + } + + job_input_map := *(job.CWL_collection.Job_input_map) + + // copy inputs into task + for _, wsi := range task.WorkflowStep.In { // WorkflowStepInput + if len(wsi.Source) > 0 { + + for _, src := range wsi.Source { + fmt.Println("src: " + src) + + src_base := path.Base(src) + fmt.Println("src_base: " + src_base) + // search job input + obj, ok := job_input_map[src_base] + if ok { + fmt.Println("(locateInputs) found in job input: " + src_base) + } else { + fmt.Println("(locateInputs) NOT found in job input: " + src_base) + err = fmt.Errorf("(locateInputs) did not find %s in job_input (TODO check collection)", src_base) // this should not happen, taskReady makes sure everything is available + return + } + spew.Dump(job_input_map) + _ = obj + } + } + + } + + } else { + + // old AWE-style + // this code will look for the predecessor output io struct and copy it into the current task as an input + err = qm.locateAWEInputs(task, job) + if err != nil { + return + } + } + return +} + +// old AWE-style function +func (qm *ServerMgr) locateAWEInputs(task *Task, job *Job) (err error) { + jobid, err := task.GetJobId() if err != nil { return } - logger.Debug(2, "trying to locate Inputs of task "+task_id) - jobid, err := task.GetJobId() + task_id, err := task.GetId() if err != nil { return } + logger.Debug(2, "(locateInputs) trying to locate Inputs of task "+task_id.String()) inputs_modified := false for _, io := range task.Inputs { filename := io.FileName if io.Url == "" { - preId := fmt.Sprintf("%s_%s", jobid, io.Origin) + + // find predecessor task + + preId := Task_Unique_Identifier{JobId: jobid, Id: io.Origin} preTask, ok, xerr := qm.TaskMap.Get(preId, true) if xerr != nil { err = xerr return } + if !ok { + err = fmt.Errorf("(locateInputs) predecessor task %s not found", preId.String()) + return + } - if ok { - if preTask.State == TASK_STAT_SKIPPED || - preTask.State == TASK_STAT_FAIL_SKIP { - // For now we know that skipped tasks have - // just one input and one output. So we know - // that we just need to change one file (this - // may change in the future) - //locateSkippedInput(qm, preTask, io) - } else { - - output, xerr := preTask.GetOutput(filename) - if xerr != nil { - err = xerr - return - } - - if io.Node != output.Node { - io.Node = output.Node - inputs_modified = true - } + // find predecessor output + output, xerr := preTask.GetOutput(filename) + if xerr != nil { + err = xerr + return + } - } + // copy if not already done + if io.Node != output.Node { + io.Node = output.Node + inputs_modified = true } + } logger.Debug(2, "(locateInputs) processing input %s, %s", filename, io.Node) if io.Node == "-" { err = fmt.Errorf("(locateInputs) error in locate input for task, no node id found. task_id: %s, input name: %s", task_id, filename) return } - //need time out! + + // double-check that node and file exist _, modified, xerr := io.GetFileSize() if xerr != nil { err = fmt.Errorf("(locateInputs) task %s: input file %s GetFileSize returns: %s (DataToken len: %d)", task_id, filename, xerr.Error(), len(io.DataToken)) @@ -1351,13 +1678,13 @@ func (qm *ServerMgr) locateInputs(task *Task) (err error) { if (io.Node != "") && (io.Node != "-") { _, modified, xerr := io.GetFileSize() if xerr != nil { - err = fmt.Errorf("task %s: input file %s GetFileSize returns: %s", task_id, name, xerr.Error()) + err = fmt.Errorf("(locateInputs) task %s: input file %s GetFileSize returns: %s", task_id, name, xerr.Error()) return } if modified { predata_modified = true } - logger.Debug(2, "predata located %s, %s", name, io.Node) + logger.Debug(2, "(locateInputs) predata located %s, %s", name, io.Node) } } @@ -1367,20 +1694,22 @@ func (qm *ServerMgr) locateInputs(task *Task) (err error) { return } } - return } -func (qm *ServerMgr) CreateAndEnqueueWorkunits(task *Task) (err error) { - workunits, err := task.CreateWorkunits() +func (qm *ServerMgr) CreateAndEnqueueWorkunits(task *Task, job *Job) (err error) { + logger.Debug(3, "(CreateAndEnqueueWorkunits) starting") + workunits, err := task.CreateWorkunits(job) if err != nil { return err } for _, wu := range workunits { if err := qm.workQueue.Add(wu); err != nil { + err = fmt.Errorf("(CreateAndEnqueueWorkunits) error in qm.workQueue.Add: %s", err.Error()) return err } - qm.CreateWorkPerf(wu.Id) + id := wu.GetId() + qm.CreateWorkPerf(id) } return } @@ -1434,9 +1763,18 @@ func (qm *ServerMgr) createOutputNode(task *Task) (err error) { func (qm *ServerMgr) locateUpdate(task *Task, name string, origin string) (nodeid string, err error) { //jobid, _ := GetJobIdByTaskId(taskid) - task_id := task.Id - job_id := task.JobId - preId := fmt.Sprintf("%s_%s", job_id, origin) + task_id, err := task.GetId() + if err != nil { + return + } + job_id, err := task.GetJobId() + if err != nil { + return + } + + //preId := fmt.Sprintf("%s_%s", job_id, origin) + preId := Task_Unique_Identifier{JobId: job_id, Id: origin} + logger.Debug(2, "task %s: trying to locate Node of update %s from task %s", task_id, name, preId) // scan outputs in origin task preTask, ok, err := qm.TaskMap.Get(preId, true) @@ -1479,6 +1817,7 @@ func (qm *ServerMgr) ShowTasks() { //---end of task methods--- //update job info when a task in that job changed to a new state +// this is invoked everytime a task changes state, but only when remainTasks==0 and state is not yet JOB_STAT_COMPLETED it will complete the job func (qm *ServerMgr) updateJobTask(task *Task) (err error) { //parts := strings.Split(task.Id, "_") //jobid := parts[0] @@ -1494,7 +1833,7 @@ func (qm *ServerMgr) updateJobTask(task *Task) (err error) { } logger.Debug(2, "remaining tasks for job %s: %d", task.Id, remainTasks) - if remainTasks > 0 { + if remainTasks > 0 { //##################################################################### return } @@ -1507,7 +1846,7 @@ func (qm *ServerMgr) updateJobTask(task *Task) (err error) { return } - err = job.SetState(JOB_STAT_COMPLETED) + err = job.SetState(JOB_STAT_COMPLETED, nil) if err != nil { return } @@ -1523,7 +1862,11 @@ func (qm *ServerMgr) updateJobTask(task *Task) (err error) { // delete nodes that have been flagged to be deleted modified += task.DeleteOutput() modified += task.DeleteInput() - qm.TaskMap.Delete(task.Id) + //combined_id := jobid + "_" + task.Id + + id, _ := task.GetId() + + qm.TaskMap.Delete(id) } if modified > 0 { @@ -1565,23 +1908,32 @@ func (qm *ServerMgr) updateJobTask(task *Task) (err error) { } //update job/task states from "queued" to "in-progress" once the first workunit is checked out -func (qm *ServerMgr) UpdateJobTaskToInProgress(works []*Workunit) { +func (qm *ServerMgr) UpdateJobTaskToInProgress(works []*Workunit) (err error) { for _, work := range works { //job_was_inprogress := false //task_was_inprogress := false - taskid, _ := GetTaskIdByWorkId(work.Id) - jobid, _ := GetJobIdByWorkId(work.Id) + taskid := work.GetTask() + jobid := work.JobId // get job state - job_state, err := dbGetJobFieldString(jobid, "state") + job, xerr := GetJob(jobid) + if xerr != nil { + err = xerr + return + } + + job_state, xerr := job.GetState(true) + if xerr != nil { + err = xerr + return + } //update job status if job_state != JOB_STAT_INPROGRESS { - - dbUpdateJobState(jobid, JOB_STAT_INPROGRESS, "") - - DbUpdateJobField(jobid, "info.startedtime", time.Now()) - + err = job.SetState(JOB_STAT_INPROGRESS, nil) + if err != nil { + return + } qm.UpdateJobPerfStartTime(jobid) } @@ -1602,14 +1954,15 @@ func (qm *ServerMgr) UpdateJobTaskToInProgress(works []*Workunit) { } if task_state != TASK_STAT_INPROGRESS { - err := task.SetState(TASK_STAT_INPROGRESS) + err := task.SetState(TASK_STAT_INPROGRESS, true) if err != nil { logger.Error("(UpdateJobTaskToInProgress) could not update task %s", taskid) continue } - qm.UpdateTaskPerfStartTime(taskid) + qm.UpdateTaskPerfStartTime(task) } } + return } func (qm *ServerMgr) IsJobRegistered(id string) bool { @@ -1629,7 +1982,7 @@ func (qm *ServerMgr) SuspendJob(jobid string, jerror *JobError) (err error) { return } - err = job.SetState(jerror.Status) + err = job.SetState(jerror.Status, nil) if err != nil { return } @@ -1660,10 +2013,11 @@ func (qm *ServerMgr) SuspendJob(jobid string, jerror *JobError) (err error) { // update all workunits for _, workunit := range workunit_list { - workid := workunit.Id - parentid, _ := GetJobIdByWorkId(workid) + workid := workunit.Workunit_Unique_Identifier + parentid := workunit.JobId + //parentid, _ := GetJobIdByWorkId(workid) if jobid == parentid { - qm.workQueue.StatusChange(workid, nil, new_work_state) + qm.workQueue.StatusChange(workid, nil, new_work_state, "see job error") } } @@ -1674,7 +2028,7 @@ func (qm *ServerMgr) SuspendJob(jobid string, jerror *JobError) (err error) { continue } if task_state == TASK_STAT_QUEUED || task_state == TASK_STAT_INIT || task_state == TASK_STAT_INPROGRESS { - err = task.SetState(new_task_state) + err = task.SetState(new_task_state, true) if err != nil { logger.Error("(SuspendJob) : %s", err.Error()) continue @@ -1705,7 +2059,7 @@ func (qm *ServerMgr) DeleteJobByUser(jobid string, u *user.User, full bool) (err if job.Acl.Owner != u.Uuid && rights["delete"] == false && u.Admin == false { return errors.New(e.UnAuth) } - if err := job.SetState(JOB_STAT_DELETED); err != nil { + if err := job.SetState(JOB_STAT_DELETED, nil); err != nil { return err } //delete queueing workunits @@ -1714,15 +2068,19 @@ func (qm *ServerMgr) DeleteJobByUser(jobid string, u *user.User, full bool) (err return } for _, workunit := range workunit_list { - workid := workunit.Id - parentid, _ := GetJobIdByWorkId(workid) - if jobid == parentid { + + workid := workunit.Workunit_Unique_Identifier + workunit_jobid := workid.JobId + //parentid, _ := GetJobIdByWorkId(workid) + if jobid == workunit_jobid { + qm.workQueue.Delete(workid) } } //delete parsed tasks for i := 0; i < len(job.TaskList()); i++ { - task_id := fmt.Sprintf("%s_%d", jobid, i) + //task_id := fmt.Sprintf("%s_%d", jobid, i) + task_id := Task_Unique_Identifier{JobId: jobid, Id: strconv.Itoa(i)} qm.TaskMap.Delete(task_id) } qm.removeActJob(jobid) @@ -1803,9 +2161,9 @@ func (qm *ServerMgr) ResumeSuspendedJobByUser(id string, u *user.User) (err erro } if remain_tasks < len(dbjob.Tasks) { - dbjob.SetState(JOB_STAT_INPROGRESS) + dbjob.SetState(JOB_STAT_INPROGRESS, nil) } else { - dbjob.SetState(JOB_STAT_QUEUED) + dbjob.SetState(JOB_STAT_QUEUED, nil) } err = dbjob.IncrementResumed(1) @@ -1887,12 +2245,12 @@ func (qm *ServerMgr) RecoverJobs() (err error) { // Directly after AWE server restart no job can be in progress. (Unless we add this as a feature)) if job_state == JOB_STAT_INPROGRESS { - err = DbUpdateJobField(dbjob.Id, "state", JOB_STAT_QUEUED) - if err != nil { - logger.Error("error while recover: " + err.Error()) - continue - } - err = dbjob.SetState(JOB_STAT_QUEUED) + //err = DbUpdateJobField(dbjob.Id, "state", JOB_STAT_QUEUED) // SetState is already doing that for us + //if err != nil { + // logger.Error("error while recover: " + err.Error()) + // continue + //} + err = dbjob.SetState(JOB_STAT_QUEUED, nil) if err != nil { logger.Error(err.Error()) continue @@ -1950,8 +2308,11 @@ func (qm *ServerMgr) RecomputeJob(jobid string, stage string) (err error) { return xerr } - if task_id == from_task_id { - resetTask(task, dbjob.Info) + if task_id.String() == from_task_id { + err = resetTask(task, dbjob.Info) + if err != nil { + return + } remaintasks += 1 found = true } @@ -1964,8 +2325,11 @@ func (qm *ServerMgr) RecomputeJob(jobid string, stage string) (err error) { if xerr != nil { return xerr } - if isAncestor(dbjob, task_id, from_task_id) { - resetTask(task, dbjob.Info) + if isAncestor(dbjob, task_id.Id, from_task_id) { + err = resetTask(task, dbjob.Info) + if err != nil { + return + } remaintasks += 1 } } @@ -1985,7 +2349,7 @@ func (qm *ServerMgr) RecomputeJob(jobid string, stage string) (err error) { } else { new_state = JOB_STAT_QUEUED } - dbjob.SetState(new_state) + dbjob.SetState(new_state, nil) if was_suspend { qm.removeSusJob(jobid) @@ -2025,7 +2389,7 @@ func (qm *ServerMgr) ResubmitJob(jobid string) (err error) { if err != nil { return } - err = job.SetState(JOB_STAT_QUEUED) + err = job.SetState(JOB_STAT_QUEUED, nil) if err != nil { return } @@ -2044,10 +2408,16 @@ func (qm *ServerMgr) ResubmitJob(jobid string) (err error) { } // TODO Lock !!!! -func resetTask(task *Task, info *Info) { +func resetTask(task *Task, info *Info) (err error) { task.Info = info - _ = task.SetState(TASK_STAT_PENDING) - _ = task.SetRemainWork(task.TotalWork, false) // TODO err + err = task.SetState(TASK_STAT_PENDING, true) + if err != nil { + return + } + err = task.SetRemainWork(task.TotalWork, false) // TODO err + if err != nil { + return + } task.ComputeTime = 0 task.CompletedDate = time.Time{} @@ -2064,7 +2434,9 @@ func resetTask(task *Task, info *Info) { if dataUrl, _ := output.DataUrl(); dataUrl != "" { // delete dataUrl if is shock node if strings.HasSuffix(dataUrl, shock.DATA_SUFFIX) { - if err := shock.ShockDelete(output.Host, output.Node, output.DataToken); err == nil { + + err = shock.ShockDelete(output.Host, output.Node, output.DataToken) + if err == nil { logger.Debug(2, "Deleted node %s from shock", output.Node) } else { logger.Error("resetTask: unable to deleted node %s from shock: %s", output.Node, err.Error()) @@ -2077,8 +2449,9 @@ func resetTask(task *Task, info *Info) { } // delete all workunit logs for _, log := range conf.WORKUNIT_LOGS { - deleteStdLogByTask(task.Id, log) + deleteStdLogByTask(task, log) } + return } func isAncestor(job *Job, taskId string, testId string) bool { @@ -2112,8 +2485,10 @@ func isAncestor(job *Job, taskId string, testId string) bool { //update tokens for in-memory data structures func (qm *ServerMgr) UpdateQueueToken(job *Job) (err error) { + //job_id := job.Id for _, task := range job.Tasks { - mtask, ok, err := qm.TaskMap.Get(task.Id, true) + task_id, _ := task.GetId() + mtask, ok, err := qm.TaskMap.Get(task_id, true) if err != nil { return err } @@ -2152,16 +2527,18 @@ func (qm *ServerMgr) FinalizeJobPerf(jobid string) { return } -func (qm *ServerMgr) CreateTaskPerf(taskid string) { - jobid, _ := GetJobIdByTaskId(taskid) +func (qm *ServerMgr) CreateTaskPerf(task *Task) { + jobid := task.JobId + taskid := task.Id if perf, ok := qm.getActJob(jobid); ok { perf.Ptasks[taskid] = NewTaskPerf(taskid) qm.putActJob(perf) } } -func (qm *ServerMgr) UpdateTaskPerfStartTime(taskid string) { - jobid, _ := GetJobIdByTaskId(taskid) +func (qm *ServerMgr) UpdateTaskPerfStartTime(task *Task) { + jobid := task.JobId + taskid := task.Id if jobperf, ok := qm.getActJob(jobid); ok { if taskperf, ok := jobperf.Ptasks[taskid]; ok { now := time.Now().Unix() @@ -2171,10 +2548,16 @@ func (qm *ServerMgr) UpdateTaskPerfStartTime(taskid string) { } } -func (qm *ServerMgr) FinalizeTaskPerf(task *Task) { - jobid, _ := GetJobIdByTaskId(task.Id) +// TODO evaluate err +func (qm *ServerMgr) FinalizeTaskPerf(task *Task) (err error) { + //jobid, _ := GetJobIdByTaskId(task.Id) + jobid, err := task.GetJobId() + if err != nil { + return + } if jobperf, ok := qm.getActJob(jobid); ok { - if taskperf, ok := jobperf.Ptasks[task.Id]; ok { + combined_id := jobid + "_" + task.Id + if taskperf, ok := jobperf.Ptasks[combined_id]; ok { now := time.Now().Unix() taskperf.End = now taskperf.Resp = now - taskperf.Queued @@ -2189,20 +2572,22 @@ func (qm *ServerMgr) FinalizeTaskPerf(task *Task) { return } } + return } -func (qm *ServerMgr) CreateWorkPerf(workid string) { +func (qm *ServerMgr) CreateWorkPerf(id Workunit_Unique_Identifier) { if !conf.PERF_LOG_WORKUNIT { return } - jobid, _ := GetJobIdByWorkId(workid) + workid := id.String() + jobid := id.JobId if jobperf, ok := qm.getActJob(jobid); ok { - jobperf.Pworks[workid] = NewWorkPerf(workid) + jobperf.Pworks[workid] = NewWorkPerf() qm.putActJob(jobperf) } } -func (qm *ServerMgr) FinalizeWorkPerf(workid string, reportfile string) (err error) { +func (qm *ServerMgr) FinalizeWorkPerf(id Workunit_Unique_Identifier, reportfile string) (err error) { if !conf.PERF_LOG_WORKUNIT { return } @@ -2214,11 +2599,12 @@ func (qm *ServerMgr) FinalizeWorkPerf(workid string, reportfile string) (err err if err := json.Unmarshal(jsonstream, workperf); err != nil { return err } - jobid, _ := GetJobIdByWorkId(workid) + jobid := id.JobId jobperf, ok := qm.getActJob(jobid) if !ok { return errors.New("job perf not found:" + jobid) } + workid := id.String() if _, ok := jobperf.Pworks[workid]; !ok { return errors.New("work perf not found:" + workid) } @@ -2242,7 +2628,7 @@ func (qm *ServerMgr) LogJobPerf(jobid string) { //---end of perf related methods -func (qm *ServerMgr) FetchPrivateEnv(workid string, clientid string) (env map[string]string, err error) { +func (qm *ServerMgr) FetchPrivateEnv(id Workunit_Unique_Identifier, clientid string) (env map[string]string, err error) { //precheck if the client is registered client, ok, err := qm.GetClient(clientid, true) if err != nil { @@ -2251,28 +2637,22 @@ func (qm *ServerMgr) FetchPrivateEnv(workid string, clientid string) (env map[st if !ok { return env, errors.New(e.ClientNotFound) } - client_status, err := client.Get_Status(true) + + is_suspended, err := client.Get_Suspended(true) if err != nil { return } - if client_status == CLIENT_STAT_SUSPEND { - return env, errors.New(e.ClientSuspended) - } - jobid, err := GetJobIdByWorkId(workid) - if err != nil { - return env, err + if is_suspended { + err = errors.New(e.ClientSuspended) + return } - //job, err := GetJob(jobid) - //if err != nil { - // return env, err - //} - taskid, err := GetTaskIdByWorkId(workid) - //env = job.GetPrivateEnv(taskid) + jobid := id.JobId + taskid := id.TaskId env, err = dbGetPrivateEnv(jobid, taskid) if err != nil { return } - return env, nil + return } diff --git a/lib/core/task.go b/lib/core/task.go index 644f841b..4d9a6cd5 100644 --- a/lib/core/task.go +++ b/lib/core/task.go @@ -4,46 +4,68 @@ import ( "errors" "fmt" "github.com/MG-RAST/AWE/lib/conf" + "github.com/MG-RAST/AWE/lib/core/cwl" "github.com/MG-RAST/AWE/lib/logger" "github.com/MG-RAST/AWE/lib/shock" + "regexp" + //"strconv" + //"github.com/davecgh/go-spew/spew" "strings" "time" ) const ( - TASK_STAT_INIT = "init" - TASK_STAT_QUEUED = "queued" - TASK_STAT_INPROGRESS = "in-progress" - TASK_STAT_PENDING = "pending" + TASK_STAT_INIT = "init" // initial state on creation of a task + TASK_STAT_PENDING = "pending" // a task that wants to be enqueued + TASK_STAT_READY = "ready" // a task ready to be enqueued + TASK_STAT_QUEUED = "queued" // a task for which workunits have been created/queued + TASK_STAT_INPROGRESS = "in-progress" // a first workunit has been checkout (this does not guarantee a workunit is running right now) TASK_STAT_SUSPEND = "suspend" TASK_STAT_FAILED = "failed" - TASK_STAT_FAILED_PERMANENT = "failed-permanent" + TASK_STAT_FAILED_PERMANENT = "failed-permanent" // on exit code 42 TASK_STAT_COMPLETED = "completed" - TASK_STAT_SKIPPED = "user_skipped" - TASK_STAT_FAIL_SKIP = "skipped" - TASK_STAT_PASSED = "passed" + TASK_STAT_SKIPPED = "user_skipped" // deprecated + TASK_STAT_FAIL_SKIP = "skipped" // deprecated + TASK_STAT_PASSED = "passed" // deprecated ? ) type TaskRaw struct { RWMutex - Id string `bson:"taskid" json:"taskid"` - JobId string `bson:"jobid" json:"jobid"` - Info *Info `bson:"-" json:"-"` - Cmd *Command `bson:"cmd" json:"cmd"` - Partition *PartInfo `bson:"partinfo" json:"-"` - DependsOn []string `bson:"dependsOn" json:"dependsOn"` // only needed if dependency cannot be inferred from Input.Origin - TotalWork int `bson:"totalwork" json:"totalwork"` - MaxWorkSize int `bson:"maxworksize" json:"maxworksize"` - RemainWork int `bson:"remainwork" json:"remainwork"` - //WorkStatus []string `bson:"workstatus" json:"-"` - State string `bson:"state" json:"state"` - //Skip int `bson:"skip" json:"-"` - CreatedDate time.Time `bson:"createdDate" json:"createddate"` - StartedDate time.Time `bson:"startedDate" json:"starteddate"` - CompletedDate time.Time `bson:"completedDate" json:"completeddate"` - ComputeTime int `bson:"computetime" json:"computetime"` - UserAttr map[string]string `bson:"userattr" json:"userattr"` - ClientGroups string `bson:"clientgroups" json:"clientgroups"` + Task_Unique_Identifier `bson:",inline"` + Info *Info `bson:"-" json:"-"` // this is just a pointer to the job.Info + Cmd *Command `bson:"cmd" json:"cmd"` + Partition *PartInfo `bson:"partinfo" json:"-"` + DependsOn []string `bson:"dependsOn" json:"dependsOn"` // only needed if dependency cannot be inferred from Input.Origin + TotalWork int `bson:"totalwork" json:"totalwork"` + MaxWorkSize int `bson:"maxworksize" json:"maxworksize"` + RemainWork int `bson:"remainwork" json:"remainwork"` + State string `bson:"state" json:"state"` + CreatedDate time.Time `bson:"createdDate" json:"createddate"` + StartedDate time.Time `bson:"startedDate" json:"starteddate"` + CompletedDate time.Time `bson:"completedDate" json:"completeddate"` + ComputeTime int `bson:"computetime" json:"computetime"` + UserAttr map[string]string `bson:"userattr" json:"userattr"` + ClientGroups string `bson:"clientgroups" json:"clientgroups"` + WorkflowStep *cwl.WorkflowStep `bson:"workflowStep" json:"workflowStep"` // CWL-only +} + +type Task_Unique_Identifier struct { + Id string `bson:"taskid" json:"taskid"` // local identifier + JobId string `bson:"jobid" json:"jobid"` +} + +func New_Task_Unique_Identifier(old_style_id string) (t Task_Unique_Identifier, err error) { + + array := strings.Split(old_style_id, "_") + + if len(array) != 2 { + err = fmt.Errorf("(New_Task_Unique_Identifier) Cannot parse task identifier: %s", old_style_id) + return + } + + t = Task_Unique_Identifier{JobId: array[0], Id: array[1]} + + return } type Task struct { @@ -71,12 +93,12 @@ type TaskLog struct { Workunits []*WorkLog `bson:"workunits" json:"workunits"` } -func NewTaskRaw(task_id string, info *Info) TaskRaw { +func NewTaskRaw(task_id Task_Unique_Identifier, info *Info) TaskRaw { logger.Debug(3, "task_id: %s", task_id) return TaskRaw{ - Id: task_id, + Task_Unique_Identifier: task_id, Info: info, Cmd: &Command{}, Partition: nil, @@ -104,34 +126,9 @@ func (task *TaskRaw) InitRaw(job *Job) (changed bool, err error) { if task.State == "" { task.State = TASK_STAT_INIT - } - - if !strings.Contains(task.Id, "_") { - // is not standard taskid, convert it - task.Id = fmt.Sprintf("%s_%s", job.Id, task.Id) changed = true } - fix_DependsOn := false - for _, dependency := range task.DependsOn { - if !strings.Contains(dependency, "_") { - fix_DependsOn = true - } - } - - if fix_DependsOn { - changed = true - new_DependsOn := []string{} - for _, dependency := range task.DependsOn { - if strings.Contains(dependency, "_") { - new_DependsOn = append(new_DependsOn, dependency) - } else { - new_DependsOn = append(new_DependsOn, fmt.Sprintf("%s_%s", job.Id, dependency)) - } - } - task.DependsOn = new_DependsOn - } - if job.Info == nil { err = fmt.Errorf("(NewTask) job.Info empty") return @@ -153,50 +150,141 @@ func (task *TaskRaw) InitRaw(job *Job) (changed bool, err error) { task.Cmd.HasPrivateEnv = true } + if strings.HasPrefix(task.Id, task.JobId+"_") { + task.Id = strings.TrimPrefix(task.Id, task.JobId+"_") + changed = true + } + + if strings.HasPrefix(task.Id, "_") { + task.Id = strings.TrimPrefix(task.Id, "_") + changed = true + } + return } -func (task *Task) Init(job *Job) (changed bool, err error) { - changed, err = task.InitRaw(job) +func (task Task_Unique_Identifier) String() (s string) { + + id := task.Id + jobId := task.JobId + + return fmt.Sprintf("%s_%s", jobId, id) +} + +// func (task *TaskRaw) String() (s string, err error) { +// err = task.LockNamed("String") +// if err != nil { +// return +// } +// defer task.Unlock() +// +// s = task.Task_Unique_Identifier.String() +// +// return +// } + +func IsValidUUID(uuid string) bool { + if len(uuid) != 36 { + return false + } + r := regexp.MustCompile("^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-4[a-fA-F0-9]{3}-[8|9|aA|bB][a-fA-F0-9]{3}-[a-fA-F0-9]{12}$") + return r.MatchString(uuid) +} + +// populate DependsOn +func (task *Task) CollectDependencies() (changed bool, err error) { + + deps := make(map[Task_Unique_Identifier]bool) + deps_changed := false + + jobid, err := task.GetJobId() if err != nil { return } + job_prefix := jobid + "_" - // populate DependsOn - deps := make(map[string]bool) - deps_changed := false // collect explicit dependencies for _, deptask := range task.DependsOn { - if !strings.Contains(deptask, "_") { - err = fmt.Errorf("deptask \"%s\" is missing _", deptask) + + if deptask == "" { + deps_changed = true + continue + } + + if !strings.HasPrefix(deptask, job_prefix) { + deptask = job_prefix + deptask + deps_changed = true + } + + t, yerr := New_Task_Unique_Identifier(deptask) + if yerr != nil { + err = fmt.Errorf("Cannot parse entry in DependsOn: %s", yerr.Error()) return } - deps[deptask] = true + + if t.Id == "" { + // this is to fix a bug + deps_changed = true + continue + } + + deps[t] = true } for _, input := range task.Inputs { - if input.Origin != "" { - origin := input.Origin - if !strings.Contains(origin, "_") { - origin = fmt.Sprintf("%s_%s", job.Id, origin) - } - _, ok := deps[origin] - if !ok { - // this was not yet in deps - deps[origin] = true - deps_changed = true - } + + deptask := input.Origin + if deptask == "" { + deps_changed = true + continue + } + + if !strings.HasPrefix(deptask, job_prefix) { + deptask = job_prefix + deptask + deps_changed = true + } + + t, yerr := New_Task_Unique_Identifier(deptask) + if yerr != nil { + + err = fmt.Errorf("Cannot parse Origin entry in Input: %s", yerr.Error()) + return + + } + + _, ok := deps[t] + if !ok { + // this was not yet in deps + deps[t] = true + deps_changed = true } + } // write all dependencies if different from before if deps_changed { task.DependsOn = []string{} for deptask, _ := range deps { - task.DependsOn = append(task.DependsOn, deptask) + task.DependsOn = append(task.DependsOn, deptask.String()) } changed = true } + return +} + +func (task *Task) Init(job *Job) (changed bool, err error) { + changed, err = task.InitRaw(job) + if err != nil { + return + } + + dep_changes, err := task.CollectDependencies() + if err != nil { + return + } + if dep_changes { + changed = true + } // set node / host / url for files for _, io := range task.Inputs { @@ -243,9 +331,12 @@ func (task *Task) Init(job *Job) (changed bool, err error) { } // currently this is only used to make a new task from a depricated task -func NewTask(job *Job, task_id string) (t *Task, err error) { +func NewTask(job *Job, task_id string) (t *Task) { + + tui := Task_Unique_Identifier{Id: task_id, JobId: job.Id} + t = &Task{ - TaskRaw: NewTaskRaw(task_id, job.Info), + TaskRaw: NewTaskRaw(tui, job.Info), Inputs: []*IO{}, Outputs: []*IO{}, Predata: []*IO{}, @@ -355,13 +446,13 @@ func (task *TaskRaw) GetStateNamed(name string) (state string, err error) { return } -func (task *TaskRaw) GetId() (id string, err error) { +func (task *TaskRaw) GetId() (id Task_Unique_Identifier, err error) { lock, err := task.RLockNamed("GetId") if err != nil { return } defer task.RUnlockNamed(lock) - id = task.Id + id = task.Task_Unique_Identifier return } @@ -375,12 +466,14 @@ func (task *TaskRaw) GetJobId() (id string, err error) { return } -func (task *TaskRaw) SetState(new_state string) (err error) { - err = task.LockNamed("SetState") - if err != nil { - return +func (task *TaskRaw) SetState(new_state string, write_lock bool) (err error) { + if write_lock { + err = task.LockNamed("SetState") + if err != nil { + return + } + defer task.Unlock() } - defer task.Unlock() old_state := task.State taskid := task.Id @@ -402,6 +495,8 @@ func (task *TaskRaw) SetState(new_state string) (err error) { if err != nil { return } + + logger.Debug(3, "(Task/SetState) %s new state: \"%s\" (old state \"%s\")", taskid, new_state, old_state) task.State = new_state if new_state == TASK_STAT_COMPLETED { @@ -425,6 +520,22 @@ func (task *TaskRaw) SetState(new_state string) (err error) { return } } + + //r, err := dbGetJobTaskString(jobid, taskid, "state") + //if err != nil { + // panic(err.Error()) + //} + //if r != new_state { + // text := fmt.Sprintf("did set: \"%s\" got: \"%s\"", new_state, r) + // panic(text) + //} + + //result_test, err := dbGetJobTask(jobid, taskid) + + //spew_config := spew.NewDefaultConfig() + //spew_config.DisableMethods = true + //spew_config.Dump(*result_test) + return } @@ -749,16 +860,25 @@ func (task *Task) setTokenForIO(writelock bool) (err error) { return } -func (task *Task) CreateWorkunits() (wus []*Workunit, err error) { +func (task *Task) CreateWorkunits(job *Job) (wus []*Workunit, err error) { //if a task contains only one workunit, assign rank 0 + if task.TotalWork == 1 { - workunit := NewWorkunit(task, 0) + workunit, xerr := NewWorkunit(task, 0, job) + if xerr != nil { + err = fmt.Errorf("(CreateWorkunits) NewWorkunit failed: %s", xerr.Error()) + return + } wus = append(wus, workunit) return } // if a task contains N (N>1) workunits, assign rank 1..N for i := 1; i <= task.TotalWork; i++ { - workunit := NewWorkunit(task, i) + workunit, xerr := NewWorkunit(task, i, job) + if xerr != nil { + err = fmt.Errorf("(CreateWorkunits) NewWorkunit failed: %s", xerr.Error()) + return + } wus = append(wus, workunit) } return @@ -770,11 +890,16 @@ func (task *Task) GetTaskLogs() (tlog *TaskLog) { tlog.State = task.State tlog.TotalWork = task.TotalWork tlog.CompletedDate = task.CompletedDate + + workunit_id := Workunit_Unique_Identifier{JobId: task.JobId, TaskId: task.Id} + if task.TotalWork == 1 { - tlog.Workunits = append(tlog.Workunits, NewWorkLog(task.Id, 0)) + workunit_id.Rank = 0 + tlog.Workunits = append(tlog.Workunits, NewWorkLog(workunit_id)) } else { for i := 1; i <= task.TotalWork; i++ { - tlog.Workunits = append(tlog.Workunits, NewWorkLog(task.Id, i)) + workunit_id.Rank = i + tlog.Workunits = append(tlog.Workunits, NewWorkLog(workunit_id)) } } return diff --git a/lib/core/taskmap.go b/lib/core/taskmap.go index c274a2e5..87eefbd1 100644 --- a/lib/core/taskmap.go +++ b/lib/core/taskmap.go @@ -1,13 +1,17 @@ package core +import ( + "fmt" +) + type TaskMap struct { RWMutex - _map map[string]*Task + _map map[Task_Unique_Identifier]*Task } func NewTaskMap() (t *TaskMap) { t = &TaskMap{ - _map: make(map[string]*Task), + _map: make(map[Task_Unique_Identifier]*Task), } t.RWMutex.Init("TaskMap") return t @@ -26,7 +30,7 @@ func (tm *TaskMap) Len() (length int, err error) { return } -func (tm *TaskMap) Get(taskid string, lock bool) (task *Task, ok bool, err error) { +func (tm *TaskMap) Get(taskid Task_Unique_Identifier, lock bool) (task *Task, ok bool, err error) { if lock { read_lock, xerr := tm.RLockNamed("Get") if xerr != nil { @@ -40,6 +44,20 @@ func (tm *TaskMap) Get(taskid string, lock bool) (task *Task, ok bool, err error return } +func (tm *TaskMap) Has(taskid Task_Unique_Identifier, lock bool) (ok bool, err error) { + if lock { + read_lock, xerr := tm.RLockNamed("Get") + if xerr != nil { + err = xerr + return + } + defer tm.RUnlockNamed(read_lock) + } + + _, ok = tm._map[taskid] + return +} + func (tm *TaskMap) GetTasks() (tasks []*Task, err error) { tasks = []*Task{} @@ -57,16 +75,34 @@ func (tm *TaskMap) GetTasks() (tasks []*Task, err error) { return } -func (tm *TaskMap) Delete(taskid string) (task *Task, ok bool) { +func (tm *TaskMap) Delete(taskid Task_Unique_Identifier) (task *Task, ok bool) { tm.LockNamed("Delete") defer tm.Unlock() delete(tm._map, taskid) // TODO should get write lock on task first return } -func (tm *TaskMap) Add(task *Task) { +func (tm *TaskMap) Add(task *Task) (err error) { tm.LockNamed("Add") defer tm.Unlock() - tm._map[task.Id] = task // TODO prevent overwriting + id, err := task.GetId() + if err != nil { + return + } + _, has_task := tm._map[id] + if has_task { + err = fmt.Errorf("(TaskMap/Add) task is already in TaskMap") + return + } + + task_state, _ := task.GetState() + if task_state == TASK_STAT_INIT { + err = task.SetState(TASK_STAT_PENDING, false) + if err != nil { + return + } + } + + tm._map[id] = task return } diff --git a/lib/core/utilproxy.go b/lib/core/utilproxy.go index 640881d2..a8aa32d6 100644 --- a/lib/core/utilproxy.go +++ b/lib/core/utilproxy.go @@ -36,7 +36,10 @@ func proxy_relay_workunit(work *Workunit, perfstat *WorkPerf) (err error) { logger.Event(event.WORK_RETURN, "workid="+work.Id) Self.Increment_total_failed(true) } - Self.Current_work_delete(work.Id, true) + err = Self.Current_work.Delete(work.Workunit_Unique_Identifier, true) + if err != nil { + return + } //delete(Self.Current_work, work.Id) return } diff --git a/lib/core/worklog.go b/lib/core/worklog.go new file mode 100644 index 00000000..fd5bc6c5 --- /dev/null +++ b/lib/core/worklog.go @@ -0,0 +1,25 @@ +package core + +import ( + "fmt" + "github.com/MG-RAST/AWE/lib/conf" +) + +type WorkLog struct { + Id string `bson:"wuid" json:"wuid"` // TODO change ! + Rank int `bson:"rank" json:"rank"` + Logs map[string]string `bson:"logs" json:"logs"` +} + +func NewWorkLog(id Workunit_Unique_Identifier) (wlog *WorkLog) { + work_id := fmt.Sprintf("%s_%d", id.TaskId, id.Rank) + wlog = new(WorkLog) + wlog.Id = work_id + wlog.Rank = id.Rank + wlog.Logs = map[string]string{} + for _, log := range conf.WORKUNIT_LOGS { + + wlog.Logs[log], _ = QMgr.GetReportMsg(id, log) + } + return +} diff --git a/lib/core/workqueue.go b/lib/core/workqueue.go index e2f1bfe5..4d491fc1 100644 --- a/lib/core/workqueue.go +++ b/lib/core/workqueue.go @@ -6,6 +6,7 @@ import ( "github.com/MG-RAST/AWE/lib/logger" "sort" //"sync" + "fmt" ) type WorkQueue struct { @@ -20,8 +21,8 @@ type WorkQueue struct { func NewWorkQueue() *WorkQueue { wq := &WorkQueue{ all: *NewWorkunitMap(), - Queue: *NewWorkunitMap(), - Checkout: *NewWorkunitMap(), + Queue: *NewWorkunitMap(), // these workunits that are ready to be checked out + Checkout: *NewWorkunitMap(), // workunits that are checked out right now Suspend: *NewWorkunitMap(), } @@ -49,14 +50,14 @@ func (wq *WorkQueue) Add(workunit *Workunit) (err error) { if err != nil { return } - err = wq.StatusChange("", workunit, WORK_STAT_QUEUED) + err = wq.StatusChange(Workunit_Unique_Identifier{}, workunit, WORK_STAT_QUEUED, "") if err != nil { return } return } -func (wq *WorkQueue) Get(id string) (w *Workunit, ok bool, err error) { +func (wq *WorkQueue) Get(id Workunit_Unique_Identifier) (w *Workunit, ok bool, err error) { w, ok, err = wq.all.Get(id) return } @@ -68,7 +69,8 @@ func (wq *WorkQueue) GetForJob(jobid string) (worklist []*Workunit, err error) { return } for _, work := range workunits { - parentid, _ := GetJobIdByWorkId(work.Id) + parentid := work.JobId + //parentid := , _ := GetJobIdByWorkId(work.Id) if jobid == parentid { worklist = append(worklist, work) } @@ -80,15 +82,15 @@ func (wq *WorkQueue) GetAll() (worklist []*Workunit, err error) { return wq.all.GetWorkunits() } -func (wq *WorkQueue) Clean() (workids []string) { +func (wq *WorkQueue) Clean() (workunits []*Workunit) { workunt_list, err := wq.all.GetWorkunits() if err != nil { return } for _, work := range workunt_list { - id := work.Id + id := work.Workunit_Unique_Identifier if work == nil || work.Info == nil { - workids = append(workids, id) + workunits = append(workunits, work) wq.Queue.Delete(id) wq.Checkout.Delete(id) wq.Suspend.Delete(id) @@ -99,7 +101,7 @@ func (wq *WorkQueue) Clean() (workids []string) { return } -func (wq *WorkQueue) Delete(id string) (err error) { +func (wq *WorkQueue) Delete(id Workunit_Unique_Identifier) (err error) { err = wq.Queue.Delete(id) if err != nil { return @@ -120,14 +122,14 @@ func (wq *WorkQueue) Delete(id string) (err error) { } -func (wq *WorkQueue) Has(id string) (has bool, err error) { +func (wq *WorkQueue) Has(id Workunit_Unique_Identifier) (has bool, err error) { _, has, err = wq.all.Get(id) return } //--------end of accessors------- -func (wq *WorkQueue) StatusChange(id string, workunit *Workunit, new_status string) (err error) { +func (wq *WorkQueue) StatusChange(id Workunit_Unique_Identifier, workunit *Workunit, new_status string, reason string) (err error) { //move workunit id between maps. no need to care about the old status because //delete function will do nothing if the operated map has no such key. @@ -138,7 +140,7 @@ func (wq *WorkQueue) StatusChange(id string, workunit *Workunit, new_status stri return } if !ok { - return errors.New("WQueue.statusChange: invalid workunit id:" + id) + return errors.New("WQueue.statusChange: invalid workunit id:" + id.String()) } } @@ -153,26 +155,44 @@ func (wq *WorkQueue) StatusChange(id string, workunit *Workunit, new_status stri case WORK_STAT_CHECKOUT: wq.Queue.Delete(id) wq.Suspend.Delete(id) - workunit.SetState(new_status) + err = workunit.SetState(new_status, reason) + if err != nil { + return + } + wq.Checkout.Set(workunit) case WORK_STAT_QUEUED: wq.Checkout.Delete(id) wq.Suspend.Delete(id) - workunit.SetState(new_status) + err = workunit.SetState(new_status, reason) + if err != nil { + return + } wq.Queue.Set(workunit) case WORK_STAT_SUSPEND: + + if reason == "" { + err = fmt.Errorf("suspend workunit only with reason!") + return + } + wq.Checkout.Delete(id) wq.Queue.Delete(id) - workunit.SetState(new_status) + err = workunit.SetState(new_status, reason) + if err != nil { + return + } wq.Suspend.Set(workunit) default: wq.Checkout.Delete(id) wq.Queue.Delete(id) wq.Suspend.Delete(id) - workunit.SetState(new_status) - + err = workunit.SetState(new_status, reason) + if err != nil { + return + } } return diff --git a/lib/core/workunit.go b/lib/core/workunit.go index 58da19f7..ae793f87 100644 --- a/lib/core/workunit.go +++ b/lib/core/workunit.go @@ -1,10 +1,20 @@ package core import ( + "bytes" + "encoding/json" "fmt" "github.com/MG-RAST/AWE/lib/conf" "github.com/MG-RAST/AWE/lib/core/cwl" + cwl_types "github.com/MG-RAST/AWE/lib/core/cwl/types" + "github.com/davecgh/go-spew/spew" + "github.com/robertkrimen/otto" + "gopkg.in/mgo.v2/bson" "os" + "path" + "reflect" + "regexp" + "regexp/syntax" "strings" "time" ) @@ -12,7 +22,7 @@ import ( const ( WORK_STAT_INIT = "init" // initial state WORK_STAT_QUEUED = "queued" // after requeue ; after failures below max ; on WorkQueue.Add() - WORK_STAT_RESERVED = "reserved" // short lived state between queued and checkout + WORK_STAT_RESERVED = "reserved" // short lived state between queued and checkout. when a worker checks the workunit out, the state is reserved. WORK_STAT_CHECKOUT = "checkout" // normal work checkout ; client registers that already has a workunit (e.g. after reboot of server) WORK_STAT_SUSPEND = "suspend" // on MAX_FAILURE ; on SuspendJob WORK_STAT_FAILED_PERMANENT = "failed-permanent" // app had exit code 42 @@ -25,141 +35,356 @@ const ( ) type Workunit struct { - Id string `bson:"wuid" json:"wuid"` - Info *Info `bson:"info" json:"info"` - Inputs []*IO `bson:"inputs" json:"inputs"` - Outputs []*IO `bson:"outputs" json:"outputs"` - Predata []*IO `bson:"predata" json:"predata"` - Cmd *Command `bson:"cmd" json:"cmd"` - Rank int `bson:"rank" json:"rank"` - TotalWork int `bson:"totalwork" json:"totalwork"` - Partition *PartInfo `bson:"part" json:"part"` - State string `bson:"state" json:"state"` - Failed int `bson:"failed" json:"failed"` - CheckoutTime time.Time `bson:"checkout_time" json:"checkout_time"` - Client string `bson:"client" json:"client"` - ComputeTime int `bson:"computetime" json:"computetime"` - ExitStatus int `bson:"exitstatus" json:"exitstatus"` // Linux Exit Status Code (0 is success) - Notes []string `bson:"notes" json:"notes"` - UserAttr map[string]string `bson:"userattr" json:"userattr"` - WorkPath string // this is the working directory. If empty, it will be computed. - WorkPerf *WorkPerf - CWL *CWL_workunit + Workunit_Unique_Identifier `bson:",inline" json:",inline" mapstructure:",squash"` + Id string `bson:"id,omitempty" json:"id,omitempty" mapstructure:"id,omitempty"` // global identifier: jobid_taskid_rank (for backwards coompatibility only) + WuId string `bson:"wuid,omitempty" json:"wuid,omitempty" mapstructure:"wuid,omitempty"` // deprecated ! + Info *Info `bson:"info,omitempty" json:"info,omitempty" mapstructure:"info,omitempty"` + Inputs []*IO `bson:"inputs,omitempty" json:"inputs,omitempty" mapstructure:"inputs,omitempty"` + Outputs []*IO `bson:"outputs,omitempty" json:"outputs,omitempty" mapstructure:"outputs,omitempty"` + Predata []*IO `bson:"predata,omitempty" json:"predata,omitempty" mapstructure:"predata,omitempty"` + Cmd *Command `bson:"cmd,omitempty" json:"cmd,omitempty" mapstructure:"cmd,omitempty"` + TotalWork int `bson:"totalwork,omitempty" json:"totalwork,omitempty" mapstructure:"totalwork,omitempty"` + Partition *PartInfo `bson:"part,omitempty" json:"part,omitempty" mapstructure:"part,omitempty"` + State string `bson:"state,omitempty" json:"state,omitempty" mapstructure:"state,omitempty"` + Failed int `bson:"failed,omitempty" json:"failed,omitempty" mapstructure:"failed,omitempty"` + CheckoutTime time.Time `bson:"checkout_time,omitempty" json:"checkout_time,omitempty" mapstructure:"checkout_time,omitempty"` + Client string `bson:"client,omitempty" json:"client,omitempty" mapstructure:"client,omitempty"` + ComputeTime int `bson:"computetime,omitempty" json:"computetime,omitempty" mapstructure:"computetime,omitempty"` + ExitStatus int `bson:"exitstatus,omitempty" json:"exitstatus,omitempty" mapstructure:"exitstatus,omitempty"` // Linux Exit Status Code (0 is success) + Notes []string `bson:"notes,omitempty" json:"notes,omitempty" mapstructure:"notes,omitempty"` + UserAttr map[string]string `bson:"userattr,omitempty" json:"userattr,omitempty" mapstructure:"userattr,omitempty"` + CWL_workunit *CWL_workunit `bson:"cwl,omitempty" json:"cwl,omitempty" mapstructure:"cwl,omitempty"` + WorkPath string // this is the working directory. If empty, it will be computed. + WorkPerf *WorkPerf } -type CWL_workunit struct { - Job_input *cwl.Job_document - Job_input_filename string - CWL_tool *cwl.CommandLineTool - CWL_tool_filename string - Tool_results *cwl.Job_document - OutputsExpected *cwl.WorkflowStepOutput // this is the subset of outputs that are needed by the workflow -} +func NewWorkunit(task *Task, rank int, job *Job) (workunit *Workunit, err error) { + + workunit = &Workunit{ + Workunit_Unique_Identifier: Workunit_Unique_Identifier{ + Rank: rank, + TaskId: task.Id, + JobId: task.JobId, + }, + Id: "defined below", + Cmd: task.Cmd, + //App: task.App, + Info: task.Info, + Inputs: task.Inputs, + Outputs: task.Outputs, + Predata: task.Predata, + + TotalWork: task.TotalWork, //keep this info in workunit for load balancing + Partition: task.Partition, + State: WORK_STAT_INIT, + Failed: 0, + UserAttr: task.UserAttr, + ExitStatus: -1, -func NewCWL_workunit() *CWL_workunit { - return &CWL_workunit{ - Job_input: nil, - CWL_tool: nil, - Tool_results: nil, - OutputsExpected: nil, + //AppVariables: task.AppVariables // not needed yet } -} + workunit.Id = workunit.String() + workunit.WuId = workunit.String() -type WorkLog struct { - Id string `bson:"wuid" json:"wuid"` - Rank int `bson:"rank" json:"rank"` - Logs map[string]string `bson:"logs" json:"logs"` -} + if task.WorkflowStep != nil { -func NewWorkLog(tid string, rank int) (wlog *WorkLog) { - wid := fmt.Sprintf("%s_%d", tid, rank) - wlog = new(WorkLog) - wlog.Id = wid - wlog.Rank = rank - wlog.Logs = map[string]string{} - for _, log := range conf.WORKUNIT_LOGS { - wlog.Logs[log], _ = QMgr.GetReportMsg(wid, log) - } - return -} + workflow_step := task.WorkflowStep -// create workunit slice type to use for sorting + workunit.CWL_workunit = &CWL_workunit{} -type WorkunitsSortby struct { - Order string - Direction string - Workunits []*Workunit -} + // ****** get CommandLineTool (or whatever can be executed) + p := workflow_step.Run -func (w WorkunitsSortby) Len() int { - return len(w.Workunits) -} + tool_name := "" -func (w WorkunitsSortby) Swap(i, j int) { - w.Workunits[i], w.Workunits[j] = w.Workunits[j], w.Workunits[i] -} + switch p.(type) { + case cwl.ProcessPointer: -func (w WorkunitsSortby) Less(i, j int) bool { - // default is ascending - if w.Direction == "desc" { - i, j = j, i - } - switch w.Order { - // default is info.submittime - default: - return w.Workunits[i].Info.SubmitTime.Before(w.Workunits[j].Info.SubmitTime) - case "wuid": - return w.Workunits[i].Id < w.Workunits[j].Id - case "client": - return w.Workunits[i].Client < w.Workunits[j].Client - case "info.submittime": - return w.Workunits[i].Info.SubmitTime.Before(w.Workunits[j].Info.SubmitTime) - case "checkout_time": - return w.Workunits[i].CheckoutTime.Before(w.Workunits[j].CheckoutTime) - case "info.name": - return w.Workunits[i].Info.Name < w.Workunits[j].Info.Name - case "cmd.name": - return w.Workunits[i].Cmd.Name < w.Workunits[j].Cmd.Name - case "rank": - return w.Workunits[i].Rank < w.Workunits[j].Rank - case "totalwork": - return w.Workunits[i].TotalWork < w.Workunits[j].TotalWork - case "state": - return w.Workunits[i].State < w.Workunits[j].State - case "failed": - return w.Workunits[i].Failed < w.Workunits[j].Failed - case "info.priority": - return w.Workunits[i].Info.Priority < w.Workunits[j].Info.Priority - } -} + pp, _ := p.(cwl.ProcessPointer) -func NewWorkunit(task *Task, rank int) *Workunit { + tool_name = pp.Value - return &Workunit{ - Id: fmt.Sprintf("%s_%d", task.Id, rank), - Cmd: task.Cmd, - //App: task.App, - Info: task.Info, - Inputs: task.Inputs, - Outputs: task.Outputs, - Predata: task.Predata, - Rank: rank, - TotalWork: task.TotalWork, //keep this info in workunit for load balancing - Partition: task.Partition, - State: WORK_STAT_INIT, - Failed: 0, - UserAttr: task.UserAttr, - ExitStatus: -1, + case bson.M: // I have no idea why we get a bson.M here + + p_bson := p.(bson.M) + + tool_name_interface, ok := p_bson["value"] + if !ok { + err = fmt.Errorf("(NewWorkunit) bson.M did not hold a field named value") + return + } + + tool_name, ok = tool_name_interface.(string) + if !ok { + err = fmt.Errorf("(NewWorkunit) bson.M value field is not a string") + return + } + + default: + err = fmt.Errorf("(NewWorkunit) Process type %s unknown, cannot create Workunit", reflect.TypeOf(p)) + return + + } + + if tool_name == "" { + err = fmt.Errorf("(NewWorkunit) No tool name found") + return + } + + if job.CWL_collection == nil { + err = fmt.Errorf("(NewWorkunit) job.CWL_collection == nil ") + return + } + + clt, xerr := job.CWL_collection.GetCommandLineTool(tool_name) + if xerr != nil { + err = fmt.Errorf("(NewWorkunit) Object %s not found in collection: %s", xerr.Error()) + return + } + clt.CwlVersion = job.CwlVersion + + if clt.CwlVersion == "" { + err = fmt.Errorf("(NewWorkunit) CommandLineTool misses CwlVersion") + return + } + workunit.CWL_workunit.CWL_tool = clt + + // ****** get inputs + job_input_map := *job.CWL_collection.Job_input_map + if job_input_map == nil { + err = fmt.Errorf("(NewWorkunit) job.CWL_collection.Job_input_map is empty") + return + } + //job_input_map := *job.CWL_collection.Job_input_map + + //job_input := *job.CWL_collection.Job_input + workunit_input_map := make(map[string]cwl_types.CWLType) + + spew.Dump(workflow_step.In) + for _, input := range workflow_step.In { + // input is a WorkflowStepInput + + id := input.Id + + cmd_id := path.Base(id) + + // get data from Source, Default or valueFrom + + if input.LinkMerge != nil { + err = fmt.Errorf("(NewWorkunit) sorry, LinkMergeMethod not supported yet") + return + } + + source_object_array := []cwl_types.CWLType{} + //resolve pointers in source + for _, src := range input.Source { + // src is a string, an id to another cwl object (workflow input of step output) + + src_base := path.Base(src) + + job_obj, ok := job_input_map[src_base] + if !ok { + fmt.Printf("%s not found in \n", src_base) + } else { + fmt.Printf("%s found in job_input!!!\n", src_base) + source_object_array = append(source_object_array, job_obj) + continue + } + + coll_obj, xerr := job.CWL_collection.GetType(src) + if xerr != nil { + fmt.Printf("%s not found in CWL_collection\n", src) + } else { + fmt.Printf("%s found in CWL_collection!!!\n", src) + source_object_array = append(source_object_array, coll_obj) + continue + } + //_ = coll_obj + + err = fmt.Errorf("Source object %s not found", src) + return + + } + + // input.Default The default value for this parameter to use if either there is no source field, or the value produced by the source is null. The default must be applied prior to scattering or evaluating valueFrom. + + if len(input.Source) == 1 { + workunit_input_map[cmd_id] = source_object_array[0] + continue + //object := source_object_array[0] + //fmt.Println("WORLD") + //spew.Dump(object) + + //file, ok := object.(*cwl_types.File) + //if ok { + // fmt.Println("A FILE") + // fmt.Printf("%+v\n", *file) + //file_id := file.GetId() + + //} else { + + //err = fmt.Errorf("(NewWorkunit) not implemented") + //return + // } + + } else if len(input.Source) > 1 { + cwl_array := cwl_types.Array{} + for _, obj := range source_object_array { + cwl_array.Add(obj) + } + workunit_input_map[cmd_id] = &cwl_array + continue + } else { + if input.Default != nil { + err = fmt.Errorf("(NewWorkunit) sorry, Default not supported yet") + return + } + } + + if input.ValueFrom != "" { + + // from CWL doc: The self value of in the parameter reference or expression must be the value of the parameter(s) specified in the source field, or null if there is no source field. + + vm := otto.New() + //TODO vm.Set("input", 11) + + if len(source_object_array) == 1 { + obj := source_object_array[0] + + vm.Set("self", obj.String()) + fmt.Printf("SET self=%s\n", input.Source[0]) + } else if len(input.Source) > 1 { + + source_b, xerr := json.Marshal(input.Source) + if xerr != nil { + err = fmt.Errorf("(NewWorkunit) cannot marshal source: %s", xerr.Error()) + return + } + vm.Set("self", string(source_b[:])) + fmt.Printf("SET self=%s\n", string(source_b[:])) + } + fmt.Printf("input.ValueFrom=%s\n", input.ValueFrom) + + // evaluate $(...) ECMAScript expression + reg := regexp.MustCompile(`\$\([\w.]+\)`) + // CWL documentation: http://www.commonwl.org/v1.0/Workflow.html#Expressions + + parsed := input.ValueFrom.String() + for { + + matches := reg.FindAll([]byte(parsed), -1) + fmt.Printf("Matches: %d\n", len(matches)) + if len(matches) == 0 { + break + } + for _, match := range matches { + expression_string := bytes.TrimPrefix(match, []byte("$(")) + expression_string = bytes.TrimSuffix(expression_string, []byte(")")) + + javascript_function := fmt.Sprintf("(function(){\n return %s;\n})()", expression_string) + fmt.Printf("%s\n", javascript_function) + + value, xerr := vm.Run(javascript_function) + if xerr != nil { + err = fmt.Errorf("Javascript complained: %s", xerr.Error()) + return + } + fmt.Println(reflect.TypeOf(value)) + + value_str, xerr := value.ToString() + if xerr != nil { + err = fmt.Errorf("Cannot convert value to string: %s", xerr) + return + } + parsed = strings.Replace(parsed, string(match), value_str, 1) + } + + } + + fmt.Printf("parsed: %s\n", parsed) + + // evaluate ${...} ECMAScript function body + reg = regexp.MustCompile(`\$\{[\w.]+\}`) + // CWL documentation: http://www.commonwl.org/v1.0/Workflow.html#Expressions + + //parsed = input.ValueFrom.String() + + for { + + matches := reg.FindAll([]byte(parsed), -1) + fmt.Printf("Matches: %d\n", len(matches)) + if len(matches) == 0 { + break + } + for _, match := range matches { + expression_string := bytes.TrimPrefix(match, []byte("${")) + expression_string = bytes.TrimSuffix(expression_string, []byte("}")) + + javascript_function := fmt.Sprintf("(function(){\n %s \n})()", expression_string) + fmt.Printf("%s\n", javascript_function) + + value, xerr := vm.Run(javascript_function) + if xerr != nil { + err = fmt.Errorf("Javascript complained: %s", xerr.Error()) + return + } + fmt.Println(reflect.TypeOf(value)) + + value_str, xerr := value.ToString() + if xerr != nil { + err = fmt.Errorf("Cannot convert value to string: %s", xerr) + return + } + parsed = strings.Replace(parsed, string(match), value_str, 1) + } + + } + + fmt.Printf("parsed: %s\n", parsed) + + new_string := cwl_types.NewString(id, parsed) + workunit_input_map[cmd_id] = new_string + continue + //job_input = append(job_input, new_string) + // TODO does this have to be storted in job_input ??? + + //err = fmt.Errorf("(NewWorkunit) sorry, ValueFrom not supported yet") + + } + + job_input := cwl.Job_document{} + + for _, elem := range workunit_input_map { + job_input = append(job_input, elem) + } + + workunit.CWL_workunit.Job_input = &job_input + //spew.Dump(job_input) + + } - //AppVariables: task.AppVariables // not needed yet } + //panic("done") + //spew.Dump(workunit.Cmd) + //panic("done") + + return +} + +func (w *Workunit) GetId() (id Workunit_Unique_Identifier) { + id = w.Workunit_Unique_Identifier + return } func (work *Workunit) Mkdir() (err error) { // delete workdir just in case it exists; will not work if awe-worker is not in docker container AND tasks are in container - os.RemoveAll(work.Path()) - err = os.MkdirAll(work.Path(), 0777) + work_path, err := work.Path() + if err != nil { + return + } + os.RemoveAll(work_path) + err = os.MkdirAll(work_path, 0777) if err != nil { return } @@ -167,30 +392,77 @@ func (work *Workunit) Mkdir() (err error) { } func (work *Workunit) RemoveDir() (err error) { - err = os.RemoveAll(work.Path()) + work_path, err := work.Path() + if err != nil { + return + } + err = os.RemoveAll(work_path) if err != nil { return } return } -func (work *Workunit) SetState(new_state string) { +func (work *Workunit) SetState(new_state string, reason string) (err error) { + + if new_state == WORK_STAT_SUSPEND && reason == "" { + err = fmt.Errorf("To suspend you need to provide a reason") + return + } + work.State = new_state if new_state != WORK_STAT_CHECKOUT { work.Client = "" } + + if reason != "" { + if len(work.Notes) == 0 { + work.Notes = append(work.Notes, reason) + } + } + + return } -func (work *Workunit) Path() string { +func (work *Workunit) Path() (path string, err error) { if work.WorkPath == "" { - id := work.Id - work.WorkPath = fmt.Sprintf("%s/%s/%s/%s/%s", conf.WORK_PATH, id[0:2], id[2:4], id[4:6], id) + id := work.Workunit_Unique_Identifier.JobId + + if id == "" { + err = fmt.Errorf("(Workunit/Path) JobId is missing") + return + } + + task_id_array := strings.Split(work.Workunit_Unique_Identifier.TaskId, "/") + task_name := "" + if len(task_id_array) > 1 { + task_name = strings.Join(task_id_array[1:], "/") + } else { + task_name = work.Workunit_Unique_Identifier.TaskId + } + + // convert name to make it filesystem compatible + task_name = strings.Map( + func(r rune) rune { + if syntax.IsWordChar(r) || r == '-' { // word char: [0-9A-Za-z] and '-' + return r + } + return '_' + }, + task_name) + + work.WorkPath = fmt.Sprintf("%s/%s/%s/%s/%s_%s_%d", conf.WORK_PATH, id[0:2], id[2:4], id[4:6], id, task_name, work.Workunit_Unique_Identifier.Rank) } - return work.WorkPath + path = work.WorkPath + return } func (work *Workunit) CDworkpath() (err error) { - return os.Chdir(work.Path()) + work_path, err := work.Path() + if err != nil { + return + } + return os.Chdir(work_path) } func (work *Workunit) GetNotes() string { diff --git a/lib/core/workunitUniqueIdentifier.go b/lib/core/workunitUniqueIdentifier.go new file mode 100644 index 00000000..c112f0dc --- /dev/null +++ b/lib/core/workunitUniqueIdentifier.go @@ -0,0 +1,45 @@ +package core + +import ( + "fmt" + "strconv" + "strings" +) + +type Workunit_Unique_Identifier struct { + Rank int `bson:"rank" json:"rank" mapstructure:"rank"` // this is the local identifier + TaskId string `bson:"taskid" json:"taskid" mapstructure:"taskid"` + JobId string `bson:"jobid" json:"jobid" mapstructure:"jobid"` +} + +func (w Workunit_Unique_Identifier) String() string { + return fmt.Sprintf("%s_%s_%d", w.JobId, w.TaskId, w.Rank) +} + +func (w Workunit_Unique_Identifier) GetTask() Task_Unique_Identifier { + return Task_Unique_Identifier{JobId: w.JobId, Id: w.TaskId} +} + +func New_Workunit_Unique_Identifier(old_style_id string) (w Workunit_Unique_Identifier, err error) { + + array := strings.Split(old_style_id, "_") + + if len(array) != 3 { + err = fmt.Errorf("Cannot parse workunit identifier: %s", old_style_id) + return + } + + rank, err := strconv.Atoi(array[2]) + if err != nil { + return + } + + if !IsValidUUID(array[0]) { + err = fmt.Errorf("Cannot parse workunit identifier, job id is not valid uuid: %s", old_style_id) + return + } + + w = Workunit_Unique_Identifier{JobId: array[0], TaskId: array[1], Rank: rank} + + return +} diff --git a/lib/core/workunitlist.go b/lib/core/workunitlist.go new file mode 100644 index 00000000..a6e74a5a --- /dev/null +++ b/lib/core/workunitlist.go @@ -0,0 +1,146 @@ +package core + +type WorkunitList struct { + RWMutex `bson:"-" json:"-"` + _map map[Workunit_Unique_Identifier]bool `json:"-"` + Data []string `json:"data"` +} + +func NewWorkunitList() *WorkunitList { + return &WorkunitList{_map: make(map[Workunit_Unique_Identifier]bool)} + +} + +func (this *WorkunitList) Init(name string) { + this.RWMutex.Init(name) + if this._map == nil { + this._map = make(map[Workunit_Unique_Identifier]bool) + } + if this.Data == nil { + this.Data = []string{} + } +} + +// lock always +func (cl *WorkunitList) Add(workid Workunit_Unique_Identifier) (err error) { + + err = cl.LockNamed("Add") + if err != nil { + return + } + defer cl.Unlock() + + cl._map[workid] = true + cl.sync() + + return +} + +func (cl *WorkunitList) Length(lock bool) (clength int, err error) { + if lock { + read_lock, xerr := cl.RLockNamed("Length") + if xerr != nil { + err = xerr + return + } + defer cl.RUnlockNamed(read_lock) + } + clength = len(cl.Data) + + return +} + +func (cl *WorkunitList) Delete(workid Workunit_Unique_Identifier, write_lock bool) (err error) { + if write_lock { + err = cl.LockNamed("Delete") + defer cl.Unlock() + } + delete(cl._map, workid) + cl.sync() + return +} + +func (cl *WorkunitList) Delete_all(workid string, write_lock bool) (err error) { + if write_lock { + err = cl.LockNamed("Delete_all") + defer cl.Unlock() + } + + cl._map = make(map[Workunit_Unique_Identifier]bool) + cl.sync() + + return +} + +func (cl *WorkunitList) Has(workid Workunit_Unique_Identifier) (ok bool, err error) { + + err = cl.LockNamed("Has") + defer cl.Unlock() + + _, ok = cl._map[workid] + + return +} + +func (cl *WorkunitList) Get_list(do_read_lock bool) (assigned_work_ids []Workunit_Unique_Identifier, err error) { + if do_read_lock { + read_lock, xerr := cl.RLockNamed("Get_list") + if xerr != nil { + err = xerr + return + } + defer cl.RUnlockNamed(read_lock) + } + assigned_work_ids = []Workunit_Unique_Identifier{} + for id, _ := range cl._map { + + assigned_work_ids = append(assigned_work_ids, id) + } + return +} + +func (cl *WorkunitList) Get_string_list(do_read_lock bool) (work_ids []string, err error) { + work_ids = []string{} + if do_read_lock { + read_lock, xerr := cl.RLockNamed("Get_string_list") + if xerr != nil { + err = xerr + return + } + defer cl.RUnlockNamed(read_lock) + } + for id, _ := range cl._map { + + work_ids = append(work_ids, id.String()) + } + return +} + +func (cl *WorkunitList) sync() (err error) { + + cl.Data = []string{} + for id, _ := range cl._map { + + id_string := id.String() + + cl.Data = append(cl.Data, id_string) + } + return +} + +// opposite of sync; take Data entries and copy them into map +func (cl *WorkunitList) FillMap() (err error) { + + for _, id_str := range cl.Data { + + id, xerr := New_Workunit_Unique_Identifier(id_str) + if xerr != nil { + err = xerr + return + } + + cl._map[id] = true + } + + return +} diff --git a/lib/core/workunitmap.go b/lib/core/workunitmap.go index 4a4897b2..28879dbc 100644 --- a/lib/core/workunitmap.go +++ b/lib/core/workunitmap.go @@ -2,11 +2,11 @@ package core type WorkunitMap struct { RWMutex - Map map[string]*Workunit + Map map[Workunit_Unique_Identifier]*Workunit } func NewWorkunitMap() *WorkunitMap { - return &WorkunitMap{Map: map[string]*Workunit{}} + return &WorkunitMap{Map: map[Workunit_Unique_Identifier]*Workunit{}} } func (wm *WorkunitMap) Len() (length int, err error) { @@ -25,11 +25,11 @@ func (wm *WorkunitMap) Set(workunit *Workunit) (err error) { return } defer wm.Unlock() - wm.Map[workunit.Id] = workunit + wm.Map[workunit.Workunit_Unique_Identifier] = workunit return } -func (wm *WorkunitMap) Get(id string) (workunit *Workunit, ok bool, err error) { +func (wm *WorkunitMap) Get(id Workunit_Unique_Identifier) (workunit *Workunit, ok bool, err error) { rlock, err := wm.RLockNamed("WorkunitMap/Get") if err != nil { return @@ -53,7 +53,7 @@ func (wm *WorkunitMap) GetWorkunits() (workunits []*Workunit, err error) { return } -func (wm *WorkunitMap) Delete(id string) (err error) { +func (wm *WorkunitMap) Delete(id Workunit_Unique_Identifier) (err error) { err = wm.LockNamed("WorkunitMap/Delete") if err != nil { return diff --git a/lib/core/workunitsSortby.go b/lib/core/workunitsSortby.go new file mode 100644 index 00000000..55d7df36 --- /dev/null +++ b/lib/core/workunitsSortby.go @@ -0,0 +1,51 @@ +package core + +// create workunit slice type to use for sorting + +type WorkunitsSortby struct { + Order string + Direction string + Workunits []*Workunit +} + +func (w WorkunitsSortby) Len() int { + return len(w.Workunits) +} + +func (w WorkunitsSortby) Swap(i, j int) { + w.Workunits[i], w.Workunits[j] = w.Workunits[j], w.Workunits[i] +} + +func (w WorkunitsSortby) Less(i, j int) bool { + // default is ascending + if w.Direction == "desc" { + i, j = j, i + } + switch w.Order { + // default is info.submittime + default: + return w.Workunits[i].Info.SubmitTime.Before(w.Workunits[j].Info.SubmitTime) + case "wuid": + return w.Workunits[i].Id < w.Workunits[j].Id + case "client": + return w.Workunits[i].Client < w.Workunits[j].Client + case "info.submittime": + return w.Workunits[i].Info.SubmitTime.Before(w.Workunits[j].Info.SubmitTime) + case "checkout_time": + return w.Workunits[i].CheckoutTime.Before(w.Workunits[j].CheckoutTime) + case "info.name": + return w.Workunits[i].Info.Name < w.Workunits[j].Info.Name + case "cmd.name": + return w.Workunits[i].Cmd.Name < w.Workunits[j].Cmd.Name + case "rank": + return w.Workunits[i].Rank < w.Workunits[j].Rank + case "totalwork": + return w.Workunits[i].TotalWork < w.Workunits[j].TotalWork + case "state": + return w.Workunits[i].State < w.Workunits[j].State + case "failed": + return w.Workunits[i].Failed < w.Workunits[j].Failed + case "info.priority": + return w.Workunits[i].Info.Priority < w.Workunits[j].Info.Priority + } +} diff --git a/lib/errors/errors.go b/lib/errors/errors.go index e9a7d2e0..32d9e3f5 100644 --- a/lib/errors/errors.go +++ b/lib/errors/errors.go @@ -25,4 +25,5 @@ const ( QueueFull = "Server queue is full" QueueSuspend = "Server queue is suspended" UnAuth = "User Unauthorized" + ServerNotFound = "Server not found" ) diff --git a/lib/shock/shock.go b/lib/shock/shock.go index ab4af787..17eecdb9 100644 --- a/lib/shock/shock.go +++ b/lib/shock/shock.go @@ -76,10 +76,10 @@ type shockFile struct { type IdxInfo struct { //Type string `bson:"index_type" json:"-"` - TotalUnits int64 `bson:"total_units" json:"total_units"` - AvgUnitSize int64 `bson:"average_unit_size" json:"average_unit_size"` + TotalUnits int64 `bson:"total_units" json:"total_units" mapstructure:"total_units"` + AvgUnitSize int64 `bson:"average_unit_size" json:"average_unit_size" mapstructure:"average_unit_size"` //Format string `bson:"format" json:"-"` - CreatedOn time.Time `bson:"created_on" json:"created_on"` + CreatedOn time.Time `bson:"created_on" json:"created_on" mapstructure:"created_on"` } type linkage struct { diff --git a/lib/worker/dataMover.go b/lib/worker/dataDownloader.go similarity index 72% rename from lib/worker/dataMover.go rename to lib/worker/dataDownloader.go index f37af4d3..c7c01f4d 100644 --- a/lib/worker/dataMover.go +++ b/lib/worker/dataDownloader.go @@ -14,7 +14,8 @@ import ( "github.com/MG-RAST/AWE/lib/logger/event" "github.com/MG-RAST/AWE/lib/shock" "github.com/MG-RAST/golib/httpclient" - //"github.com/davecgh/go-spew/spew" + "github.com/davecgh/go-spew/spew" + "gopkg.in/yaml.v2" "io" "io/ioutil" "os" @@ -174,161 +175,179 @@ func prepareAppTask(parsed *Mediumwork, work *core.Workunit) (err error) { return } -func dataMover(control chan int) { - //var err error - fmt.Printf("dataMover launched, client=%s\n", core.Self.Id) - logger.Debug(1, "dataMover launched, client=%s\n", core.Self.Id) +func downloadWorkunitData(workunit *core.Workunit) (err error) { + work_id := workunit.Workunit_Unique_Identifier - defer fmt.Printf("dataMover exiting...\n") - for { - workunit := <-FromStealer + work_path, err := workunit.Path() + if err != nil { + return + } + workmap.Set(work_id, ID_DATADOWNLOADER, "dataDownloader") - logger.Debug(3, "(dataMover) received some work") - //parsed := &Mediumwork{ - // Workunit: raw.Workunit, - // Perfstat: raw.Perfstat, - // CWL_job: raw.CWL_job, - // CWL_tool: raw.CWL_tool, - //} - //work := raw.Workunit + if Client_mode == "online" { + //make a working directory for the workunit (not for commandline execution !!!!!!) + err = workunit.Mkdir() + if err != nil { + logger.Error("[dataDownloader#workunit.Mkdir], workid=" + work_id.String() + " error=" + err.Error()) + workunit.Notes = append(workunit.Notes, "[dataDownloader#work.Mkdir]"+err.Error()) + workunit.SetState(core.WORK_STAT_ERROR, "see notes") + //hand the parsed workunit to next stage and continue to get new workunit to process + return + } - workmap.Set(workunit.Id, ID_DATAMOVER, "dataMover") + //run the PreWorkExecutionScript + err = runPreWorkExecutionScript(workunit) + if err != nil { + logger.Error("[dataDownloader#runPreWorkExecutionScript], workid=" + work_id.String() + " error=" + err.Error()) + workunit.Notes = append(workunit.Notes, "[dataDownloader#runPreWorkExecutionScript]"+err.Error()) + workunit.SetState(core.WORK_STAT_ERROR, "see notes") + //hand the parsed workunit to next stage and continue to get new workunit to process + return + } - if conf.CWL_TOOL == "" { - //make a working directory for the workunit (not for commandline execution !!!!!!) - if err := workunit.Mkdir(); err != nil { - logger.Error("[dataMover#workunit.Mkdir], workid=" + workunit.Id + " error=" + err.Error()) - workunit.Notes = append(workunit.Notes, "[dataMover#work.Mkdir]"+err.Error()) - workunit.SetState(core.WORK_STAT_ERROR) - //hand the parsed workunit to next stage and continue to get new workunit to process - fromMover <- workunit - continue + //check the availability prerequisite data and download if needed + predatamove_start := time.Now().UnixNano() + moved_data, xerr := movePreData(workunit) + if xerr != nil { + err = xerr + logger.Error("[dataDownloader#movePreData], workid=" + work_id.String() + " error=" + err.Error()) + workunit.Notes = append(workunit.Notes, "[dataDownloader#movePreData]"+err.Error()) + workunit.SetState(core.WORK_STAT_ERROR, "see notes") + //hand the parsed workunit to next stage and continue to get new workunit to process + return + } else { + if moved_data > 0 { + workunit.WorkPerf.PreDataSize = moved_data + predatamove_end := time.Now().UnixNano() + workunit.WorkPerf.PreDataIn = float64(predatamove_end-predatamove_start) / 1e9 } } - if conf.CWL_TOOL == "" { - //run the PreWorkExecutionScript - if err := runPreWorkExecutionScript(workunit); err != nil { - logger.Error("[dataMover#runPreWorkExecutionScript], workid=" + workunit.Id + " error=" + err.Error()) - workunit.Notes = append(workunit.Notes, "[dataMover#runPreWorkExecutionScript]"+err.Error()) - workunit.SetState(core.WORK_STAT_ERROR) - //hand the parsed workunit to next stage and continue to get new workunit to process - fromMover <- workunit - continue + + if workunit.CWL_workunit != nil { + + job_input := workunit.CWL_workunit.Job_input + cwl_tool := workunit.CWL_workunit.CWL_tool + job_input_filename := path.Join(work_path, "cwl_job_input.yaml") + cwl_tool_filename := path.Join(work_path, "cwl_tool.yaml") + + if job_input == nil { + err = fmt.Errorf("Job_input is empty") + return } - } - if conf.CWL_TOOL == "" { - //check the availability prerequisite data and download if needed - predatamove_start := time.Now().UnixNano() - if moved_data, err := movePreData(workunit); err != nil { - logger.Error("[dataMover#movePreData], workid=" + workunit.Id + " error=" + err.Error()) - workunit.Notes = append(workunit.Notes, "[dataMover#movePreData]"+err.Error()) - workunit.SetState(core.WORK_STAT_ERROR) - //hand the parsed workunit to next stage and continue to get new workunit to process - fromMover <- workunit - continue - } else { - if moved_data > 0 { - workunit.WorkPerf.PreDataSize = moved_data - predatamove_end := time.Now().UnixNano() - workunit.WorkPerf.PreDataIn = float64(predatamove_end-predatamove_start) / 1e9 - } + if cwl_tool == nil { + err = fmt.Errorf("CWL_tool is empty") + return + } + + // create job_input file + var job_input_bytes []byte + job_input_bytes, err = yaml.Marshal(*job_input) + if err != nil { + return + } + + err = ioutil.WriteFile(job_input_filename, job_input_bytes, 0644) + if err != nil { + return + } + + // create cwt_tool file + var cwl_tool_bytes []byte + cwl_tool_bytes, err = yaml.Marshal(*cwl_tool) + if err != nil { + return + } + + err = ioutil.WriteFile(cwl_tool_filename, cwl_tool_bytes, 0644) + if err != nil { + return } + } - //parse the args, replacing @input_name to local file path (file not downloaded yet) + } + + //parse the args, replacing @input_name to local file path (file not downloaded yet) + + err = ParseWorkunitArgs(workunit) + if err != nil { + err = fmt.Errorf("err@dataDownloader.ParseWorkunitArgs, workid=%s error=%s", work_id.String(), err.Error()) + workunit.Notes = append(workunit.Notes, "[dataDownloader#ParseWorkunitArgs]"+err.Error()) + workunit.SetState(core.WORK_STAT_ERROR, "see notes") + //hand the parsed workunit to next stage and continue to get new workunit to process + return + } + + //download input data + if Client_mode == "online" { - if err := ParseWorkunitArgs(workunit); err != nil { - logger.Error("err@dataMover.ParseWorkunitArgs, workid=" + workunit.Id + " error=" + err.Error()) - workunit.Notes = append(workunit.Notes, "[dataMover#ParseWorkunitArgs]"+err.Error()) - workunit.SetState(core.WORK_STAT_ERROR) + datamove_start := time.Now().UnixNano() + moved_data, xerr := cache.MoveInputData(workunit) + if xerr != nil { + err = fmt.Errorf("(dataDownloader) workid=%s error=%s", work_id.String(), xerr.Error()) + workunit.Notes = append(workunit.Notes, "[dataDownloader#MoveInputData]"+err.Error()) + workunit.SetState(core.WORK_STAT_ERROR, "see notes") //hand the parsed workunit to next stage and continue to get new workunit to process - fromMover <- workunit - continue + return + } else { + workunit.WorkPerf.InFileSize = moved_data + datamove_end := time.Now().UnixNano() + workunit.WorkPerf.DataIn = float64(datamove_end-datamove_start) / 1e9 } + } - //download input data - if conf.CWL_TOOL == "" { + if Client_mode == "online" { - datamove_start := time.Now().UnixNano() - if moved_data, err := cache.MoveInputData(workunit); err != nil { - logger.Error("err@dataMover.MoveInputData, workid=" + workunit.Id + " error=" + err.Error()) - workunit.Notes = append(workunit.Notes, "[dataMover#MoveInputData]"+err.Error()) - workunit.SetState(core.WORK_STAT_ERROR) + //create userattr.json + var work_path string + work_path, err = workunit.Path() + if err != nil { + return + } + user_attr := getUserAttr(workunit) + if len(user_attr) > 0 { + attr_json, _ := json.Marshal(user_attr) + err = ioutil.WriteFile(fmt.Sprintf("%s/userattr.json", work_path), attr_json, 0644) + if err != nil { + err = fmt.Errorf("err@dataDownloader_work.getUserAttr, workid=%s error=%s", work_id.String(), err.Error()) + workunit.Notes = append(workunit.Notes, "[dataDownloader#getUserAttr]"+err.Error()) + workunit.SetState(core.WORK_STAT_ERROR, "see notes") //hand the parsed workunit to next stage and continue to get new workunit to process - fromMover <- workunit - continue - } else { - workunit.WorkPerf.InFileSize = moved_data - datamove_end := time.Now().UnixNano() - workunit.WorkPerf.DataIn = float64(datamove_end-datamove_start) / 1e9 + return } - } else { - // download required remote files (local files are ok, mostly for commandline execution) - // job_doc := workunit.CWL.Job_input - // for name, thing := range *job_doc { - // - // array, is_array := thing.(cwl_types.CWL_array_type) - // - // fmt.Println(name) - // if is_array { - // - // for _, element := range *array.Get_Array() { - // switch element.(type) { - // case *cwl_types.File: - // file, ok := element.(*cwl_types.File) - // if !ok { - // panic("not file") - // } - // fmt.Printf("%+v\n", *file) - // - // if file.Location != "" { // this is an IRI (URI) - // // TODO: do something !!! download - // - // } - // - // if file.Path != "" { - // - // logger.Debug(1, "(dataMover) checking file %s=%s...", name, file.Path) - // - // if _, err := os.Stat(file.Path); os.IsNotExist(err) { - // logger.Error("(dataMover) file %s=%s not found", name, file.Path) - // workunit.SetState(core.WORK_STAT_ERROR) - // //hand the parsed workunit to next stage and continue to get new workunit to process - // fromMover <- workunit - // continue - // } else { - // logger.Debug(1, "(dataMover) ok, found file %s=%s...", name, file.Path) - // } - // } - // default: - // spew.Dump(element) - // } - // } - // } - // } } + } + return +} - if conf.CWL_TOOL == "" { - - //create userattr.json - user_attr := getUserAttr(workunit) - if len(user_attr) > 0 { - attr_json, _ := json.Marshal(user_attr) - if err := ioutil.WriteFile(fmt.Sprintf("%s/userattr.json", workunit.Path()), attr_json, 0644); err != nil { - logger.Error("err@dataMover_work.getUserAttr, workid=" + workunit.Id + " error=" + err.Error()) - workunit.Notes = append(workunit.Notes, "[dataMover#getUserAttr]"+err.Error()) - workunit.SetState(core.WORK_STAT_ERROR) - //hand the parsed workunit to next stage and continue to get new workunit to process - fromMover <- workunit - continue - } - } - } +func dataDownloader(control chan int) { + //var err error + fmt.Printf("dataDownloader launched, client=%s\n", core.Self.Id) + logger.Debug(1, "dataDownloader launched, client=%s\n", core.Self.Id) + + defer fmt.Printf("dataDownloader exiting...\n") + for { + workunit := <-FromStealer + logger.Debug(3, "(dataDownloader) received some work") + + err := downloadWorkunitData(workunit) + if err != nil { + logger.Error("(dataDownloader) downloadWorkunitData returned: %s", err.Error()) + } + //parsed := &Mediumwork{ + // Workunit: raw.Workunit, + // Perfstat: raw.Perfstat, + // CWL_job: raw.CWL_job, + // CWL_tool: raw.CWL_tool, + //} + //work := raw.Workunit + spew.Dump(workunit.Cmd) + //panic("done") fromMover <- workunit } - control <- ID_DATAMOVER //we are ending + control <- ID_DATADOWNLOADER //we are ending } func proxyDataMover(control chan int) { @@ -337,23 +356,27 @@ func proxyDataMover(control chan int) { for { workunit := <-FromStealer - - workmap.Set(workunit.Id, ID_DATAMOVER, "proxyDataMover") + work_id := workunit.Workunit_Unique_Identifier + workmap.Set(work_id, ID_DATADOWNLOADER, "proxyDataMover") //check the availability prerequisite data and download if needed if err := proxyMovePreData(workunit); err != nil { - logger.Error("err@dataMover_work.movePreData, workid=" + workunit.Id + " error=" + err.Error()) - workunit.Notes = append(workunit.Notes, "[dataMover#proxyMovePreData]"+err.Error()) - workunit.SetState(core.WORK_STAT_ERROR) + logger.Error("err@dataDownloader_work.movePreData, workid=" + work_id.String() + " error=" + err.Error()) + workunit.Notes = append(workunit.Notes, "[dataDownloader#proxyMovePreData]"+err.Error()) + workunit.SetState(core.WORK_STAT_ERROR, "see notes") } fromMover <- workunit } - control <- ID_DATAMOVER + control <- ID_DATADOWNLOADER } //parse workunit, fetch input data, compose command arguments func ParseWorkunitArgs(work *core.Workunit) (err error) { - workpath := work.Path() + workpath, err := work.Path() + if err != nil { + return + } + if work.Cmd.Dockerimage != "" || work.Cmd.DockerPull != "" { workpath = conf.DOCKER_WORK_DIR } @@ -424,7 +447,7 @@ func ParseWorkunitArgs(work *core.Workunit) (err error) { work.Cmd.ParsedArgs = args logger.Debug(1, "work.Cmd.ParsedArgs: %v (%d)", work.Cmd.ParsedArgs, len(work.Cmd.ParsedArgs)) - work.SetState(core.WORK_STAT_PREPARED) + work.SetState(core.WORK_STAT_PREPARED, "") return nil } @@ -542,7 +565,13 @@ func movePreData(workunit *core.Workunit) (size int64, err error) { } // copy or create symlink in work dir - linkname := path.Join(workunit.Path(), name) + work_path, xerr := workunit.Path() + if xerr != nil { + + return 0, xerr + } + + linkname := path.Join(work_path, name) if conf.NO_SYMLINK { // some programs do not accept symlinks (e.g. emirge), need to copy the file into the work directory logger.Debug(1, "copy predata: "+file_path+" -> "+linkname) diff --git a/lib/worker/deliverer.go b/lib/worker/deliverer.go index 5fcddd6a..75fac417 100644 --- a/lib/worker/deliverer.go +++ b/lib/worker/deliverer.go @@ -35,7 +35,7 @@ func deliverer_run(control chan int) { return } - work_id := workunit.Id + work_id := workunit.Workunit_Unique_Identifier work_state, ok, err := workmap.Get(work_id) if err != nil { logger.Error("error: %s", err.Error()) @@ -47,8 +47,8 @@ func deliverer_run(control chan int) { } if work_state == ID_DISCARDED { - workunit.SetState(core.WORK_STAT_DISCARDED) - logger.Event(event.WORK_DISCARD, "workid="+work_id) + workunit.SetState(core.WORK_STAT_DISCARDED, "workmap indicated discarded") + logger.Event(event.WORK_DISCARD, "workid="+work_id.String()) } else { workmap.Set(work_id, ID_DELIVERER, "deliverer") perfstat := workunit.WorkPerf @@ -59,11 +59,11 @@ func deliverer_run(control chan int) { if workunit.State == core.WORK_STAT_COMPUTED { data_moved, err := cache.UploadOutputData(workunit) if err != nil { - workunit.SetState(core.WORK_STAT_ERROR) - logger.Error("[deliverer#UploadOutputData]workid=" + work_id + ", err=" + err.Error()) + workunit.SetState(core.WORK_STAT_ERROR, "UploadOutputData failed") + logger.Error("[deliverer#UploadOutputData]workid=" + work_id.String() + ", err=" + err.Error()) workunit.Notes = append(workunit.Notes, "[deliverer#UploadOutputData]"+err.Error()) } else { - workunit.SetState(core.WORK_STAT_DONE) + workunit.SetState(core.WORK_STAT_DONE, "") perfstat.OutFileSize = data_moved } } @@ -80,23 +80,23 @@ func deliverer_run(control chan int) { for do_retry { response, err := core.NotifyWorkunitProcessedWithLogs(workunit, perfstat, conf.PRINT_APP_MSG) if err != nil { - logger.Error("[deliverer: workid=" + work_id + ", err=" + err.Error()) + logger.Error("[deliverer: workid=" + work_id.String() + ", err=" + err.Error()) workunit.Notes = append(workunit.Notes, "[deliverer]"+err.Error()) // keep retry } else { - error_message := strings.Join(response.E, ",") + error_message := strings.Join(response.Error, ",") if strings.Contains(error_message, e.ClientNotFound) { // TODO need better method than string search. Maybe a field awe_status. //mark this work in Current_work map as false, something needs to be done in the future //to clean this kind of work that has been proccessed but its result can't be sent to server! //core.Self.Current_work_false(work.Id) //server doesn't know this yet do_retry = false } - if response.S == http.StatusOK { + if response.Status == http.StatusOK { // success, work delivered logger.Debug(1, "work delivered successfully") do_retry = false } else { - logger.Error("deliverer: workid=%s, err=%s", work_id, error_message) + logger.Error("deliverer: workid=%s, err=%s", work_id.String(), error_message) } } @@ -112,31 +112,37 @@ func deliverer_run(control chan int) { } } + work_path, err := workunit.Path() + if err != nil { + return + } + // now final status report sent to server, update some local info if workunit.State == core.WORK_STAT_DONE { - logger.Event(event.WORK_DONE, "workid="+work_id) + logger.Event(event.WORK_DONE, "workid="+work_id.String()) core.Self.Increment_total_completed() if conf.AUTO_CLEAN_DIR && workunit.Cmd.Local == false { - go removeDirLater(workunit.Path(), conf.CLIEN_DIR_DELAY_DONE) + go removeDirLater(work_path, conf.CLIEN_DIR_DELAY_DONE) } } else { if workunit.State == core.WORK_STAT_DISCARDED { - logger.Event(event.WORK_DISCARD, "workid="+work_id) + logger.Event(event.WORK_DISCARD, "workid="+work_id.String()) } else { - logger.Event(event.WORK_RETURN, "workid="+work_id) + logger.Event(event.WORK_RETURN, "workid="+work_id.String()) } core.Self.Increment_total_failed(true) if conf.AUTO_CLEAN_DIR && workunit.Cmd.Local == false { - go removeDirLater(workunit.Path(), conf.CLIEN_DIR_DELAY_FAIL) + go removeDirLater(work_path, conf.CLIEN_DIR_DELAY_FAIL) } } // cleanup - err = core.Self.Current_work_delete(work_id, true) + err = core.Self.Current_work.Delete(work_id, true) if err != nil { logger.Error("Could not remove work_id %s", work_id) } workmap.Delete(work_id) + core.Self.Busy = false } func removeDirLater(path string, duration time.Duration) (err error) { diff --git a/lib/worker/heartbeater.go b/lib/worker/heartbeater.go index c52708cf..e4dd71c4 100644 --- a/lib/worker/heartbeater.go +++ b/lib/worker/heartbeater.go @@ -24,9 +24,9 @@ import ( ) type HeartbeatResponse struct { - Code int `bson:"status" json:"status"` - Data core.HBmsg `bson:"data" json:"data"` - Errs []string `bson:"error" json:"error"` + Code int `bson:"status" json:"status"` + Data core.HeartbeatInstructions `bson:"data" json:"data"` + Errs []string `bson:"error" json:"error"` } type ClientResponse struct { @@ -42,7 +42,10 @@ func heartBeater(control chan int) { for { time.Sleep(10 * time.Second) - SendHeartBeat() + err := SendHeartBeat() + if err != nil { + logger.Error("SendHeartBeat returned: %s", err.Error()) + } } control <- 2 //we are ending } @@ -66,13 +69,19 @@ type Openstack_Metadata_meta struct { } //client sends heartbeat to server to maintain active status and re-register when needed -func SendHeartBeat() { +func SendHeartBeat() (err error) { hbmsg, err := heartbeating(conf.SERVER_URL, core.Self.Id) if err != nil { - if err.Error() == e.ClientNotFound { - ReRegisterWithSelf(conf.SERVER_URL) - } logger.Debug(3, "(SendHeartBeat) heartbeat returned error: "+err.Error()) + if strings.Contains(err.Error(), e.ClientNotFound) { + logger.Debug(3, "(SendHeartBeat) invoke ReRegisterWithSelf: ") + xerr := ReRegisterWithSelf(conf.SERVER_URL) + if xerr != nil { + err = fmt.Errorf("(SendHeartBeat) needed to register, but that failed: %s", xerr.Error()) + return + } + } + } val, ok := hbmsg["server-uuid"] @@ -90,6 +99,7 @@ func SendHeartBeat() { for _, work := range all_work { DiscardWorkunit(work) } + core.Self.Busy = false core.Server_UUID = val } } @@ -99,12 +109,17 @@ func SendHeartBeat() { } } - //handle requested ops from the server + //handle requested ops from the server (HeartbeatInstructions) for op, objs := range hbmsg { if op == "discard" { //discard suspended workunits suspendedworks := strings.Split(objs, ",") for _, work := range suspendedworks { - DiscardWorkunit(work) + work_id, xerr := core.New_Workunit_Unique_Identifier(work) + if xerr != nil { + err = xerr + return + } + DiscardWorkunit(work_id) } } else if op == "restart" { RestartClient() @@ -114,34 +129,50 @@ func SendHeartBeat() { CleanDisk() } } + return } -func heartbeating(host string, clientid string) (msg core.HBmsg, err error) { +func heartbeating(host string, clientid string) (msg core.HeartbeatInstructions, err error) { response := new(HeartbeatResponse) targeturl := fmt.Sprintf("%s/client/%s?heartbeat", host, clientid) //res, err := http.Get(targeturl) - var headers httpclient.Header + + worker_state_b, err := json.Marshal(core.Self.WorkerState) + if err != nil { + err = fmt.Errorf("(heartbeating) json.Marshal failed: %s", err.Error()) + return + } + + headers := httpclient.Header{"Content-Type": []string{"application/json"}} + if conf.CLIENT_GROUP_TOKEN != "" { - headers = httpclient.Header{ - "Authorization": []string{"CG_TOKEN " + conf.CLIENT_GROUP_TOKEN}, - } + headers["Authorization"] = []string{"CG_TOKEN " + conf.CLIENT_GROUP_TOKEN} } - res, err := httpclient.Get(targeturl, headers, nil, nil) - logger.Debug(3, fmt.Sprintf("client %s sent a heartbeat to %s", host, clientid)) + + res, err := httpclient.Put(targeturl, headers, bytes.NewBuffer(worker_state_b), nil) if err != nil { + err = fmt.Errorf("(heartbeating) httpclient.Put failed: %s", err.Error()) return } + logger.Debug(3, fmt.Sprintf("client %s sent a heartbeat to %s", host, clientid)) + defer res.Body.Close() jsonstream, err := ioutil.ReadAll(res.Body) if err != nil { + err = fmt.Errorf("(heartbeating) ioutil.ReadAll failed: %s", err.Error()) return } - if err = json.Unmarshal(jsonstream, response); err == nil { - if len(response.Errs) > 0 { - return msg, errors.New(strings.Join(response.Errs, ",")) - } - return response.Data, nil + err = json.Unmarshal(jsonstream, response) + if err != nil { + err = fmt.Errorf("(heartbeating) json.Unmarshal response failed: %s", err.Error()) + return + } + + if len(response.Errs) > 0 { + err = fmt.Errorf("(heartbeating) errors in response: %s ", strings.Join(response.Errs, ",")) + return } + msg = response.Data return } @@ -192,10 +223,10 @@ func RegisterWithProfile(host string, profile *core.Client) (client *core.Client } // invoked on start of AWE worker AND on ReRegisterWithSelf -func RegisterWithAuth(host string, pclient *core.Client) (client *core.Client, err error) { +func RegisterWithAuth(host string, pclient *core.Client) (err error) { logger.Debug(3, "Try to register client") if conf.CLIENT_GROUP_TOKEN == "" { - fmt.Println("clientgroup token not set, register as a public client (can only access public data)") + logger.Info("(RegisterWithAuth) clientgroup token not set, register as a public client (can only access public data)") } //serialize profile @@ -207,9 +238,9 @@ func RegisterWithAuth(host string, pclient *core.Client) (client *core.Client, e } // write profile to file - logger.Debug(3, "client_jsonstream: %s ", string(client_jsonstream)) + logger.Debug(3, "(RegisterWithAuth) client_jsonstream: %s ", string(client_jsonstream)) profile_path := conf.DATA_PATH + "/clientprofile.json" - logger.Debug(3, "profile_path: %s", profile_path) + logger.Debug(3, "(RegisterWithAuth) profile_path: %s", profile_path) err = ioutil.WriteFile(profile_path, []byte(client_jsonstream), 0644) if err != nil { err = fmt.Errorf("(RegisterWithAuth) error in ioutil.WriteFile: %s", err.Error()) @@ -220,7 +251,7 @@ func RegisterWithAuth(host string, pclient *core.Client) (client *core.Client, e form := httpclient.NewForm() form.AddFile("profile", profile_path) if err = form.Create(); err != nil { - err = fmt.Errorf("form.Create() error: %s", err.Error()) + err = fmt.Errorf("(RegisterWithAuth) form.Create() error: %s", err.Error()) return } var headers httpclient.Header @@ -243,36 +274,37 @@ func RegisterWithAuth(host string, pclient *core.Client) (client *core.Client, e resp, err := httpclient.DoTimeout("POST", targetUrl, headers, form.Reader, nil, time.Second*10) if err != nil { - err = fmt.Errorf("POST %s error: %s", targetUrl, err.Error()) + err = fmt.Errorf("(RegisterWithAuth) POST %s error: %s", targetUrl, err.Error()) return } defer resp.Body.Close() // evaluate response response := new(ClientResponse) - logger.Debug(3, "client registration: got response") + logger.Debug(3, "(RegisterWithAuth) client registration: got response") jsonstream, err := ioutil.ReadAll(resp.Body) + logger.Debug(3, "(RegisterWithAuth) client registration: got response: %s", string(jsonstream[:])) if err = json.Unmarshal(jsonstream, response); err != nil { - err = errors.New("fail to unmashal response:" + string(jsonstream)) + err = errors.New("(RegisterWithAuth) fail to unmashal response:" + string(jsonstream)) return } if len(response.Errs) > 0 { - err = fmt.Errorf("Server returned: %s", strings.Join(response.Errs, ",")) + err = fmt.Errorf("(RegisterWithAuth) Server returned: %s", strings.Join(response.Errs, ",")) return } - client = &response.Data + //client = &response.Data - client.Init() - core.SetClientProfile(client) + //client.Init() + //core.SetClientProfile(client) - logger.Debug(3, "Client registered") + logger.Debug(3, "(RegisterWithAuth) Client registered") return } -func ReRegisterWithSelf(host string) (client *core.Client, err error) { +func ReRegisterWithSelf(host string) (err error) { fmt.Printf("lost contact with server, try to re-register\n") - client, err = RegisterWithAuth(host, core.Self) + err = RegisterWithAuth(host, core.Self) if err != nil { logger.Error("Error: fail to re-register, clientid=" + core.Self.Id) fmt.Printf("failed to re-register\n") @@ -297,7 +329,7 @@ func Set_Metadata(profile *core.Client) { instance_hostname, err := getMetaDataField(metadata_url, "hostname") if err == nil { //instance_hostname = strings.TrimSuffix(instance_hostname, ".novalocal") - profile.Name = instance_hostname + profile.WorkerRuntime.Name = instance_hostname profile.Hostname = instance_hostname } instance_id, err := getMetaDataField(metadata_url, "instance-id") @@ -339,7 +371,7 @@ func ComposeProfile() (profile *core.Client, err error) { //profile = new(core.Client) profile = core.NewClient() // includes init - profile.Name = conf.CLIENT_NAME + profile.WorkerRuntime.Name = conf.CLIENT_NAME profile.Host = conf.CLIENT_HOST profile.Group = conf.CLIENT_GROUP profile.CPUs = runtime.NumCPU() @@ -373,7 +405,7 @@ func ComposeProfile() (profile *core.Client, err error) { return } -func DiscardWorkunit(id string) (err error) { +func DiscardWorkunit(id core.Workunit_Unique_Identifier) (err error) { //fmt.Printf("try to discard workunit %s\n", id) logger.Info("trying to discard workunit %s", id) stage, ok, err := workmap.Get(id) @@ -386,7 +418,7 @@ func DiscardWorkunit(id string) (err error) { } workmap.Set(id, ID_DISCARDED, "DiscardWorkunit") - err = core.Self.Current_work_delete(id, true) + err = core.Self.Current_work.Delete(id, true) if err != nil { logger.Error("(DiscardWorkunit) Could not remove workunit %s from client", id) err = nil diff --git a/lib/worker/processor.go b/lib/worker/processor.go index da3dd429..67589668 100644 --- a/lib/worker/processor.go +++ b/lib/worker/processor.go @@ -63,21 +63,24 @@ func processor(control chan int) { //} //if the work is not succesfully parsed in last stage, pass it into the next one immediately - work_state, ok, xerr := workmap.Get(workunit.Id) + + work_id := workunit.Workunit_Unique_Identifier + + work_state, ok, xerr := workmap.Get(work_id) if xerr != nil { logger.Error("error: %s", xerr.Error()) continue } if !ok { - logger.Error("(processor) workunit.id %s not found", workunit.Id) + logger.Error("(processor) workunit.id %s not found", work_id.String()) continue } if workunit.State == core.WORK_STAT_ERROR || work_state == ID_DISCARDED { if work_state == ID_DISCARDED { - workunit.SetState(core.WORK_STAT_DISCARDED) + workunit.SetState(core.WORK_STAT_DISCARDED, "workmap indicates that workunit has been discarded") } else { - workunit.SetState(core.WORK_STAT_ERROR) + workunit.SetState(core.WORK_STAT_ERROR, "workmap indicates WORK_STAT_ERROR") } fromProcessor <- workunit //release the permit lock, for work overlap inhibitted mode only @@ -87,7 +90,7 @@ func processor(control chan int) { continue } - workmap.Set(workunit.Id, ID_WORKER, "processor") + workmap.Set(work_id, ID_WORKER, "processor") var err error var envkeys []string @@ -101,9 +104,9 @@ func processor(control chan int) { if !wants_docker { envkeys, err = SetEnv(workunit) if err != nil { - logger.Error("(processor) SetEnv(): workid=" + workunit.Id + ", " + err.Error()) + logger.Error("(processor) SetEnv(): workid=" + work_id.String() + ", " + err.Error()) workunit.Notes = append(workunit.Notes, "[processor#SetEnv]"+err.Error()) - workunit.SetState(core.WORK_STAT_ERROR) + workunit.SetState(core.WORK_STAT_ERROR, "see notes") //release the permit lock, for work overlap inhibitted mode only //if !conf.WORKER_OVERLAP && core.Service != "proxy" { // <-chanPermit @@ -117,17 +120,17 @@ func processor(control chan int) { exit_status := workunit.ExitStatus logger.Debug(1, "(processor) ExitStatus of process: %d", exit_status) if err != nil { - logger.Error("(processor) returned error , workid=" + workunit.Id + ", " + err.Error()) + logger.Error("(processor) RunWorkunit returned error , workid=%s, %s", work_id.String(), err.Error()) workunit.Notes = append(workunit.Notes, "[processor#RunWorkunit]"+err.Error()) if exit_status == 42 { - workunit.SetState(core.WORK_STAT_FAILED_PERMANENT) // process told us that is an error where resubmission does not make sense. + workunit.SetState(core.WORK_STAT_FAILED_PERMANENT, "exit_status == 42") // process told us that is an error where resubmission does not make sense. } else { - workunit.SetState(core.WORK_STAT_ERROR) + workunit.SetState(core.WORK_STAT_ERROR, "RunWorkunit failed") } } else { - logger.Debug(1, "(processor) RunWorkunit() returned without error, workid="+workunit.Id) - workunit.SetState(core.WORK_STAT_COMPUTED) + logger.Debug(1, "(processor) RunWorkunit() returned without error, workid=%s", work_id.String()) + workunit.SetState(core.WORK_STAT_COMPUTED, "") workunit.WorkPerf.MaxMemUsage = pstat.MaxMemUsage workunit.WorkPerf.MaxMemoryTotalRss = pstat.MaxMemoryTotalRss workunit.WorkPerf.MaxMemoryTotalSwap = pstat.MaxMemoryTotalSwap @@ -160,17 +163,26 @@ func RunWorkunit(workunit *core.Workunit) (pstats *core.WorkPerf, err error) { if workunit.Cmd.Dockerimage != "" || workunit.Cmd.DockerPull != "" { pstats, err = RunWorkunitDocker(workunit) + if err != nil { + err = fmt.Errorf("(RunWorkunit) RunWorkunitDocker returned: %s", err.Error()) + return + } } else { pstats, err = RunWorkunitDirect(workunit) + if err != nil { + err = fmt.Errorf("(RunWorkunit) RunWorkunitDirect returned: %s", err.Error()) + return + } } - if err != nil { - return - } - - if workunit.CWL != nil { + if workunit.CWL_workunit != nil { + work_path, xerr := workunit.Path() + if xerr != nil { + err = xerr + return + } - stdout_file := workunit.Path() + "/" + conf.STDOUT_FILENAME + stdout_file := work_path + "/" + conf.STDOUT_FILENAME // wait for awe_stdout to be available //for true { @@ -208,7 +220,7 @@ func RunWorkunit(workunit *core.Workunit) (pstats *core.WorkPerf, err error) { } fmt.Println("CWL-runner receipt:") spew.Dump(result_doc) - workunit.CWL.Tool_results = result_doc + workunit.CWL_workunit.Tool_results = result_doc } @@ -233,8 +245,13 @@ func RunWorkunitDocker(workunit *core.Workunit) (pstats *core.WorkPerf, err erro use_wrapper_script := false + work_path, err := workunit.Path() + if err != nil { + return + } + wrapper_script_filename := "awe_workunit_wrapper.sh" - wrapper_script_filename_host := path.Join(workunit.Path(), wrapper_script_filename) + wrapper_script_filename_host := path.Join(work_path, wrapper_script_filename) wrapper_script_filename_docker := path.Join(conf.DOCKER_WORK_DIR, wrapper_script_filename) if len(workunit.Cmd.Cmd_script) > 0 { @@ -525,7 +542,7 @@ func RunWorkunitDocker(workunit *core.Workunit) (pstats *core.WorkPerf, err erro container_cmd := []string{bash_command} //var empty_struct struct{} - bindstr_workdir := workunit.Path() + "/:" + conf.DOCKER_WORK_DIR + bindstr_workdir := work_path + "/:" + conf.DOCKER_WORK_DIR logger.Debug(1, "bindstr_workdir: "+bindstr_workdir) var bindarray = []string{} @@ -661,7 +678,7 @@ func RunWorkunitDocker(workunit *core.Workunit) (pstats *core.WorkPerf, err erro logger.Debug(3, "Container status: %s", cont.State.Status) - inspect_filename := path.Join(workunit.Path(), "container_inspect.json") + inspect_filename := path.Join(work_path, "container_inspect.json") b_inspect, _ := json.MarshalIndent(cont, "", " ") @@ -859,7 +876,13 @@ func RunWorkunitDocker(workunit *core.Workunit) (pstats *core.WorkPerf, err erro func RunWorkunitDirect(workunit *core.Workunit) (pstats *core.WorkPerf, err error) { - args := workunit.Cmd.ParsedArgs + var args []string + + if len(workunit.Cmd.ArgsArray) > 0 { + args = workunit.Cmd.ArgsArray + } else { + args = workunit.Cmd.ParsedArgs + } //change cwd to the workunit's working directory if err := workunit.CDworkpath(); err != nil { @@ -894,10 +917,15 @@ func RunWorkunitDirect(workunit *core.Workunit) (pstats *core.WorkPerf, err erro } } - logger.Debug(3, "(RunWorkunitDirect) Using workpath: %s", workunit.Path()) + work_path, xerr := workunit.Path() + if xerr != nil { + err = xerr + return + } + logger.Debug(3, "(RunWorkunitDirect) Using workpath: %s", work_path) - stdoutFilePath := fmt.Sprintf("%s/%s", workunit.Path(), conf.STDOUT_FILENAME) - stderrFilePath := fmt.Sprintf("%s/%s", workunit.Path(), conf.STDERR_FILENAME) + stdoutFilePath := fmt.Sprintf("%s/%s", work_path, conf.STDOUT_FILENAME) + stderrFilePath := fmt.Sprintf("%s/%s", work_path, conf.STDERR_FILENAME) outfile, err := os.Create(stdoutFilePath) defer outfile.Close() errfile, err := os.Create(stderrFilePath) @@ -1035,8 +1063,13 @@ func runPreWorkExecutionScript(workunit *core.Workunit) (err error) { } } - stdoutFilePath := fmt.Sprintf("%s/%s", workunit.Path(), conf.STDOUT_FILENAME) - stderrFilePath := fmt.Sprintf("%s/%s", workunit.Path(), conf.STDERR_FILENAME) + work_path, xerr := workunit.Path() + if xerr != nil { + err = xerr + return + } + stdoutFilePath := fmt.Sprintf("%s/%s", work_path, conf.STDOUT_FILENAME) + stderrFilePath := fmt.Sprintf("%s/%s", work_path, conf.STDERR_FILENAME) outfile, err := os.Create(stdoutFilePath) defer outfile.Close() errfile, err := os.Create(stderrFilePath) diff --git a/lib/worker/workStealer.go b/lib/worker/workStealer.go index 8aa3608a..a4748eb7 100644 --- a/lib/worker/workStealer.go +++ b/lib/worker/workStealer.go @@ -2,14 +2,17 @@ package worker import ( "encoding/json" - "errors" + //"errors" "fmt" "github.com/MG-RAST/AWE/lib/conf" "github.com/MG-RAST/AWE/lib/core" + //"github.com/MG-RAST/AWE/lib/core/cwl" e "github.com/MG-RAST/AWE/lib/errors" "github.com/MG-RAST/AWE/lib/logger" "github.com/MG-RAST/AWE/lib/logger/event" "github.com/MG-RAST/golib/httpclient" + "github.com/davecgh/go-spew/spew" + "github.com/mitchellh/mapstructure" "io/ioutil" "os" "strings" @@ -18,9 +21,8 @@ import ( ) type WorkResponse struct { - Code int `bson:"status" json:"status"` - Data *core.Workunit `bson:"data" json:"data"` - Errs []string `bson:"error" json:"error"` + core.BaseResponse `bson:",inline" json:",inline" mapstructure:",squash"` + Data *core.Workunit `bson:"data" json:"data" mapstructure:"data"` } type TokenResponse struct { @@ -42,6 +44,7 @@ func workStealer(control chan int) { } workunit, err := CheckoutWorkunitRemote() if err != nil { + core.Self.Busy = false if err.Error() == e.QueueEmpty || err.Error() == e.QueueSuspend || err.Error() == e.NoEligibleWorkunitFound { //normal, do nothing logger.Debug(3, "(workStealer) client %s received status %s from server %s", core.Self.Id, err.Error(), conf.SERVER_URL) @@ -61,6 +64,9 @@ func workStealer(control chan int) { } else if err.Error() == e.ClientDeleted { fmt.Printf("(workStealer) client deleted, exiting...\n") os.Exit(1) // TODO is there a better way of exiting ? E.g. in regard of the logger who wants to flush.... + } else if err.Error() == e.ServerNotFound { + logger.Error("(workStealer) ServerNotFound...\n") + retry += 1 } else { //something is wrong, server may be down logger.Error("(workStealer) checking out workunit: %s, retry=%d", err.Error(), retry) @@ -79,20 +85,23 @@ func workStealer(control chan int) { } else { retry = 0 } - logger.Debug(1, "(workStealer) checked out workunit, id="+workunit.Id) + + work_id := workunit.Workunit_Unique_Identifier + + logger.Debug(1, "(workStealer) checked out workunit, id="+work_id.String()) //log event about work checktout (WC) - logger.Event(event.WORK_CHECKOUT, "workid="+workunit.Id) + logger.Event(event.WORK_CHECKOUT, "workid="+work_id.String()) - err = core.Self.Add_work(workunit.Id) + err = core.Self.Current_work.Add(work_id) if err != nil { logger.Error("(workStealer) error: %s", err.Error()) return } - workmap.Set(workunit.Id, ID_WORKSTEALER, "workStealer") + workmap.Set(work_id, ID_WORKSTEALER, "workStealer") //hand the work to the next step handler: dataMover - workstat := core.NewWorkPerf(workunit.Id) + workstat := core.NewWorkPerf() workstat.Checkout = time.Now().Unix() //rawWork := &Mediumwork{ // Workunit: wu, @@ -100,6 +109,13 @@ func workStealer(control chan int) { //} workunit.WorkPerf = workstat + // make sure cwl-runner is invoked + if workunit.CWL_workunit != nil { + workunit.Cmd.Name = "/usr/bin/cwl-runner" + workunit.Cmd.ArgsArray = []string{"--leave-outputs", "--leave-tmpdir", "--tmp-outdir-prefix", "./tmp/", "--tmpdir-prefix", "./tmp/", "--disable-pull", "--rm-container", "--on-error", "stop", "./cwl_tool.yaml", "./cwl_job_input.yaml"} + + } + //FromStealer <- rawWork // sends to dataMover FromStealer <- workunit // sends to dataMover @@ -114,15 +130,14 @@ func workStealer(control chan int) { } func CheckoutWorkunitRemote() (workunit *core.Workunit, err error) { - logger.Debug(3, "CheckoutWorkunitRemote()") + logger.Debug(3, "(CheckoutWorkunitRemote) start") // get available work dir disk space var stat syscall.Statfs_t syscall.Statfs(conf.WORK_PATH, &stat) availableBytes := stat.Bavail * uint64(stat.Bsize) - response := new(WorkResponse) if core.Self == nil { - err = fmt.Errorf("core.Self == nil") + err = fmt.Errorf("(CheckoutWorkunitRemote) core.Self == nil") return } targeturl := fmt.Sprintf("%s/work?client=%s&available=%d", conf.SERVER_URL, core.Self.Id, availableBytes) @@ -133,37 +148,188 @@ func CheckoutWorkunitRemote() (workunit *core.Workunit, err error) { "Authorization": []string{"CG_TOKEN " + conf.CLIENT_GROUP_TOKEN}, } } - logger.Debug(3, fmt.Sprintf("client %s sends a checkout request to %s with available %d", core.Self.Id, conf.SERVER_URL, availableBytes)) + logger.Debug(3, fmt.Sprintf("(CheckoutWorkunitRemote) client %s sends a checkout request to %s with available %d", core.Self.Id, conf.SERVER_URL, availableBytes)) res, err := httpclient.DoTimeout("GET", targeturl, headers, nil, nil, time.Second*0) - logger.Debug(3, fmt.Sprintf("client %s sent a checkout request to %s with available %d", core.Self.Id, conf.SERVER_URL, availableBytes)) + logger.Debug(3, fmt.Sprintf("(CheckoutWorkunitRemote) client %s sent a checkout request to %s with available %d", core.Self.Id, conf.SERVER_URL, availableBytes)) if err != nil { - err = fmt.Errorf("error sending checkout request: %s", err.Error()) + err = fmt.Errorf("(CheckoutWorkunitRemote) error sending checkout request: %s", err.Error()) return } defer res.Body.Close() jsonstream, err := ioutil.ReadAll(res.Body) if err != nil { - return nil, err + return + } + + var response core.StandardResponse + response.Status = -1 + + if err = json.Unmarshal(jsonstream, &response); err != nil { // response + fmt.Printf("jsonstream:\n%s\n", jsonstream) + err = fmt.Errorf("(CheckoutWorkunitRemote) json.Unmarshal error: %s", err.Error()) + return + } + + spew.Dump(response) + + if response.Status == -1 { + err = fmt.Errorf(e.ServerNotFound) + return } - if err = json.Unmarshal(jsonstream, response); err != nil { + + if len(response.Error) > 0 { + message := strings.Join(response.Error, ",") + err = fmt.Errorf("%s", message) return } - if len(response.Errs) > 0 { - return nil, errors.New(strings.Join(response.Errs, ",")) + + data_generic := response.Data + if data_generic == nil { + err = fmt.Errorf("(CheckoutWorkunitRemote) Data field missing") + return } - if response.Code == 200 { - workunit = response.Data - if workunit.Info.Auth == true { - if token, err := FetchDataTokenByWorkId(workunit.Id); err == nil && token != "" { - workunit.Info.DataToken = token - } else { - return workunit, errors.New("need data token but failed to fetch one") + + // remove CWL + data_map, ok := data_generic.(map[string]interface{}) + if !ok { + err = fmt.Errorf("(CheckoutWorkunitRemote) Could not make data field map[string]interface{}") + return + } + var cwl_object *core.CWL_workunit + + cwl_generic, has_cwl := data_map["cwl"] + if has_cwl { + if cwl_generic != nil { + var xerr error + cwl_object, xerr = core.NewCWL_workunit_from_interface(cwl_generic) + if xerr != nil { + err = fmt.Errorf("(CheckoutWorkunitRemote) NewCWL_workunit_from_interface failed: %s", xerr.Error()) + return } + //response_generic["CWL"] = nil + } else { + has_cwl = false } - logger.Debug(3, fmt.Sprintf("client %s got a workunit", core.Self.Id)) - workunit.State = core.WORK_STAT_CHECKOUT - return workunit, nil + delete(data_map, "cwl") + } + + info := &core.Info{} + info_if, has_info := data_map["info"] + if has_info { // interface -> json -> struct // this is a bit ugly + + info_byte, xerr := json.Marshal(info_if) + if xerr != nil { + err = xerr + return + } + + yerr := json.Unmarshal(info_byte, info) + if yerr != nil { + err = yerr + return + } + delete(data_map, "info") + + } + + command := &core.Command{} + command_if, has_command := data_map["command"] + if has_command { // interface -> json -> struct // this is a bit ugly + + command_byte, xerr := json.Marshal(command_if) + if xerr != nil { + err = xerr + return + } + + yerr := json.Unmarshal(command_byte, command) + if yerr != nil { + err = yerr + return + } + delete(data_map, "command") + + } + + partinfo := &core.PartInfo{} + partinfo_if, has_partinfo := data_map["partinfo"] + if has_partinfo { // interface -> json -> struct // this is a bit ugly + + partinfo_byte, xerr := json.Marshal(partinfo_if) + if xerr != nil { + err = xerr + return + } + + yerr := json.Unmarshal(partinfo_byte, partinfo) + if yerr != nil { + err = yerr + return + } + delete(data_map, "partinfo") + + } + + _, has_checkout_time := data_map["checkout_time"] + if has_checkout_time { + delete(data_map, "checkout_time") // TODO add checkout_time as time.Time + + } + + //delete(data_map, "info") + + workunit = &core.Workunit{} + workunit.Info = info + workunit.Workunit_Unique_Identifier = core.Workunit_Unique_Identifier{} + //if has_checkout_time { + // workunit_checkout_time_str, ok := workunit_checkout_time_if.(string) + // if !ok { + // err = fmt.Errorf("(CheckoutWorkunitRemote) cannot type assert checkout_time") + // return + // } + // workunit.CheckoutTime = workunit_checkout_time + //} + + err = mapstructure.Decode(data_map, workunit) + if err != nil { + err = fmt.Errorf("(CheckoutWorkunitRemote) mapstructure.Decode error: %s", err.Error()) + return + } + if has_cwl { + workunit.CWL_workunit = cwl_object + } + //spew.Dump(response) + + if response.Status == 0 { // this is ugly + err = fmt.Errorf(e.ServerNotFound) + return + } + + if response.Status != 200 { + err = fmt.Errorf("(CheckoutWorkunitRemote) response_generic.Status != 200 : %d", response.Status) + return + } + + //workunit = response.Data + + if workunit.Info.Auth == true { + token, xerr := FetchDataTokenByWorkId(workunit.Id) + if xerr == nil && token != "" { + workunit.Info.DataToken = token + } else { + err = fmt.Errorf("(CheckoutWorkunitRemote) need data token but failed to fetch one %s", xerr.Error()) + return + } + } + + logger.Debug(3, "(CheckoutWorkunitRemote) workunit id: %s", workunit.Id) + + logger.Debug(3, "(CheckoutWorkunitRemote) workunit Rank:%d TaskId:%s JobId:%s", workunit.Rank, workunit.TaskId, workunit.JobId) + + logger.Debug(3, fmt.Sprintf("(CheckoutWorkunitRemote) client %s got a workunit", core.Self.Id)) + workunit.State = core.WORK_STAT_CHECKOUT + core.Self.Busy = true return } diff --git a/lib/worker/workermgr.go b/lib/worker/workermgr.go index f63f0bac..b7164731 100644 --- a/lib/worker/workermgr.go +++ b/lib/worker/workermgr.go @@ -26,13 +26,13 @@ type Mediumwork struct { } const ( - ID_HEARTBEATER = 0 - ID_WORKSTEALER = 1 - ID_DATAMOVER = 2 - ID_WORKER = 3 - ID_DELIVERER = 4 - ID_REDISTRIBUTOR = 5 - ID_DISCARDED = 6 // flag acts as a message + ID_HEARTBEATER = 0 + ID_WORKSTEALER = 1 + ID_DATADOWNLOADER = 2 + ID_WORKER = 3 + ID_DELIVERER = 4 + ID_REDISTRIBUTOR = 5 + ID_DISCARDED = 6 // flag acts as a message ) func InitWorkers() { @@ -57,7 +57,7 @@ func StartClientWorkers() { go heartBeater(control) go workStealer(control) } - go dataMover(control) + go dataDownloader(control) go processor(control) go deliverer(control) @@ -76,8 +76,8 @@ func StartClientWorkers() { case ID_WORKSTEALER: go workStealer(control) logger.Error("workStealer died and restarted") - case ID_DATAMOVER: - go dataMover(control) + case ID_DATADOWNLOADER: + go dataDownloader(control) logger.Error("dataMover died and restarted") case ID_WORKER: go processor(control) diff --git a/lib/worker/workmap.go b/lib/worker/workmap.go index c39e5ce0..ccf6c013 100644 --- a/lib/worker/workmap.go +++ b/lib/worker/workmap.go @@ -6,16 +6,16 @@ import ( type WorkMap struct { core.RWMutex - _map map[string]int + _map map[core.Workunit_Unique_Identifier]int } func NewWorkMap() *WorkMap { - wm := &WorkMap{_map: make(map[string]int)} + wm := &WorkMap{_map: make(map[core.Workunit_Unique_Identifier]int)} wm.RWMutex.Init("WorkMap") return wm } -func (this *WorkMap) Get(id string) (value int, ok bool, err error) { +func (this *WorkMap) Get(id core.Workunit_Unique_Identifier) (value int, ok bool, err error) { rlock, err := this.RLockNamed("Get") if err != nil { return @@ -25,7 +25,7 @@ func (this *WorkMap) Get(id string) (value int, ok bool, err error) { return } -func (this *WorkMap) GetKeys() (value []string, err error) { +func (this *WorkMap) GetKeys() (value []core.Workunit_Unique_Identifier, err error) { rlock, err := this.RLockNamed("Get") if err != nil { return @@ -37,7 +37,7 @@ func (this *WorkMap) GetKeys() (value []string, err error) { return } -func (this *WorkMap) Set(id string, value int, name string) (err error) { +func (this *WorkMap) Set(id core.Workunit_Unique_Identifier, value int, name string) (err error) { err = this.LockNamed("Set_" + name) if err != nil { return @@ -49,7 +49,7 @@ func (this *WorkMap) Set(id string, value int, name string) (err error) { return } -func (this *WorkMap) Delete(id string) (err error) { +func (this *WorkMap) Delete(id core.Workunit_Unique_Identifier) (err error) { rlock, err := this.RLockNamed("Get") if err != nil { return diff --git a/site/widgets/widget.awe_monitor.js b/site/widgets/widget.awe_monitor.js index 3c3497df..a179c566 100644 --- a/site/widgets/widget.awe_monitor.js +++ b/site/widgets/widget.awe_monitor.js @@ -240,7 +240,7 @@ qwt.settings.navigation_url = RetinaConfig["awe_ip"]+"/work?query"; qwt.settings.rows_per_page = 10; qwt.settings.minwidths = [1,1,1,1,65,78,75,75,83]; - qwt.settings.asynch_column_mapping = { "wuid": "wuid", + qwt.settings.asynch_column_mapping = { "id": "id", "submission time": "info.submittime", "cmd name": "cmd.name", "cmd args": "cmd.args", @@ -270,7 +270,7 @@ qwt.settings.disable_sort = {}; qwt.settings.filter_autodetect = false; qwt.settings.sort_autodetect = false; - qwt.settings.data = { data: [], header: [ "wuid", + qwt.settings.data = { data: [], header: [ "id", "submission time", "cmd name", "cmd args", @@ -473,7 +473,7 @@ var result_data = []; for (var i=0;i"+obj.wuid+"", + result_data.push( { "id": ""+obj.wuid+"", "client": ""+obj.client+"", "checkout time": obj.checkout_time, "cmd name": obj.cmd.name, @@ -487,7 +487,7 @@ } ); } if (! result_data.length) { - result_data.push({"wuid": "-", "client": "-", "checkout time": "-", "cmd name": "-", "cmd args": "-", "rank": "-", "tot": "-", "state": "-", "failed": "-"}); + result_data.push({"id": "-", "client": "-", "checkout time": "-", "cmd name": "-", "cmd args": "-", "rank": "-", "tot": "-", "state": "-", "failed": "-"}); } return result_data; diff --git a/tests/test-startat.cwf b/tests/test-startat.cwf new file mode 100644 index 00000000..7353fa52 --- /dev/null +++ b/tests/test-startat.cwf @@ -0,0 +1,21 @@ +{ + "info": { + "pipeline": "test_sleep", + "name": "testname_sleep", + "project": "testproject", + "clientgroups": "docker", + "start_at" : "2017-09-05T20:06:36+00:00" + }, + "tasks": [ + { + "cmd": { + "name": "sleep", + "args": "10s", + "description": "sleep test", + "_Dockerimage": "ubuntu" + }, + "taskid": "0", + "totalwork": 1 + } + ] +} \ No newline at end of file