From d30239cd84b1f14050ffb3c2055d9229d9f0fd26 Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Wed, 13 May 2020 16:18:23 -0400 Subject: [PATCH 1/3] added getJobQueues function in Jobs module, added some fixes in other functions (e.g., jobsList) --- NAMESPACE | 2 +- R/Jobs.r | 53 +++++++++++++++++++++++-------- man/Jobs.getJobQueues.Rd | 28 ++++++++++++++++ man/Jobs.submitNotebookJob.Rd | 2 +- man/Jobs.submitShellCommandJob.Rd | 2 +- 5 files changed, 70 insertions(+), 17 deletions(-) create mode 100644 man/Jobs.getJobQueues.Rd diff --git a/NAMESPACE b/NAMESPACE index e02a863..7ebdb88 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -6,5 +6,5 @@ export(SciDrive.createContainer,SciDrive.upload,SciDrive.publicUrl,SciDrive.down export(Config.isSciServerComputeEnvironment,Config.CasJobsRESTUri,Config.AuthenticationURL,Config.SciDriveHost,Config.SkyQueryUrl,Config.DataRelease,Config.SkyServerWSurl,Config.Version,Config.KeystoneTokenFilePath,Config.ComputeJobDirectoryFile,Config.RacmApiURL) export(SkyQuery.getJobStatus,SkyQuery.cancelJob,SkyQuery.listQueues,SkyQuery.getQueueInfo,SkyQuery.submitJob,SkyQuery.waitForJob,SkyQuery.listJobs,SkyQuery.listAllDatasets,SkyQuery.getDatasetInfo,SkyQuery.listDatasetTables,SkyQuery.getTableInfo,SkyQuery.listTableColumns,SkyQuery.getTable,SkyQuery.dropTable,SkyQuery.uploadTable) export(Files.getFileServices,Files.getFileServicesNames,Files.getFileServiceFromName,Files.__getFileServiceAPIUrl,Files.getRootVolumesInfo,Files.getDataVolumesInfo,Files.getUserVolumesInfo,Files.splitPath,Files.createUserVolume,Files.deleteUserVolume,Files.createDir,Files.delete,Files.upload,Files.download,Files.dirList,Files.move,Files.shareUserVolume) -export(Jobs.getDockerComputeDomains,Jobs.getDockerComputeDomainsNames,Jobs.getDockerComputeDomainFromName,Jobs.getRDBComputeDomains,Jobs.getRDBComputeDomainsNames,Jobs.getRDBComputeDomainFromName,Jobs.getJobsList,Jobs.getDockerJobsListQuick,Jobs.getJobDescription,Jobs.getJobStatus,Jobs.submitNotebookJob,Jobs.submitShellCommandJob,Jobs.submitRDBQueryJob,Jobs.cancelJob,Jobs.waitForJob) +export(Jobs.getDockerComputeDomains,Jobs.getDockerComputeDomainsNames,Jobs.getDockerComputeDomainFromName,Jobs.getRDBComputeDomains,Jobs.getRDBComputeDomainsNames,Jobs.getRDBComputeDomainFromName,Jobs.getJobsList,Jobs.getDockerJobsListQuick,Jobs.getJobDescription,Jobs.getJobStatus,Jobs.submitNotebookJob,Jobs.submitShellCommandJob,Jobs.submitRDBQueryJob,Jobs.cancelJob,Jobs.waitForJob,Jobs.getJobQueues) import(httr,jsonlite,utils,data.table) diff --git a/R/Jobs.r b/R/Jobs.r index 23a4364..2811519 100644 --- a/R/Jobs.r +++ b/R/Jobs.r @@ -162,7 +162,7 @@ Jobs.getJobsList <- function(top=10, open=NULL, start=NULL, end=NULL, type="all" url = paste(Config.RacmApiURL,"/jobm/rest/dockerjobs?",sep="") } - url = paste(url,topString,startString,endString,"TaskName=",taskName,sep="") + url = paste(url,topString,startString,endString,openString,"TaskName=",taskName,sep="") r = GET(url,add_headers('X-Auth-Token'=token),accept("application/json")) @@ -198,7 +198,7 @@ Jobs.getDockerJobsListQuick <- function(top=10, open=NULL, start=NULL, end=NULL, url = paste(Config.RacmApiURL,"/jobm/rest/dockerjobs/quick?",sep="") - url = paste(url,topString,startString,endString,"TaskName=",taskName,sep="") + url = paste(url,topString,startString,endString,labRegString,openString,"TaskName=",taskName,sep="") r = GET(url,add_headers('X-Auth-Token'=token),accept("application/json")) @@ -212,6 +212,31 @@ Jobs.getDockerJobsListQuick <- function(top=10, open=NULL, start=NULL, end=NULL, } } +Jobs.getJobQueues <- function(){ + + token = Authentication.getToken() + if(!is.null(token) && token != "") + { + + if(Config.isSciServerComputeEnvironment()){ + taskName = "Compute.SciScript-R.Jobs.getJobQueues" + }else{ + taskName = "SciScript-R.Jobs.getJobQueues" + } + + url = paste(Config.RacmApiURL,"/jobm/rest/jobs/queues?","TaskName=",taskName,sep="") + + r = GET(url,add_headers('X-Auth-Token'=token),accept("application/json")) + + if(r$status_code != 200) { + stop(paste("Error when getting job queues from JOBM API.\nHttp Response from JOBM API returned status code ", r$status_code, ":\n", content(r, as="text", encoding="UTF-8"))) + } else { + return(content(r)) + } + }else{ + stop(paste("User token is not defined. First log into SciServer.")) + } +} Jobs.getJobDescription <- function(jobId){ @@ -316,7 +341,7 @@ Jobs.submitNotebookJob <- function(notebookPath, dockerComputeDomain=NULL, docke if(uVol$name == vol$name && uVol$rootVolumeName == vol$rootVolumeName && uVol$owner == vol$owner){ found = TRUE - if(uVol$needsWriteAccess){ + if(!is.null(uVol$needsWriteAccess)){ if(uVol$needsWriteAccess == TRUE && ('write' %in% vol$allowedActions) ){ uVols[[length(uVols)+1]] <- list(userVolumeId= vol$id, needsWriteAccess= TRUE) @@ -346,7 +371,7 @@ Jobs.submitNotebookJob <- function(notebookPath, dockerComputeDomain=NULL, docke if( is.null(dataVolumes)){ for( i in 1:length(dockerComputeDomain$volumes)){ vol = dockerComputeDomain$volumes[[i]] - if("write" %in% vol$allowedActions){ + if(vol$writable){ dataVols[[length(dataVols)+1]] <- list(id=vol$id, name= vol$name, writable=TRUE) }else{ dataVols[[length(dataVols)+1]] <- list(id=vol$id, name= vol$name, writable=FALSE) @@ -360,9 +385,9 @@ Jobs.submitNotebookJob <- function(notebookPath, dockerComputeDomain=NULL, docke vol = dockerComputeDomain$volumes[[j]] if( vol$name == dVol$name ){ found = TRUE; - if(dVol$needsWriteAccess){ + if(!is.null(dVol$needsWriteAccess)){ - if(dVol$needsWriteAccess == TRUE && ('write' %in% vol$allowedActions) ){ + if(dVol$needsWriteAccess == TRUE) ){ dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= TRUE) }else{ dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= FALSE) @@ -370,7 +395,7 @@ Jobs.submitNotebookJob <- function(notebookPath, dockerComputeDomain=NULL, docke }else{ - if('write' %in% vol$allowedActions ){ + if(vol$writable){ dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= TRUE) }else{ dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= FALSE) @@ -465,7 +490,7 @@ Jobs.submitShellCommandJob <- function(shellCommand, dockerComputeDomain = NULL, if(uVol$name == vol$name && uVol$rootVolumeName == vol$rootVolumeName && uVol$owner == vol$owner){ found = TRUE - if(uVol$needsWriteAccess){ + if(!is.null(uVol$needsWriteAccess)){ if(uVol$needsWriteAccess == TRUE && ('write' %in% vol$allowedActions) ){ uVols[[length(uVols)+1]] <- list(userVolumeId= vol$id, needsWriteAccess= TRUE) @@ -495,7 +520,7 @@ Jobs.submitShellCommandJob <- function(shellCommand, dockerComputeDomain = NULL, if( is.null(dataVolumes)){ for( i in 1:length(dockerComputeDomain$volumes)){ vol = dockerComputeDomain$volumes[[i]] - if("write" %in% vol$allowedActions){ + if(vol$writable){ dataVols[[length(dataVols)+1]] <- list(id=vol$id, name= vol$name, writable=TRUE) }else{ dataVols[[length(dataVols)+1]] <- list(id=vol$id, name= vol$name, writable=FALSE) @@ -509,24 +534,24 @@ Jobs.submitShellCommandJob <- function(shellCommand, dockerComputeDomain = NULL, vol = dockerComputeDomain$volumes[[j]] if( vol$name == dVol$name ){ found = TRUE; - if(dVol$needsWriteAccess){ + if(!is.null(dVol$needsWriteAccess)){ - if(dVol$needsWriteAccess == TRUE && ('write' %in% vol$allowedActions) ){ + if(dVol$needsWriteAccess == TRUE) ){ dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= TRUE) }else{ dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= FALSE) } - + }else{ - if('write' %in% vol$allowedActions ){ + if(vol$writable){ dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= TRUE) }else{ dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= FALSE) } } - + } } diff --git a/man/Jobs.getJobQueues.Rd b/man/Jobs.getJobQueues.Rd new file mode 100644 index 0000000..e2136b2 --- /dev/null +++ b/man/Jobs.getJobQueues.Rd @@ -0,0 +1,28 @@ +\name{Jobs.getJobQueues} +\alias{Jobs.getJobQueues} +\title{ +Get Jobs Queues +} +\description{ +Gets information about queues of jobs submitted to all compute domains, including the ranking jobs that users have already submitted to the queue for execution, as well as the ranking that a new job would get if submitted to a queue. +} +\usage{Jobs.getJobQueues()} +\arguments{ +} +\value{a list containing job queues information.} +\author{ +Manuchehr Taghizadeh-Popp \cr +Maintainer: Manuchehr Taghizadeh-Popp +} +\seealso{ +\code{\link[SciServer]{Jobs.getJobsList}},\code{\link[SciServer]{Jobs.submitNotebookJob}},\code{\link[SciServer]{Jobs.submitShellCommandJob}},\code{\link[SciServer]{Jobs.getJobStatus}},\code{\link[SciServer]{Jobs.getDockerComputeDomains}},\code{\link[SciServer]{Jobs.cancelJob}} +} +\references{ +\href{http://www.sciserver.org}{http://www.sciserver.org}\cr +\href{http://apps.sciserver.org/}{http://apps.sciserver.org}\cr +\href{http://www.github.com/sciserver/SciScript-R}{http://www.github.com/sciserver/SciScript-R} +} +\examples{ +jobs = Jobs.getJobQueues() +} +\keyword{SciServer} \ No newline at end of file diff --git a/man/Jobs.submitNotebookJob.Rd b/man/Jobs.submitNotebookJob.Rd index 58277c2..ef5f1d6 100644 --- a/man/Jobs.submitNotebookJob.Rd +++ b/man/Jobs.submitNotebookJob.Rd @@ -13,7 +13,7 @@ Submits a Jupyter Notebook for execution (as an asynchronous job) inside a Docke \item{dockerImageName}{name (string) of the Docker image for executing the notebook. E.g., dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object.} \item{userVolumes}{a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: userVolumes = list( list(name="JobsTestVolume", rootVolumeName="Storage", owner="myUserName", needsWriteAccess=TRUE), list(name="scratch", rootVolumeName="Temporary", owner="myUserName", needsWriteAccess=TRUE) ). A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=Null, then all available user volumes are mounted, with 'needsWriteAccess' = TRUE if the user has Write permissions on the volume.} - \item{dataVolumes}{a list with the names of data volumes that will be mounted to the docker Image. E.g.: dataVolumes=list(list(name='SDSS DAS')). A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain list. If dataVolumes=NULL, then all available data volumes are mounted.} + \item{dataVolumes}{a list with the names of data volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: dataVolumes=list(list(name='SDSS DAS', needsWriteAccess=TRUE)). A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain list. If dataVolumes=NULL, then all available data volumes are mounted.} \item{resultsFolderPath}{full path to results folder (string) where the original notebook is copied to and executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically.} \item{parameters}{string containing parameters that the notebook might need during its execution. This string is written in the 'parameters.txt' file in the same directory level where the notebook is being executed.} \item{jobAlias}{alias (string) of job, defined by the user.} diff --git a/man/Jobs.submitShellCommandJob.Rd b/man/Jobs.submitShellCommandJob.Rd index d7d1f46..02f62fb 100644 --- a/man/Jobs.submitShellCommandJob.Rd +++ b/man/Jobs.submitShellCommandJob.Rd @@ -13,7 +13,7 @@ Submits a shell command for execution (as an asynchronous job) inside a Docker c \item{dockerImageName}{name (string) of the Docker image for executing the notebook. E.g., dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object.} \item{userVolumes}{a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: userVolumes = list( list(name="JobsTestVolume", rootVolumeName="Storage", owner="myUserName", needsWriteAccess=TRUE), list(name="scratch", rootVolumeName="Temporary", owner="myUserName", needsWriteAccess=TRUE) ). A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=Null, then all available user volumes are mounted, with 'needsWriteAccess' = TRUE if the user has Write permissions on the volume.} - \item{dataVolumes}{a list with the names of data volumes that will be mounted to the docker Image. E.g.: dataVolumes=list(list(name='SDSS DAS')). A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain list. If dataVolumes=NULL, then all available data volumes are mounted.} + \item{dataVolumes}{a list with the names of data volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: dataVolumes=list(list(name='SDSS DAS', needsWriteAccess=FALSE)). A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain list. If dataVolumes=NULL, then all available data volumes are mounted.} \item{resultsFolderPath}{full path to results folder (string) where the original notebook is copied to and executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically.} \item{jobAlias}{alias (string) of job, defined by the user.} } From 369a0e97b3c02e15edd37d40e9be805f5584372c Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Wed, 13 May 2020 16:43:23 -0400 Subject: [PATCH 2/3] fixed typo --- R/Jobs.r | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/Jobs.r b/R/Jobs.r index 2811519..b30020f 100644 --- a/R/Jobs.r +++ b/R/Jobs.r @@ -387,7 +387,7 @@ Jobs.submitNotebookJob <- function(notebookPath, dockerComputeDomain=NULL, docke found = TRUE; if(!is.null(dVol$needsWriteAccess)){ - if(dVol$needsWriteAccess == TRUE) ){ + if(dVol$needsWriteAccess == TRUE){ dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= TRUE) }else{ dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= FALSE) @@ -536,7 +536,7 @@ Jobs.submitShellCommandJob <- function(shellCommand, dockerComputeDomain = NULL, found = TRUE; if(!is.null(dVol$needsWriteAccess)){ - if(dVol$needsWriteAccess == TRUE) ){ + if(dVol$needsWriteAccess == TRUE){ dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= TRUE) }else{ dataVols[[length(dataVols)+1]] <- list(id= vol$id, name=vol$name, writable= FALSE) From 946bcab78de1857b90f5ad1478bc79a86d0c156a Mon Sep 17 00:00:00 2001 From: mtaghiza Date: Wed, 13 May 2020 17:05:14 -0400 Subject: [PATCH 3/3] add dataframe as output format option for getJobQueues method in Jobs module --- R/Jobs.r | 12 ++++++++++-- man/Jobs.getJobQueues.Rd | 3 ++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/R/Jobs.r b/R/Jobs.r index b30020f..7737ef0 100644 --- a/R/Jobs.r +++ b/R/Jobs.r @@ -212,7 +212,7 @@ Jobs.getDockerJobsListQuick <- function(top=10, open=NULL, start=NULL, end=NULL, } } -Jobs.getJobQueues <- function(){ +Jobs.getJobQueues <- function(format="dataframe"){ token = Authentication.getToken() if(!is.null(token) && token != "") @@ -231,7 +231,15 @@ Jobs.getJobQueues <- function(){ if(r$status_code != 200) { stop(paste("Error when getting job queues from JOBM API.\nHttp Response from JOBM API returned status code ", r$status_code, ":\n", content(r, as="text", encoding="UTF-8"))) } else { - return(content(r)) + + if(format == "dataframe"){ + r = fromJSON(content(r, "text", encoding="UTF-8")) + df <- data.frame(r$rows) + colnames(df) <- r$columns + return(df) + }else{ + return(content(r)) + } } }else{ stop(paste("User token is not defined. First log into SciServer.")) diff --git a/man/Jobs.getJobQueues.Rd b/man/Jobs.getJobQueues.Rd index e2136b2..0eadd52 100644 --- a/man/Jobs.getJobQueues.Rd +++ b/man/Jobs.getJobQueues.Rd @@ -8,8 +8,9 @@ Gets information about queues of jobs submitted to all compute domains, includin } \usage{Jobs.getJobQueues()} \arguments{ + \item{format}{Format (string) of the returned object. If equal to "dataframe" (default setting), then the reult is a dataframe, else it is a list.} } -\value{a list containing job queues information.} +\value{a dataframe or list containing job queues information.} \author{ Manuchehr Taghizadeh-Popp \cr Maintainer: Manuchehr Taghizadeh-Popp