> byMonth = getClusterUsageByMonth(Arrays.asList("bimberlab", "onprcgenetics"), 12);
+ byMonth.sort(Comparator.comparing(o -> String.valueOf(o.get("Account"))));
+
+ msg.append("Cluster Usage By Month:");
+ msg.append("
| Account | Month | CPU | GPU | Compute Units |
");
+ byMonth.forEach(map -> {
+ long cpu = (Long)map.get("CPU");
+ long gpu = (Long)map.get("GPU");
+ double units = ((double)cpu/6000) + ((double)gpu/600);
+ Date start = (Date)map.get("Start");
+
+ msg.append("| ").append(map.get("Account")).append(" | ").append(getDateTimeFormat(c).format(start)).append(" | ").append(String.format("%,d", cpu)).append(" | ").append(String.format("%,d", gpu)).append(" | ").append(String.format("%,.2f", units)).append(" |
");
+ });
+
+ msg.append("
");
+ msg.append("
\n");
+
try
{
SimpleScriptWrapper wrapper = new SimpleScriptWrapper(_log);
String results = wrapper.executeWithOutput(Arrays.asList("ssh", "-q", "labkey_submit@arc", "sshare", "-U", "-u", "labkey_submit"));
- msg.append("Cluster Usage:");
+ msg.append("Cluster Priority By Account:
");
msg.append("
| Account | NormShares | RawUsage | EffectiveUsage | FairShare |
");
AtomicBoolean foundHeader = new AtomicBoolean(false);
@@ -148,14 +158,12 @@ else if (!foundHeader.get())
msg.append("");
});
msg.append("
");
+ msg.append("
\n");
}
catch (PipelineJobException e)
{
_log.error("Error fetching slurm summary", e);
}
-
- msg.append("\n");
-
}
private void getDiskUsageStats(Container c, User u, final StringBuilder msg)
@@ -194,6 +202,107 @@ private void getDiskUsageStats(Container c, User u, final StringBuilder msg)
_log.error("Error running df", e);
}
- msg.append("
\n");
+ msg.append("
\n");
+ }
+
+ private List> getClusterUsageByMonth(List accounts, int numMonths)
+ {
+ Calendar currentCal = Calendar.getInstance();
+ int currentMonth = currentCal.get(Calendar.MONTH);
+ int currentYear = currentCal.get(Calendar.YEAR);
+
+ List> results = new ArrayList<>();
+
+ int offset = 0;
+ while (offset < numMonths)
+ {
+ Calendar cal = Calendar.getInstance();
+ cal.set(Calendar.YEAR, currentYear);
+ cal.set(Calendar.MONTH, currentMonth - offset);
+ cal.set(Calendar.DAY_OF_MONTH, 1);
+ cal.set(Calendar.HOUR_OF_DAY, 0);
+ cal.set(Calendar.MINUTE, 0);
+ cal.set(Calendar.SECOND, 0);
+ cal.set(Calendar.MILLISECOND, 0);
+ Date start = cal.getTime();
+
+ Calendar endCal = Calendar.getInstance();
+ endCal.setTime(start);
+ endCal.set(Calendar.DAY_OF_MONTH, endCal.getActualMaximum(Calendar.DAY_OF_MONTH));
+
+ // Set to last second of the day:
+ endCal.add(Calendar.DATE, 1);
+ endCal.add(Calendar.MILLISECOND, -1);
+ Date end = endCal.getTime();
+
+ results.addAll(getClusterUsageForInterval(accounts, start, end));
+
+ offset++;
+ }
+
+ return results;
+ }
+
+ private @NotNull List> getClusterUsageForInterval(List accounts, Date start, Date end)
+ {
+ if (!SystemUtils.IS_OS_LINUX)
+ {
+ return Collections.emptyList();
+ }
+
+ final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
+
+ try
+ {
+ List args = new ArrayList<>(Arrays.asList("ssh", "-q", "labkey_submit@arc", "/usr/local/bin/sreport-accts-summary", "Accounts=" + StringUtils.join(accounts, ",")));
+ if (start != null)
+ {
+ args.add("Start=" + dateFormat.format(start));
+ }
+
+ if (end != null)
+ {
+ args.add("End=" + dateFormat.format(end));
+ }
+
+ SimpleScriptWrapper wrapper = new SimpleScriptWrapper(_log);
+ String results = wrapper.executeWithOutput(args);
+
+ AtomicBoolean foundHeader = new AtomicBoolean(false);
+ List> ret = Arrays.stream(results.split("\n")).map(x -> {
+ if (x.startsWith("Account|"))
+ {
+ foundHeader.set(true);
+ return null;
+ }
+ else if (!foundHeader.get())
+ {
+ return null;
+ }
+
+ String[] els = x.split("\\|");
+
+ if (els.length != 3)
+ {
+ _log.error("Unexpected line: " + StringUtils.join(els, "<>"));
+ return null;
+ }
+
+ Map map = new HashMap<>(Map.of("Account", els[0], "CPU", Long.parseLong(els[1]), "GPU", Long.parseLong(els[2])));
+ return map;
+ }).filter(Objects::nonNull).toList();
+
+ ret.forEach(map -> {
+ map.put("Start", start);
+ map.put("End", end);
+ });
+
+ return ret;
+ }
+ catch (PipelineJobException e)
+ {
+ _log.error("Error fetching slurm summary", e);
+ return Collections.emptyList();
+ }
}
}
\ No newline at end of file
diff --git a/primeseq/src/org/labkey/primeseq/pipeline/BlastPipelineJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/BlastPipelineJobResourceAllocator.java
index 73e1ab109..7f2a687e6 100644
--- a/primeseq/src/org/labkey/primeseq/pipeline/BlastPipelineJobResourceAllocator.java
+++ b/primeseq/src/org/labkey/primeseq/pipeline/BlastPipelineJobResourceAllocator.java
@@ -40,7 +40,7 @@ public Integer getMaxRequestMemory(PipelineJob job)
}
@Override
- public void addExtraSubmitScriptLines(PipelineJob job, RemoteExecutionEngine engine, List lines)
+ public void addExtraSubmitScriptLines(PipelineJob job, RemoteExecutionEngine engine, List lines)
{
//force BLAST jobs to top of queue, since we assume these run quickly
if ("HTCondorEngine".equals(engine.getType()))
diff --git a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java
index 1151b766f..c213266fd 100644
--- a/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java
+++ b/primeseq/src/org/labkey/primeseq/pipeline/SequenceJobResourceAllocator.java
@@ -116,8 +116,8 @@ public Integer getMaxRequestCpus(PipelineJob job)
if (isSequenceNormalizationTask(job))
{
- job.getLogger().debug("setting max CPUs to 4");
- return 4;
+ job.getLogger().debug("setting max CPUs to 2");
+ return 2;
}
if (isLuceneIndexJob(job))
@@ -137,15 +137,15 @@ public Integer getMaxRequestCpus(PipelineJob job)
//10gb
if (totalFileSize < 10e9)
{
- job.getLogger().debug("file size less than 10gb, lowering CPUs to 8");
+ job.getLogger().debug("file size less than 10gb, lowering CPUs to 4");
- return 8;
+ return 4;
}
else if (totalFileSize < 20e9)
{
- job.getLogger().debug("file size less than 20gb, lowering CPUs to 12");
+ job.getLogger().debug("file size less than 20gb, lowering CPUs to 8");
- return 12;
+ return 8;
}
job.getLogger().debug("file size greater than 20gb, using 12 CPUs");
@@ -318,7 +318,7 @@ public Integer getMaxRequestMemory(PipelineJob job)
}
@Override
- public void addExtraSubmitScriptLines(PipelineJob job, RemoteExecutionEngine engine, List lines)
+ public void addExtraSubmitScriptLines(PipelineJob job, RemoteExecutionEngine engine, List lines)
{
if (job instanceof HasJobParams)
{
@@ -332,7 +332,7 @@ public void addExtraSubmitScriptLines(PipelineJob job, RemoteExecutionEngine eng
}
@Override
- public @NotNull Map getEnvironmentVars(PipelineJob job, RemoteExecutionEngine engine)
+ public @NotNull Map getEnvironmentVars(PipelineJob job, RemoteExecutionEngine engine)
{
Map ret = new HashMap<>();
@@ -358,7 +358,7 @@ private String getTime(PipelineJob job)
return null;
}
- private void possiblyAddHighIO(PipelineJob job, RemoteExecutionEngine engine, List lines)
+ private void possiblyAddHighIO(PipelineJob job, RemoteExecutionEngine engine, List lines)
{
Map params = ((HasJobParams)job).getJobParams();
String val = StringUtils.trimToNull(params.get("resourceSettings.resourceSettings.highIO"));
@@ -522,7 +522,7 @@ private void possiblyAddQOS(PipelineJob job, RemoteExecutionEngine engine, List<
job.getLogger().debug("qos as supplied by job: " + qos);
//Exempt specific task types:
- TaskFactory factory = job.getActiveTaskFactory();
+ TaskFactory factory = job.getActiveTaskFactory();
String activeTask = factory == null ? null : factory.getId().getNamespaceClass().getSimpleName();
job.getLogger().debug("Active task simplename: " + activeTask);
if (Arrays.asList("PrepareAlignerIndexesTask", "CacheAlignerIndexesTask", "AlignmentInitTask", "VariantProcessingScatterRemotePrepareTask").contains(activeTask))
@@ -610,7 +610,7 @@ private Long getFileSize(PipelineJob job)
}
@Override
- public void processJavaOpts(PipelineJob job, RemoteExecutionEngine engine, @NotNull List existingJavaOpts)
+ public void processJavaOpts(PipelineJob job, RemoteExecutionEngine engine, @NotNull List existingJavaOpts)
{
if (job instanceof HasJobParams)
{
diff --git a/tcrdb/resources/queries/tcrdb/clone_responses/.qview.xml b/tcrdb/resources/queries/tcrdb/clone_responses/.qview.xml
new file mode 100644
index 000000000..d07972bdb
--- /dev/null
+++ b/tcrdb/resources/queries/tcrdb/clone_responses/.qview.xml
@@ -0,0 +1,23 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tcrdb/resources/queries/tcrdb/clone_responses/Cohort Info.qview.xml b/tcrdb/resources/queries/tcrdb/clone_responses/Cohort Info.qview.xml
new file mode 100644
index 000000000..4e7d72c87
--- /dev/null
+++ b/tcrdb/resources/queries/tcrdb/clone_responses/Cohort Info.qview.xml
@@ -0,0 +1,24 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tcrdb/resources/queries/tcrdb/stims/.qview.xml b/tcrdb/resources/queries/tcrdb/stims/.qview.xml
index 65170b8d8..e00aaa911 100644
--- a/tcrdb/resources/queries/tcrdb/stims/.qview.xml
+++ b/tcrdb/resources/queries/tcrdb/stims/.qview.xml
@@ -11,6 +11,7 @@
+
diff --git a/tcrdb/resources/queries/tcrdb/stims/Cohort Info.qview.xml b/tcrdb/resources/queries/tcrdb/stims/Cohort Info.qview.xml
new file mode 100644
index 000000000..2cf6da768
--- /dev/null
+++ b/tcrdb/resources/queries/tcrdb/stims/Cohort Info.qview.xml
@@ -0,0 +1,22 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.54-15.55.sql b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.54-15.55.sql
new file mode 100644
index 000000000..db2c19102
--- /dev/null
+++ b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.54-15.55.sql
@@ -0,0 +1,7 @@
+ALTER TABLE tcrdb.clone_responses ADD vGene varchar(1000);
+ALTER TABLE tcrdb.clone_responses ADD totalCells int;
+ALTER TABLE tcrdb.clone_responses ADD totalCloneSize int;
+ALTER TABLE tcrdb.clone_responses ADD fractionCloneActivated double precision;
+ALTER TABLE tcrdb.clone_responses ADD totalCellsForSample int;
+ALTER TABLE tcrdb.clone_responses ADD oddsRatio double precision;
+ALTER TABLE tcrdb.clone_responses ADD enrichmentFDR double precision;
diff --git a/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.55-15.56.sql b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.55-15.56.sql
new file mode 100644
index 000000000..b42680c3d
--- /dev/null
+++ b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.55-15.56.sql
@@ -0,0 +1 @@
+ALTER TABLE tcrdb.clone_responses ADD jGene varchar(1000);
diff --git a/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.56-15.57.sql b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.56-15.57.sql
new file mode 100644
index 000000000..21758fc7e
--- /dev/null
+++ b/tcrdb/resources/schemas/dbscripts/postgresql/tcrdb-15.56-15.57.sql
@@ -0,0 +1 @@
+ALTER TABLE tcrdb.stims ADD nClones int;
diff --git a/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.54-15.55.sql b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.54-15.55.sql
new file mode 100644
index 000000000..db2c19102
--- /dev/null
+++ b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.54-15.55.sql
@@ -0,0 +1,7 @@
+ALTER TABLE tcrdb.clone_responses ADD vGene varchar(1000);
+ALTER TABLE tcrdb.clone_responses ADD totalCells int;
+ALTER TABLE tcrdb.clone_responses ADD totalCloneSize int;
+ALTER TABLE tcrdb.clone_responses ADD fractionCloneActivated double precision;
+ALTER TABLE tcrdb.clone_responses ADD totalCellsForSample int;
+ALTER TABLE tcrdb.clone_responses ADD oddsRatio double precision;
+ALTER TABLE tcrdb.clone_responses ADD enrichmentFDR double precision;
diff --git a/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.55-15.56.sql b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.55-15.56.sql
new file mode 100644
index 000000000..b42680c3d
--- /dev/null
+++ b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.55-15.56.sql
@@ -0,0 +1 @@
+ALTER TABLE tcrdb.clone_responses ADD jGene varchar(1000);
diff --git a/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.56-15.57.sql b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.56-15.57.sql
new file mode 100644
index 000000000..21758fc7e
--- /dev/null
+++ b/tcrdb/resources/schemas/dbscripts/sqlserver/tcrdb-15.56-15.57.sql
@@ -0,0 +1 @@
+ALTER TABLE tcrdb.stims ADD nClones int;
diff --git a/tcrdb/resources/schemas/tcrdb.xml b/tcrdb/resources/schemas/tcrdb.xml
index 88951dd56..60a93c0ad 100644
--- a/tcrdb/resources/schemas/tcrdb.xml
+++ b/tcrdb/resources/schemas/tcrdb.xml
@@ -134,6 +134,30 @@
Background Freq
+
+ V-Gene
+
+
+ J-Gene
+
+
+ Total Cells
+
+
+ Total Clone Size
+
+
+ Fraction of Clone Activated
+
+
+ Total Cells For Sample
+
+
+ Enrichment (Odds Ratio)
+
+
+ Enrichment (FDR)
+
@@ -296,6 +320,9 @@
Flow Quantification
+
+ # Clones
+
diff --git a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java
index dd7b9d912..d452b218b 100644
--- a/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java
+++ b/tcrdb/src/org/labkey/tcrdb/TCRdbModule.java
@@ -46,7 +46,7 @@ public String getName()
@Override
public Double getSchemaVersion()
{
- return 15.54;
+ return 15.57;
}
@Override