From 71e20ef97920196c2b961bbc0cce0b2bf0a2e03d Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Wed, 6 May 2026 10:28:23 -0700 Subject: [PATCH 1/3] Reduce logging for exporting large folder archives --- .../org/labkey/experiment/XarExporter.java | 80 ++++--------------- .../experiment/pipeline/XarGeneratorTask.java | 2 +- 2 files changed, 17 insertions(+), 65 deletions(-) diff --git a/experiment/src/org/labkey/experiment/XarExporter.java b/experiment/src/org/labkey/experiment/XarExporter.java index f3a658db3cd..49a311c8373 100644 --- a/experiment/src/org/labkey/experiment/XarExporter.java +++ b/experiment/src/org/labkey/experiment/XarExporter.java @@ -70,6 +70,7 @@ import org.labkey.api.util.DateUtil; import org.labkey.api.util.FileUtil; import org.labkey.api.util.Pair; +import org.labkey.api.util.logging.LogHelper; import org.labkey.data.xml.DerivationDataScopeTypes; import org.labkey.experiment.api.Data; import org.labkey.experiment.api.DataInput; @@ -105,7 +106,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.zip.ZipEntry; @@ -121,6 +121,7 @@ */ public class XarExporter { + private static final Logger DEFAULT_LOG = LogHelper.getLogger(XarExporter.class, "XAR (experiment archive) exporting"); private final URLRewriter _urlRewriter; private final User _user; private final String _fileRootPath; @@ -154,7 +155,7 @@ public class XarExporter private final Map _rootMaterialRowIdsToLSIDs = new LRUMap<>(1_000); private final LSIDRelativizer.RelativizedLSIDs _relativizedLSIDs; - private Logger _log; + private final Logger _log; private boolean _includeXML = true; @@ -166,7 +167,7 @@ public class XarExporter AssayProvider.XarCallbacks assayCallbacks = null; - public XarExporter(LSIDRelativizer.RelativizedLSIDs relativizedLSIDs, URLRewriter urlRewriter, User user, Container container) + private XarExporter(LSIDRelativizer.RelativizedLSIDs relativizedLSIDs, URLRewriter urlRewriter, User user, Container container, Logger log) { // UNDONE: Is it ok to share the relativizedLSIDs across XarExporters and tsv writers? _relativizedLSIDs = relativizedLSIDs; @@ -176,11 +177,12 @@ public XarExporter(LSIDRelativizer.RelativizedLSIDs relativizedLSIDs, URLRewrite _document = ExperimentArchiveDocument.Factory.newInstance(); _archive = _document.addNewExperimentArchive(); + _log = log; } public XarExporter(LSIDRelativizer lsidRelativizer, URLRewriter urlRewriter, User user, Container container) { - this(new LSIDRelativizer.RelativizedLSIDs(lsidRelativizer), urlRewriter, user, container); + this(new LSIDRelativizer.RelativizedLSIDs(lsidRelativizer), urlRewriter, user, container, DEFAULT_LOG); } public XarExporter(LSIDRelativizer lsidRelativizer, XarExportSelection selection, User user, String xarXmlFileName, Logger log, Container container) throws ExperimentException @@ -190,8 +192,7 @@ public XarExporter(LSIDRelativizer lsidRelativizer, XarExportSelection selection public XarExporter(LSIDRelativizer.RelativizedLSIDs relativizedLSIDs, XarExportSelection selection, User user, String xarXmlFileName, Logger log, Container container) throws ExperimentException { - this(relativizedLSIDs, selection.createURLRewriter(), user, container); - _log = log; + this(relativizedLSIDs, selection.createURLRewriter(), user, container, log); selection.addContent(this); @@ -202,14 +203,6 @@ public XarExporter(LSIDRelativizer.RelativizedLSIDs relativizedLSIDs, XarExportS setFileIncludes(selection.isIncludeXarXml()); } - private void logProgress(String message) - { - if (_log != null) - { - _log.info(message); - } - } - public void setFileIncludes(boolean xarXML) { _includeXML = xarXML; @@ -226,7 +219,7 @@ public void addExpData(ExpData data) throws ExperimentException { return; } - logProgress("Adding experiment data " + data.getRowId()); + _log.debug("Adding experiment data " + data.getRowId()); _expDataIDs.add(data.getRowId()); ArchiveURLRewriter u = (ArchiveURLRewriter)_urlRewriter; @@ -241,7 +234,7 @@ public void addExperimentRun(ExpRun run) throws ExperimentException { return; } - logProgress("Adding experiment run " + run.getLSID()); + _log.debug("Adding experiment run " + run.getLSID()); _experimentRunLSIDs.add(run.getLSID()); ExpExperiment batch = run.getBatch(); @@ -592,7 +585,7 @@ private boolean isDefaultCpasType(String cpasType, @NotNull String defaultType) private void populateMaterial(MaterialBaseType xMaterial, ExpMaterial material) throws ExperimentException { - logProgress("Adding material " + material.getLSID()); + _log.debug("Adding material " + material.getLSID()); addSampleType(material.getCpasType()); xMaterial.setAbout(_relativizedLSIDs.relativize(material.getLSID())); xMaterial.setCpasType(isDefaultCpasType(material.getCpasType(), ExpMaterial.DEFAULT_CPAS_TYPE) ? ExpMaterial.DEFAULT_CPAS_TYPE : _relativizedLSIDs.relativize(material.getCpasType())); @@ -985,7 +978,7 @@ private PropertyCollectionType addOriginalURLProperty(PropertyCollectionType pro private void populateData(DataBaseType xData, ExpData data, @Nullable String role, ExpRun run) throws ExperimentException { - logProgress("Adding data " + data.getLSID()); + _log.debug("Adding data " + data.getLSID()); xData.setName(data.getName()); xData.setAbout(_relativizedLSIDs.relativize(data)); xData.setCpasType(isDefaultCpasType(data.getCpasType(), ExpData.DEFAULT_CPAS_TYPE) ? ExpData.DEFAULT_CPAS_TYPE : _relativizedLSIDs.relativize(data.getCpasType())); @@ -1014,7 +1007,7 @@ public void addProtocol(ExpProtocol protocol, boolean includeChildren) throws Ex { return; } - logProgress("Adding protocol " + protocol.getLSID()); + _log.debug("Adding protocol " + protocol.getLSID()); _protocolLSIDs.add(protocol.getLSID()); ExperimentArchiveType.ProtocolDefinitions protocolDefs = _archive.getProtocolDefinitions(); @@ -1250,7 +1243,7 @@ public void addExperiment(ExpExperimentImpl exp) throws ExperimentException { return; } - logProgress("Adding experiment " + experiment.getLSID()); + _log.debug("Adding experiment " + experiment.getLSID()); Set runLsids = new HashSet<>(); for (ExpRun expRun : exp.getRuns()) { @@ -1422,7 +1415,7 @@ private PropertyCollectionType getProperties(Map propert } break; default: - logProgress("Warning: skipping export of " + value.getName() + " -- unknown type " + value.getPropertyType()); + _log.warn("Warning: skipping export of " + value.getName() + " -- unknown type " + value.getPropertyType()); } } } @@ -1516,7 +1509,7 @@ public void writeAsArchive(OutputStream out) throws IOException, ExperimentExcep { ZipEntry xmlEntry = new ZipEntry(_xarXmlFileName); zOut.putNextEntry(xmlEntry); - logProgress("Adding XAR XML to archive"); + _log.info("Adding XAR XML to archive"); dumpXML(zOut); zOut.closeEntry(); } @@ -1525,7 +1518,7 @@ public void writeAsArchive(OutputStream out) throws IOException, ExperimentExcep { if (fileInfo.hasContentToExport()) { - logProgress("Adding data file to archive: " + fileInfo.getName()); + _log.debug("Adding data file to archive: " + fileInfo.getName()); ZipEntry fileEntry = new ZipEntry(fileInfo.getName()); zOut.putNextEntry(fileEntry); @@ -1549,47 +1542,6 @@ public void writeAsArchive(OutputStream out) throws IOException, ExperimentExcep } } - public void writeAsDirectory(File dir) throws IOException - { - try - { - if (_includeXML) - { - File xmlEntry = FileUtil.appendName(dir, _xarXmlFileName); - logProgress("Writing XAR XML file"); - try (FileOutputStream os = new FileOutputStream(xmlEntry)) - { - dumpXML(os); - } - } - - for (URLRewriter.FileInfo fileInfo : _urlRewriter.getFileInfos()) - { - if (fileInfo.hasContentToExport()) - { - logProgress("Adding data file to archive: " + fileInfo.getName()); - File fileEntry = new File(dir, fileInfo.getName()); - FileUtil.mkdirs(fileEntry.getParentFile()); - try (FileOutputStream os = new FileOutputStream(fileEntry)) - { - fileInfo.writeFile(os); - } - } - } - } - catch (Exception e) - { - // insert the stack trace into the zip file - File errorEntry = FileUtil.appendName(dir,"error.log"); - - try (FileOutputStream os = new FileOutputStream(errorEntry); PrintStream ps = new PrintStream(os)) - { - ps.println("Failed to complete export of the XAR file: "); - e.printStackTrace(ps); - } - } - } - private ContactType getContactType(String parentLSID, Container parentContainer) throws ExperimentException { Map parentProperties = getProperties(parentContainer, parentLSID); diff --git a/experiment/src/org/labkey/experiment/pipeline/XarGeneratorTask.java b/experiment/src/org/labkey/experiment/pipeline/XarGeneratorTask.java index e7ac1fddd5f..a1aadd46225 100644 --- a/experiment/src/org/labkey/experiment/pipeline/XarGeneratorTask.java +++ b/experiment/src/org/labkey/experiment/pipeline/XarGeneratorTask.java @@ -71,7 +71,7 @@ public Factory() } @Override - public PipelineJob.Task createTask(PipelineJob job) + public XarGeneratorTask createTask(PipelineJob job) { return new XarGeneratorTask(this, job); } From 81cd7f7f66df63f55548c2ee08af4772b54f7b74 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Wed, 6 May 2026 14:41:01 -0700 Subject: [PATCH 2/3] Cleanup --- experiment/src/org/labkey/experiment/XarExporter.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/experiment/src/org/labkey/experiment/XarExporter.java b/experiment/src/org/labkey/experiment/XarExporter.java index 49a311c8373..f213e934b44 100644 --- a/experiment/src/org/labkey/experiment/XarExporter.java +++ b/experiment/src/org/labkey/experiment/XarExporter.java @@ -86,8 +86,6 @@ import org.labkey.experiment.xar.XarExportSelection; import javax.xml.namespace.QName; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.PrintStream; @@ -177,7 +175,7 @@ private XarExporter(LSIDRelativizer.RelativizedLSIDs relativizedLSIDs, URLRewrit _document = ExperimentArchiveDocument.Factory.newInstance(); _archive = _document.addNewExperimentArchive(); - _log = log; + _log = log == null ? DEFAULT_LOG : log; } public XarExporter(LSIDRelativizer lsidRelativizer, URLRewriter urlRewriter, User user, Container container) @@ -1415,7 +1413,7 @@ private PropertyCollectionType getProperties(Map propert } break; default: - _log.warn("Warning: skipping export of " + value.getName() + " -- unknown type " + value.getPropertyType()); + _log.warn("Skipping export of {} -- unknown type {}", value.getName(), value.getPropertyType()); } } } From 5c534cb57f55f458fa68b4f904cc08dd4f48f1e5 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Wed, 6 May 2026 14:51:05 -0700 Subject: [PATCH 3/3] Parameterize consistently --- .../src/org/labkey/experiment/XarExporter.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/experiment/src/org/labkey/experiment/XarExporter.java b/experiment/src/org/labkey/experiment/XarExporter.java index f213e934b44..c2159405c05 100644 --- a/experiment/src/org/labkey/experiment/XarExporter.java +++ b/experiment/src/org/labkey/experiment/XarExporter.java @@ -217,7 +217,7 @@ public void addExpData(ExpData data) throws ExperimentException { return; } - _log.debug("Adding experiment data " + data.getRowId()); + _log.debug("Adding experiment data {}", data.getRowId()); _expDataIDs.add(data.getRowId()); ArchiveURLRewriter u = (ArchiveURLRewriter)_urlRewriter; @@ -232,7 +232,7 @@ public void addExperimentRun(ExpRun run) throws ExperimentException { return; } - _log.debug("Adding experiment run " + run.getLSID()); + _log.debug("Adding experiment run {}", run.getLSID()); _experimentRunLSIDs.add(run.getLSID()); ExpExperiment batch = run.getBatch(); @@ -583,7 +583,7 @@ private boolean isDefaultCpasType(String cpasType, @NotNull String defaultType) private void populateMaterial(MaterialBaseType xMaterial, ExpMaterial material) throws ExperimentException { - _log.debug("Adding material " + material.getLSID()); + _log.debug("Adding material {}", material.getLSID()); addSampleType(material.getCpasType()); xMaterial.setAbout(_relativizedLSIDs.relativize(material.getLSID())); xMaterial.setCpasType(isDefaultCpasType(material.getCpasType(), ExpMaterial.DEFAULT_CPAS_TYPE) ? ExpMaterial.DEFAULT_CPAS_TYPE : _relativizedLSIDs.relativize(material.getCpasType())); @@ -976,7 +976,7 @@ private PropertyCollectionType addOriginalURLProperty(PropertyCollectionType pro private void populateData(DataBaseType xData, ExpData data, @Nullable String role, ExpRun run) throws ExperimentException { - _log.debug("Adding data " + data.getLSID()); + _log.debug("Adding data {}", data.getLSID()); xData.setName(data.getName()); xData.setAbout(_relativizedLSIDs.relativize(data)); xData.setCpasType(isDefaultCpasType(data.getCpasType(), ExpData.DEFAULT_CPAS_TYPE) ? ExpData.DEFAULT_CPAS_TYPE : _relativizedLSIDs.relativize(data.getCpasType())); @@ -1005,7 +1005,7 @@ public void addProtocol(ExpProtocol protocol, boolean includeChildren) throws Ex { return; } - _log.debug("Adding protocol " + protocol.getLSID()); + _log.debug("Adding protocol {}", protocol.getLSID()); _protocolLSIDs.add(protocol.getLSID()); ExperimentArchiveType.ProtocolDefinitions protocolDefs = _archive.getProtocolDefinitions(); @@ -1241,7 +1241,7 @@ public void addExperiment(ExpExperimentImpl exp) throws ExperimentException { return; } - _log.debug("Adding experiment " + experiment.getLSID()); + _log.debug("Adding experiment {}", experiment.getLSID()); Set runLsids = new HashSet<>(); for (ExpRun expRun : exp.getRuns()) { @@ -1516,7 +1516,7 @@ public void writeAsArchive(OutputStream out) throws IOException, ExperimentExcep { if (fileInfo.hasContentToExport()) { - _log.debug("Adding data file to archive: " + fileInfo.getName()); + _log.debug("Adding data file to archive: {}", fileInfo.getName()); ZipEntry fileEntry = new ZipEntry(fileInfo.getName()); zOut.putNextEntry(fileEntry);