Skip to content

Reduce logging for exporting large folder archives #7649

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
labkey-jeckels merged 3 commits into from
May 7, 2026
+17 −67
Merged

Reduce logging for exporting large folder archives #7649

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
71e20ef
Reduce logging for exporting large folder archives
labkey-jeckels May 6, 2026
81cd7f7
Cleanup
labkey-jeckels May 6, 2026
5c534cb
Parameterize consistently
labkey-jeckels May 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 16 additions & 66 deletions experiment/src/org/labkey/experiment/XarExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
import org.labkey.api.util.DateUtil;
import org.labkey.api.util.FileUtil;
import org.labkey.api.util.Pair;
import org.labkey.api.util.logging.LogHelper;
import org.labkey.data.xml.DerivationDataScopeTypes;
import org.labkey.experiment.api.Data;
import org.labkey.experiment.api.DataInput;
Expand All @@ -85,8 +86,6 @@
import org.labkey.experiment.xar.XarExportSelection;

import javax.xml.namespace.QName;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
Expand All @@ -105,7 +104,6 @@
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.zip.ZipEntry;
Expand All @@ -121,6 +119,7 @@
*/
public class XarExporter
{
private static final Logger DEFAULT_LOG = LogHelper.getLogger(XarExporter.class, "XAR (experiment archive) exporting");
private final URLRewriter _urlRewriter;
private final User _user;
private final String _fileRootPath;
Expand Down Expand Up @@ -154,7 +153,7 @@ public class XarExporter
private final Map<Long, String> _rootMaterialRowIdsToLSIDs = new LRUMap<>(1_000);

private final LSIDRelativizer.RelativizedLSIDs _relativizedLSIDs;
private Logger _log;
private final Logger _log;

private boolean _includeXML = true;

Expand All @@ -166,7 +165,7 @@ public class XarExporter
AssayProvider.XarCallbacks assayCallbacks = null;


public XarExporter(LSIDRelativizer.RelativizedLSIDs relativizedLSIDs, URLRewriter urlRewriter, User user, Container container)
private XarExporter(LSIDRelativizer.RelativizedLSIDs relativizedLSIDs, URLRewriter urlRewriter, User user, Container container, Logger log)
{
// UNDONE: Is it ok to share the relativizedLSIDs across XarExporters and tsv writers?
_relativizedLSIDs = relativizedLSIDs;
Expand All @@ -176,11 +175,12 @@ public XarExporter(LSIDRelativizer.RelativizedLSIDs relativizedLSIDs, URLRewrite

_document = ExperimentArchiveDocument.Factory.newInstance();
_archive = _document.addNewExperimentArchive();
_log = log == null ? DEFAULT_LOG : log;
}

public XarExporter(LSIDRelativizer lsidRelativizer, URLRewriter urlRewriter, User user, Container container)
{
this(new LSIDRelativizer.RelativizedLSIDs(lsidRelativizer), urlRewriter, user, container);
this(new LSIDRelativizer.RelativizedLSIDs(lsidRelativizer), urlRewriter, user, container, DEFAULT_LOG);
}

public XarExporter(LSIDRelativizer lsidRelativizer, XarExportSelection selection, User user, String xarXmlFileName, Logger log, Container container) throws ExperimentException
Expand All @@ -190,8 +190,7 @@ public XarExporter(LSIDRelativizer lsidRelativizer, XarExportSelection selection

public XarExporter(LSIDRelativizer.RelativizedLSIDs relativizedLSIDs, XarExportSelection selection, User user, String xarXmlFileName, Logger log, Container container) throws ExperimentException
{
this(relativizedLSIDs, selection.createURLRewriter(), user, container);
_log = log;
this(relativizedLSIDs, selection.createURLRewriter(), user, container, log);
Copy link
Copy Markdown
Contributor

@labkey-susanh labkey-susanh May 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should there be a @NotNull on the Logger argument here?

Copy link
Copy Markdown
Contributor Author

@labkey-jeckels labkey-jeckels May 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point. I switched to use the default logger when null is passed. That's a small behavior change but should be reasonable given that almost everything has moved to debug level.


selection.addContent(this);

Expand All @@ -202,14 +201,6 @@ public XarExporter(LSIDRelativizer.RelativizedLSIDs relativizedLSIDs, XarExportS
setFileIncludes(selection.isIncludeXarXml());
}

private void logProgress(String message)
{
if (_log != null)
{
_log.info(message);
}
}

public void setFileIncludes(boolean xarXML)
{
_includeXML = xarXML;
Expand All @@ -226,7 +217,7 @@ public void addExpData(ExpData data) throws ExperimentException
{
return;
}
logProgress("Adding experiment data " + data.getRowId());
_log.debug("Adding experiment data {}", data.getRowId());
_expDataIDs.add(data.getRowId());

ArchiveURLRewriter u = (ArchiveURLRewriter)_urlRewriter;
Expand All @@ -241,7 +232,7 @@ public void addExperimentRun(ExpRun run) throws ExperimentException
{
return;
}
logProgress("Adding experiment run " + run.getLSID());
_log.debug("Adding experiment run {}", run.getLSID());
_experimentRunLSIDs.add(run.getLSID());

ExpExperiment batch = run.getBatch();
Expand Down Expand Up @@ -592,7 +583,7 @@ private boolean isDefaultCpasType(String cpasType, @NotNull String defaultType)

private void populateMaterial(MaterialBaseType xMaterial, ExpMaterial material) throws ExperimentException
{
logProgress("Adding material " + material.getLSID());
_log.debug("Adding material {}", material.getLSID());
addSampleType(material.getCpasType());
xMaterial.setAbout(_relativizedLSIDs.relativize(material.getLSID()));
xMaterial.setCpasType(isDefaultCpasType(material.getCpasType(), ExpMaterial.DEFAULT_CPAS_TYPE) ? ExpMaterial.DEFAULT_CPAS_TYPE : _relativizedLSIDs.relativize(material.getCpasType()));
Expand Down Expand Up @@ -985,7 +976,7 @@ private PropertyCollectionType addOriginalURLProperty(PropertyCollectionType pro

private void populateData(DataBaseType xData, ExpData data, @Nullable String role, ExpRun run) throws ExperimentException
{
logProgress("Adding data " + data.getLSID());
_log.debug("Adding data {}", data.getLSID());
xData.setName(data.getName());
xData.setAbout(_relativizedLSIDs.relativize(data));
xData.setCpasType(isDefaultCpasType(data.getCpasType(), ExpData.DEFAULT_CPAS_TYPE) ? ExpData.DEFAULT_CPAS_TYPE : _relativizedLSIDs.relativize(data.getCpasType()));
Expand Down Expand Up @@ -1014,7 +1005,7 @@ public void addProtocol(ExpProtocol protocol, boolean includeChildren) throws Ex
{
return;
}
logProgress("Adding protocol " + protocol.getLSID());
_log.debug("Adding protocol {}", protocol.getLSID());
_protocolLSIDs.add(protocol.getLSID());

ExperimentArchiveType.ProtocolDefinitions protocolDefs = _archive.getProtocolDefinitions();
Expand Down Expand Up @@ -1250,7 +1241,7 @@ public void addExperiment(ExpExperimentImpl exp) throws ExperimentException
{
return;
}
logProgress("Adding experiment " + experiment.getLSID());
_log.debug("Adding experiment {}", experiment.getLSID());
Set<String> runLsids = new HashSet<>();
for (ExpRun expRun : exp.getRuns())
{
Expand Down Expand Up @@ -1422,7 +1413,7 @@ private PropertyCollectionType getProperties(Map<String, ObjectProperty> propert
}
break;
default:
logProgress("Warning: skipping export of " + value.getName() + " -- unknown type " + value.getPropertyType());
_log.warn("Skipping export of {} -- unknown type {}", value.getName(), value.getPropertyType());
}
}
}
Expand Down Expand Up @@ -1516,7 +1507,7 @@ public void writeAsArchive(OutputStream out) throws IOException, ExperimentExcep
{
ZipEntry xmlEntry = new ZipEntry(_xarXmlFileName);
zOut.putNextEntry(xmlEntry);
logProgress("Adding XAR XML to archive");
_log.info("Adding XAR XML to archive");
dumpXML(zOut);
zOut.closeEntry();
}
Expand All @@ -1525,7 +1516,7 @@ public void writeAsArchive(OutputStream out) throws IOException, ExperimentExcep
{
if (fileInfo.hasContentToExport())
{
logProgress("Adding data file to archive: " + fileInfo.getName());
_log.debug("Adding data file to archive: {}", fileInfo.getName());
ZipEntry fileEntry = new ZipEntry(fileInfo.getName());
zOut.putNextEntry(fileEntry);

Expand All @@ -1549,47 +1540,6 @@ public void writeAsArchive(OutputStream out) throws IOException, ExperimentExcep
}
}

public void writeAsDirectory(File dir) throws IOException
{
try
{
if (_includeXML)
{
File xmlEntry = FileUtil.appendName(dir, _xarXmlFileName);
logProgress("Writing XAR XML file");
try (FileOutputStream os = new FileOutputStream(xmlEntry))
{
dumpXML(os);
}
}

for (URLRewriter.FileInfo fileInfo : _urlRewriter.getFileInfos())
{
if (fileInfo.hasContentToExport())
{
logProgress("Adding data file to archive: " + fileInfo.getName());
File fileEntry = new File(dir, fileInfo.getName());
FileUtil.mkdirs(fileEntry.getParentFile());
try (FileOutputStream os = new FileOutputStream(fileEntry))
{
fileInfo.writeFile(os);
}
}
}
}
catch (Exception e)
{
// insert the stack trace into the zip file
File errorEntry = FileUtil.appendName(dir,"error.log");

try (FileOutputStream os = new FileOutputStream(errorEntry); PrintStream ps = new PrintStream(os))
{
ps.println("Failed to complete export of the XAR file: ");
e.printStackTrace(ps);
}
}
}

private ContactType getContactType(String parentLSID, Container parentContainer) throws ExperimentException
{
Map<String, Object> parentProperties = getProperties(parentContainer, parentLSID);
Expand Down
2 changes: 1 addition & 1 deletion experiment/src/org/labkey/experiment/pipeline/XarGeneratorTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ public Factory()
}

@Override
public PipelineJob.Task createTask(PipelineJob job)
public XarGeneratorTask createTask(PipelineJob job)
{
return new XarGeneratorTask(this, job);
}
Expand Down
Loading