Merge remote-tracking branch 'upstream/master' into enh/2210

Author: emdupre

.circle/tests.sh

@@ -17,8 +17,8 @@ fi
 # They may need to be rebalanced in the future.
 case ${CIRCLE_NODE_INDEX} in
   0)
-    docker run --rm=false -it -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_pytests.sh && \
-    docker run --rm=false -it -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py27 /usr/bin/run_pytests.sh && \
+    docker run --rm=false -it -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_pytests.sh && \
+    docker run --rm=false -it -e CI_SKIP_TEST=1 -e NIPYPE_RESOURCE_MONITOR=1 -e FSL_COURSE_DATA="/data/examples/nipype-fsl_course_data" -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py27 /usr/bin/run_pytests.sh && \
     docker run --rm=false -it -v $WORKDIR:/work -w /src/nipype/doc --entrypoint=/usr/bin/run_builddocs.sh nipype/nipype:py36 /usr/bin/run_builddocs.sh && \
     docker run --rm=false -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow3d && \
     docker run --rm=false -it -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_examples.sh test_spm Linear /data/examples/ workflow4d
@@ -30,8 +30,8 @@ case ${CIRCLE_NODE_INDEX} in
     exitcode=$?
     ;;
   2)
-    docker run --rm=false -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py27 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 && \
-    docker run --rm=false -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline
+    docker run --rm=false -it -e NIPYPE_NUMBER_OF_CPUS=4 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py36 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ level1 && \
+    docker run --rm=false -it -e NIPYPE_NUMBER_OF_CPUS=4 -e NIPYPE_RESOURCE_MONITOR=1 -v $HOME/examples:/data/examples:ro -v $WORKDIR:/work -w /work nipype/nipype:py27 /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline
     exitcode=$?
     ;;
   3)

.travis.yml

@@ -8,10 +8,10 @@ python:
 - 3.5
 - 3.6
 env:
-- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler"
-- INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler"
-- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit"
-- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler" PIP_FLAGS="--pre"
+- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler" CI_SKIP_TEST=1
+- INSTALL_DEB_DEPENDECIES=false NIPYPE_EXTRAS="doc,tests,fmri,profiler" CI_SKIP_TEST=1
+- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler,duecredit" CI_SKIP_TEST=1
+- INSTALL_DEB_DEPENDECIES=true NIPYPE_EXTRAS="doc,tests,fmri,profiler" PIP_FLAGS="--pre" CI_SKIP_TEST=1
 before_install:
 - function apt_inst {
   if $INSTALL_DEB_DEPENDECIES; then sudo rm -rf /dev/shm; fi &&

CHANGES

@@ -1,6 +1,10 @@
 Upcoming release
 ================
 
+* ENH: Improve terminal_output feature (https://github.com/nipy/nipype/pull/2209)
+* ENH: Simple interface to FSL std2imgcoords (https://github.com/nipy/nipype/pull/2209, prev #1398)
+* ENH: Centralize virtual/physical $DISPLAYs (https://github.com/nipy/nipype/pull/#2203)
+* ENH: New ResourceMonitor - replaces resource profiler (https://github.com/nipy/nipype/pull/#2200)
 
 0.13.1 (May 20, 2017)
 =====================

doc/devel/interface_specs.rst

@@ -159,6 +159,70 @@ generated depending on inputs, by the tool.  OutputSpecs inherit from
 ``interfaces.base.TraitedSpec`` directly.
 
 
+Controlling outputs to terminal
+-------------------------------
+
+It is very likely that the software wrapped within the interface writes
+to the standard output or the standard error of the terminal.
+Interfaces provide a means to access and retrieve these outputs, by
+using the ``terminal_output`` attribute: ::
+
+  import nipype.interfaces.fsl as fsl
+  mybet = fsl.BET(from_file='bet-settings.json')
+  mybet.terminal_output = 'file_split'
+
+In the example, the ``terminal_output = 'file_split'`` will redirect the
+standard output and the standard error to split files (called
+``stdout.nipype`` and ``stderr.nipype`` respectively).
+The possible values for ``terminal_output`` are:
+
+*file*
+    Redirects both standard output and standard error to the same file
+    called ``output.nipype``.
+    Messages from both streams will be overlapped as they arrive to
+    the file.
+
+*file_split*
+    Redirects the output streams separately, to ``stdout.nipype``
+    and ``stderr.nipype`` respectively, as described in the example.
+
+*file_stdout*
+    Only the standard output will be redirected to ``stdout.nipype``
+    and the standard error will be discarded.
+
+*file_stderr*
+    Only the standard error will be redirected to ``stderr.nipype``
+    and the standard output will be discarded.
+
+*stream*
+    Both output streams are redirected to the current logger printing
+    their messages interleaved and immediately to the terminal.
+
+*allatonce*
+    Both output streams will be forwarded to a buffer and stored
+    separately in the `runtime` object that the `run()` method returns.
+    No files are written nor streams printed out to terminal.
+
+*none*
+    Both outputs are discarded
+
+In all cases, except for the ``'none'`` setting of ``terminal_output``,
+the ``run()`` method will return a "runtime" object that will contain
+the streams in the corresponding properties (``runtime.stdout``
+for the standard output, ``runtime.stderr`` for the standard error, and
+``runtime.merged`` for both when streams are mixed, eg. when using the
+*file* option). ::
+
+  import nipype.interfaces.fsl as fsl
+  mybet = fsl.BET(from_file='bet-settings.json')
+  mybet.terminal_output = 'file_split'
+  ...
+  result = mybet.run()
+  result.runtime.stdout
+  ' ... captured standard output ...'
+
+
+
 Traited Attributes
 ------------------
 

doc/users/config_file.rst

@@ -14,93 +14,100 @@ Logging
 ~~~~~~~
 
 *workflow_level*
-	How detailed the logs regarding workflow should be (possible values:
-	``INFO`` and ``DEBUG``; default value: ``INFO``)
-*filemanip_level*
-	How detailed the logs regarding file operations (for example overwriting
-	warning) should be (possible values: ``INFO`` and ``DEBUG``; default value:
-	``INFO``)
+    How detailed the logs regarding workflow should be (possible values:
+    ``INFO`` and ``DEBUG``; default value: ``INFO``)
+*utils_level*
+    How detailed the logs regarding nipype utils, like file operations
+    (for example overwriting warning) or the resource profiler, should be
+    (possible values: ``INFO`` and ``DEBUG``; default value:
+    ``INFO``)
 *interface_level*
-	How detailed the logs regarding interface execution should be (possible
-	values: ``INFO`` and ``DEBUG``; default value: ``INFO``)
+    How detailed the logs regarding interface execution should be (possible
+    values: ``INFO`` and ``DEBUG``; default value: ``INFO``)
+*filemanip_level* (deprecated as of 1.0)
+    How detailed the logs regarding file operations (for example overwriting
+    warning) should be (possible values: ``INFO`` and ``DEBUG``)
 *log_to_file*
     Indicates whether logging should also send the output to a file (possible
     values: ``true`` and ``false``; default value: ``false``)
 *log_directory*
-	Where to store logs. (string, default value: home directory)
+    Where to store logs. (string, default value: home directory)
 *log_size*
-	Size of a single log file. (integer, default value: 254000)
+    Size of a single log file. (integer, default value: 254000)
 *log_rotate*
-	How many rotation should the log file make. (integer, default value: 4)
+    How many rotation should the log file make. (integer, default value: 4)
 
 Execution
 ~~~~~~~~~
 
 *plugin*
-	This defines which execution plugin to use. (possible values: ``Linear``,
-	``MultiProc``, ``SGE``, ``IPython``; default value: ``Linear``)
+    This defines which execution plugin to use. (possible values: ``Linear``,
+    ``MultiProc``, ``SGE``, ``IPython``; default value: ``Linear``)
 
 *stop_on_first_crash*
-	Should the workflow stop upon first node crashing or try to execute as many
-	nodes as possible? (possible values: ``true`` and ``false``; default value:
-	``false``)
+    Should the workflow stop upon first node crashing or try to execute as many
+    nodes as possible? (possible values: ``true`` and ``false``; default value:
+    ``false``)
 
 *stop_on_first_rerun*
-	Should the workflow stop upon first node trying to recompute (by that we
-	mean rerunning a node that has been run before - this can happen due changed
-	inputs and/or hash_method since the last run). (possible values: ``true``
-	and ``false``; default value: ``false``)
+    Should the workflow stop upon first node trying to recompute (by that we
+    mean rerunning a node that has been run before - this can happen due changed
+    inputs and/or hash_method since the last run). (possible values: ``true``
+    and ``false``; default value: ``false``)
 
 *hash_method*
-	Should the input files be checked for changes using their content (slow, but
-	100% accurate) or just their size and modification date (fast, but
-	potentially prone to errors)? (possible values: ``content`` and
-	``timestamp``; default value: ``timestamp``)
+    Should the input files be checked for changes using their content (slow, but
+    100% accurate) or just their size and modification date (fast, but
+    potentially prone to errors)? (possible values: ``content`` and
+    ``timestamp``; default value: ``timestamp``)
 
 *keep_inputs*
     Ensures that all inputs that are created in the nodes working directory are
     kept after node execution (possible values: ``true`` and ``false``; default
     value: ``false``)
 
 *single_thread_matlab*
-	Should all of the Matlab interfaces (including SPM) use only one thread?
-	This is useful if you are parallelizing your workflow using MultiProc or
-	IPython on a single multicore machine. (possible values: ``true`` and
-	``false``; default value: ``true``)
+    Should all of the Matlab interfaces (including SPM) use only one thread?
+    This is useful if you are parallelizing your workflow using MultiProc or
+    IPython on a single multicore machine. (possible values: ``true`` and
+    ``false``; default value: ``true``)
 
 *display_variable*
-	What ``DISPLAY`` variable should all command line interfaces be
-	run with. This is useful if you are using `xnest
-	<http://www.x.org/archive/X11R7.5/doc/man/man1/Xnest.1.html>`_
-	or `Xvfb <http://www.x.org/archive/X11R6.8.1/doc/Xvfb.1.html>`_
-	and you would like to redirect all spawned windows to
-	it. (possible values: any X server address; default value: not
-	set)
+	Override the ``$DISPLAY`` environment variable for interfaces that require
+    an X server. This option is useful if there is a running X server, but 
+    ``$DISPLAY`` was not defined in nipype's environment. For example, if an X 
+    server is listening on the default port of 6000, set ``display_variable = :0``
+    to enable nipype interfaces to use it. It may also point to displays provided 
+    by VNC, `xnest <http://www.x.org/archive/X11R7.5/doc/man/man1/Xnest.1.html>`_ 
+    or `Xvfb <http://www.x.org/archive/X11R6.8.1/doc/Xvfb.1.html>`_.
+    If neither ``display_variable`` nor the ``$DISPLAY`` environment variable are
+    set, nipype will try to configure a new virtual server using Xvfb.
+    (possible values: any X server address; default value: not set)
 
 *remove_unnecessary_outputs*
-	This will remove any interface outputs not needed by the workflow. If the
-	required outputs from a node changes, rerunning the workflow will rerun the
-	node. Outputs of leaf nodes (nodes whose outputs are not connected to any
-	other nodes) will never be deleted independent of this parameter. (possible
-	values: ``true`` and ``false``; default value: ``true``)
+    This will remove any interface outputs not needed by the workflow. If the
+    required outputs from a node changes, rerunning the workflow will rerun the
+    node. Outputs of leaf nodes (nodes whose outputs are not connected to any
+    other nodes) will never be deleted independent of this parameter. (possible
+    values: ``true`` and ``false``; default value: ``true``)
 
 *try_hard_link_datasink*
-	When the DataSink is used to produce an orginized output file outside
-	of nipypes internal cache structure, a file system hard link will be
-	attempted first. A hard link allow multiple file paths to point to the
-	same physical storage location on disk if the conditions allow. By
-	refering to the same physical file on disk (instead of copying files
-	byte-by-byte) we can avoid unnecessary data duplication.  If hard links
-	are not supported for the source or destination paths specified, then
-	a standard byte-by-byte copy is used.  (possible values: ``true`` and
-	``false``; default value: ``true``)
+    When the DataSink is used to produce an orginized output file outside
+    of nipypes internal cache structure, a file system hard link will be
+    attempted first. A hard link allow multiple file paths to point to the
+    same physical storage location on disk if the conditions allow. By
+    refering to the same physical file on disk (instead of copying files
+    byte-by-byte) we can avoid unnecessary data duplication.  If hard links
+    are not supported for the source or destination paths specified, then
+    a standard byte-by-byte copy is used.  (possible values: ``true`` and
+    ``false``; default value: ``true``)
 
 *use_relative_paths*
-	Should the paths stored in results (and used to look for inputs)
-	be relative or absolute. Relative paths allow moving the whole
-	working directory around but may cause problems with
-	symlinks. (possible values: ``true`` and ``false``; default
-	value: ``false``)
+    Should the paths stored in results (and used to look for inputs)
+    be relative or absolute. Relative paths allow moving the whole
+    working directory around but may cause problems with
+    symlinks. (possible values: ``true`` and ``false``; default
+    value: ``false``)
 
 *local_hash_check*
     Perform the hash check on the job submission machine. This option minimizes
@@ -115,10 +122,10 @@ Execution
     done after a job finish is detected. (float in seconds; default value: 5)
 
 *remove_node_directories (EXPERIMENTAL)*
-	Removes directories whose outputs have already been used
-	up. Doesn't work with IdentiInterface or any node that patches
-	data through (without copying) (possible values: ``true`` and
-	``false``; default value: ``false``)
+    Removes directories whose outputs have already been used
+    up. Doesn't work with IdentiInterface or any node that patches
+    data through (without copying) (possible values: ``true`` and
+    ``false``; default value: ``false``)
 
 *stop_on_unknown_version*
     If this is set to True, an underlying interface will raise an error, when no
@@ -146,18 +153,27 @@ Execution
     crashfiles allow portability across machines and shorter load time.
     (possible values: ``pklz`` and ``txt``; default value: ``pklz``)
 
+*resource_monitor*
+    Enables monitoring the resources occupation (possible values: ``true`` and
+    ``false``; default value: ``false``)
+
+*resource_monitor_frequency*
+    Sampling period (in seconds) between measurements of resources (memory, cpus)
+    being used by an interface. Requires ``resource_monitor`` to be ``true``.
+    (default value: ``1``)
+
 Example
 ~~~~~~~
 
 ::
 
-	[logging]
-	workflow_level = DEBUG
+    [logging]
+    workflow_level = DEBUG
 
-	[execution]
-	stop_on_first_crash = true
-	hash_method = timestamp
-	display_variable = :1
+    [execution]
+    stop_on_first_crash = true
+    hash_method = timestamp
+    display_variable = :1
 
 Workflow.config property has a form of a nested dictionary reflecting the
 structure of the .cfg file.

doc/users/interface_tutorial.rst

@@ -10,7 +10,7 @@ Specifying input settings
 The nipype interface modules provide a Python interface to external
 packages like FSL_ and SPM_.  Within the module are a series of Python
 classes which wrap specific package functionality.  For example, in
-the fsl module, the class :class:`nipype.interfaces.fsl.Bet` wraps the
+the fsl module, the class :class:`nipype.interfaces.fsl.BET` wraps the
 ``bet`` command-line tool.  Using the command-line tool, one would
 specify input settings using flags like ``-o``, ``-m``, ``-f <f>``, etc...
 However, in nipype, options are assigned to Python attributes and can

doc/users/plugins.rst

@@ -74,6 +74,14 @@ Optional arguments::
   n_procs :  Number of processes to launch in parallel, if not set number of
   processors/threads will be automatically detected
 
+  memory_gb : Total memory available to be shared by all simultaneous tasks
+  currently running, if not set it will be automatically set to 90\% of
+  system RAM.
+
+  raise_insufficient : Raise exception when the estimated resources of a node
+  exceed the total amount of resources available (memory and threads), when
+  ``False`` (default), only a warning will be issued.
+
 To distribute processing on a multicore machine, simply call::
 
   workflow.run(plugin='MultiProc')

doc/users/resource_sched_profiler.rst

@@ -82,7 +82,7 @@ by setting the ``status_callback`` parameter to point to this function in the
 
 ::
 
-	from nipype.pipeline.plugins.callback_log import log_nodes_cb
+	from nipype.utils.profiler import log_nodes_cb
 	args_dict = {'n_procs' : 8, 'memory_gb' : 10, 'status_callback' : log_nodes_cb}
 
 To set the filepath for the callback log the ``'callback'`` logger must be
@@ -141,7 +141,7 @@ The pandas_ Python package is required to use this feature.
 
 ::
 
-	from nipype.pipeline.plugins.callback_log import log_nodes_cb
+	from nipype.utils.profiler import log_nodes_cb
 	args_dict = {'n_procs' : 8, 'memory_gb' : 10, 'status_callback' : log_nodes_cb}
 	workflow.run(plugin='MultiProc', plugin_args=args_dict)
 

doc/users/saving_workflows.rst

@@ -82,20 +82,20 @@ This will create a file "outputtestsave.py" with the following content:
 	bet2.inputs.environ = {'FSLOUTPUTTYPE': 'NIFTI_GZ'}
 	bet2.inputs.ignore_exception = False
 	bet2.inputs.output_type = 'NIFTI_GZ'
-	bet2.inputs.terminal_output = 'stream'
+	bet2.terminal_output = 'stream'
 	# Node: testsave.bet
 	bet = Node(BET(), name="bet")
 	bet.iterables = ('frac', [0.3, 0.4])
 	bet.inputs.environ = {'FSLOUTPUTTYPE': 'NIFTI_GZ'}
 	bet.inputs.ignore_exception = False
 	bet.inputs.output_type = 'NIFTI_GZ'
-	bet.inputs.terminal_output = 'stream'
+	bet.terminal_output = 'stream'
 	# Node: testsave.maths
 	maths = Node(ImageMaths(), name="maths")
 	maths.inputs.environ = {'FSLOUTPUTTYPE': 'NIFTI_GZ'}
 	maths.inputs.ignore_exception = False
 	maths.inputs.output_type = 'NIFTI_GZ'
-	maths.inputs.terminal_output = 'stream'
+	maths.terminal_output = 'stream'
 	testsave.connect(bet2, ('mask_file', func), maths, "in_file2")
 	testsave.connect(bet, "mask_file", maths, "in_file")
 	testsave.connect(testfunc, "output", maths, "op_string")