Merge branch 'master' of https://github.com/nipy/nipype into enh/expand_ants_out

Author: rwblair

.travis.yml

@@ -42,7 +42,7 @@ install:
   echo "data_file = ${COVERAGE_DATA_FILE}" >> ${COVERAGE_PROCESS_START}; }
 - travis_retry inst
 script:
-- python -W once:FSL:UserWarning:nipype `which nosetests` --with-doctest --with-doctest-ignore-unicode --with-cov --cover-package nipype --logging-level=DEBUG --verbosity=3
+- py.test --doctest-modules --cov=nipype nipype
 after_success:
 - bash <(curl -s https://codecov.io/bash) -t ac172a50-8e66-42e5-8822-5373fcf54686 -cF unittests
 deploy:

Makefile

@@ -5,7 +5,7 @@
 PYTHON ?= python
 NOSETESTS=`which nosetests`
 
-.PHONY: zipdoc sdist egg upload_to_pypi trailing-spaces clean-pyc clean-so clean-build clean-ctags clean in inplace test-code test-doc test-coverage test html specs check-before-commit check
+.PHONY: zipdoc sdist egg upload_to_pypi trailing-spaces clean-pyc clean-so clean-build clean-ctags clean in inplace test-code test-coverage test html specs check-before-commit check
 
 zipdoc: html
 	zip documentation.zip doc/_build/html
@@ -56,16 +56,11 @@ inplace:
 	$(PYTHON) setup.py build_ext -i
 
 test-code: in
-	python -W once:FSL:UserWarning:nipype $(NOSETESTS) --with-doctest --with-doctest-ignore-unicode --logging-level=DEBUG --verbosity=3 nipype
-
-test-doc:
-	$(NOSETESTS) -s --with-doctest --with-doctest-ignore-unicode --doctest-tests --doctest-extension=rst \
-	--doctest-fixtures=_fixture doc/
+	py.test --doctest-module nipype
 
 test-coverage: clean-tests in
-	$(NOSETESTS) -s --with-doctest --with-doctest-ignore-unicode --with-coverage --cover-package=nipype \
-	--config=.coveragerc
-
+	py.test --doctest-modules --cov-config .coveragerc --cov=nipype nipype
+	
 test: tests # just another name
 tests: clean test-code
 

circle.yml

@@ -20,7 +20,7 @@ dependencies:
     - sudo apt-get -y update && sudo apt-get install -y wget bzip2
 
   override:
-    - mkdir -p ~/examples ~/scratch/nose ~/scratch/logs
+    - mkdir -p ~/examples ~/scratch/pytest ~/scratch/logs
     - if [[ ! -d ~/examples/nipype-tutorial ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-tutorial.tar.bz2 "${DATA_NIPYPE_TUTORIAL_URL}" && tar xjf nipype-tutorial.tar.bz2 -C ~/examples/; fi
     - if [[ ! -d ~/examples/nipype-fsl_course_data ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O nipype-fsl_course_data.tar.gz "${DATA_NIPYPE_FSL_COURSE}" && tar xzf nipype-fsl_course_data.tar.gz -C ~/examples/; fi
     - if [[ ! -d ~/examples/feeds ]]; then wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q -O fsl-5.0.9-feeds.tar.gz "${DATA_NIPYPE_FSL_FEEDS}" && tar xzf fsl-5.0.9-feeds.tar.gz -C ~/examples/; fi
@@ -35,10 +35,10 @@ dependencies:
 test:
   override:
     - docker run -v /etc/localtime:/etc/localtime:ro -v ~/scratch:/scratch -w /root/src/nipype/doc nipype/nipype_test:py35 /usr/bin/run_builddocs.sh
-    - docker run -v /etc/localtime:/etc/localtime:ro -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" -e TRAVIS_PYTHON_VERSION="3.5" -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /root/src/nipype nipype/nipype_test:py35 /usr/bin/run_nosetests.sh py35 :
+    - docker run -v /etc/localtime:/etc/localtime:ro -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /root/src/nipype nipype/nipype_test:py35 /usr/bin/run_pytests.sh py35 :
+        timeout: 2600
+    - docker run -v /etc/localtime:/etc/localtime:ro -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /root/src/nipype nipype/nipype_test:py27 /usr/bin/run_pytests.sh py27 :
         timeout: 2600
-    - docker run -v /etc/localtime:/etc/localtime:ro -e FSL_COURSE_DATA="/root/examples/nipype-fsl_course_data" -e TRAVIS_PYTHON_VERSION="2.7" -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /root/src/nipype nipype/nipype_test:py27 /usr/bin/run_nosetests.sh py27 :
-        timeout: 5200
     - docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow3d :
         timeout: 1600
     - docker run -v /etc/localtime:/etc/localtime:ro -v ~/examples:/root/examples:ro -v ~/scratch:/scratch -w /scratch nipype/nipype_test:py35 /usr/bin/run_examples.sh test_spm Linear /root/examples/ workflow4d :
@@ -56,8 +56,8 @@ test:
 
   post:
     - bash docker/circleci/teardown.sh
-    - for xml_f in ${CIRCLE_TEST_REPORTS}/nose/coverage*.xml; do bash <(curl -s https://codecov.io/bash) -f $xml_f -t ac172a50-8e66-42e5-8822-5373fcf54686 -F unittests; done
-    - for xml_f in ${CIRCLE_TEST_REPORTS}/nose/smoketest*.xml; do bash <(curl -s https://codecov.io/bash) -f $xml_f -t ac172a50-8e66-42e5-8822-5373fcf54686 -F smoketests; done
+    - for xml_f in ${CIRCLE_TEST_REPORTS}/pytest/coverage*.xml; do bash <(curl -s https://codecov.io/bash) -f $xml_f -t ac172a50-8e66-42e5-8822-5373fcf54686 -F unittests; done
+    - for xml_f in ${CIRCLE_TEST_REPORTS}/pytest/smoketest*.xml; do bash <(curl -s https://codecov.io/bash) -f $xml_f -t ac172a50-8e66-42e5-8822-5373fcf54686 -F smoketests; done
 
 general:
   artifacts:

doc/devel/testing_nipype.rst

@@ -40,7 +40,7 @@ or::
 Test implementation
 -------------------
 
-Nipype testing framework is built upon `nose <http://nose.readthedocs.io/en/latest/>`_.
+Nipype testing framework is built upon `pytest <http://doc.pytest.org/en/latest/>`_.
 By the time these guidelines are written, Nipype implements 17638 tests.
 
 After installation in developer mode, the tests can be run with the
@@ -50,20 +50,18 @@ following simple command at the root folder of the project ::
 
 If ``make`` is not installed in the system, it is possible to run the tests using::
 
-    python -W once:FSL:UserWarning:nipype `which nosetests` --with-doctest \
-           --with-doctest-ignore-unicode --logging-level=DEBUG --verbosity=3 nipype
+     py.test --doctest-modules --cov=nipype nipype
 
 
-A successful test run should complete in a few minutes and end with
+A successful test run should complete in 10-30 minutes and end with
 something like::
 
     ----------------------------------------------------------------------
-    Ran 17922 tests in 107.254s
+    2445 passed, 41 skipped, 7 xfailed in 1277.66 seconds
 
-    OK (SKIP=27)
 
 
-All tests should pass (unless you're missing a dependency). If the ``SUBJECTS_DIR```
+No test should fail (unless you're missing a dependency). If the ``SUBJECTS_DIR```
 environment variable is not set, some FreeSurfer related tests will fail.
 If any of the tests failed, please report them on our `bug tracker
 <http://github.com/nipy/nipype/issues>`_.
@@ -90,6 +88,11 @@ Some tests in Nipype make use of some images distributed within the `FSL course
 To enable the tests depending on these data, just unpack the targz file and set the :code:`FSL_COURSE_DATA` environment
 variable to point to that folder.
 
+Xfail tests
+~~~~~~~~~~
+
+Some tests are expect to fail until the code will be changed or for other reasons.
+
 
 Avoiding any MATLAB calls from testing
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -115,7 +118,7 @@ Nipype is as easy as follows::
                       -v ~/examples:/root/examples:ro \
                       -v ~/scratch:/scratch \
                       -w /root/src/nipype \
-                      nipype/nipype_test:py27 /usr/bin/run_nosetests.sh
+                      nipype/nipype_test:py27 /usr/bin/run_pytest.sh
 
 For running nipype in Python 3.5::
 
@@ -126,4 +129,4 @@ For running nipype in Python 3.5::
                       -v ~/examples:/root/examples:ro \
                       -v ~/scratch:/scratch \
                       -w /root/src/nipype \
-                      nipype/nipype_test:py35 /usr/bin/run_nosetests.sh
+                      nipype/nipype_test:py35 /usr/bin/run_pytest.sh

docker/circleci/run_nosetests.sh docker/circleci/run_pytests.shdocker/circleci/run_nosetests.sh renamed to docker/circleci/run_pytests.sh

@@ -6,7 +6,7 @@ set -u
 PYTHON_VERSION=$( python -c "import sys; print('{}{}'.format(sys.version_info[0], sys.version_info[1]))" )
 
 # Create necessary directories
-mkdir -p /scratch/nose /scratch/crashfiles /scratch/logs/py${PYTHON_VERSION}
+mkdir -p /scratch/pytest /scratch/crashfiles /scratch/logs/py${PYTHON_VERSION}
 
 # Create a nipype config file
 mkdir -p /root/.nipype
@@ -20,17 +20,18 @@ if [[ "${PYTHON_VERSION}" -lt "30" ]]; then
     echo 'profile_runtime = true' >> /root/.nipype/nipype.cfg
 fi
 
-# Run tests
+# Run tests using pytest
 cd /root/src/nipype/
 make clean
-nosetests -s nipype -c /root/src/nipype/.noserc --xunit-file="/scratch/nosetests_py${PYTHON_VERSION}.xml" --cover-xml-file="/scratch/coverage_py${PYTHON_VERSION}.xml"
+py.test --doctest-modules --cov-report xml:/scratch/coverage_py${PYTHON_VERSION}.xml --cov=nipype nipype
+
 
 # Workaround: run here the profiler tests in python 3
 if [[ "${PYTHON_VERSION}" -ge "30" ]]; then
     echo '[execution]' >> /root/.nipype/nipype.cfg
     echo 'profile_runtime = true' >> /root/.nipype/nipype.cfg
-    nosetests nipype/interfaces/tests/test_runtime_profiler.py --xunit-file="/scratch/nosetests_py${PYTHON_VERSION}_profiler.xml" --cover-xml-file="/scratch/coverage_py${PYTHON_VERSION}_profiler.xml"
-    nosetests nipype/pipeline/plugins/tests/test_multiproc*.py --xunit-file="/scratch/nosetests_py${PYTHON_VERSION}_multiproc.xml" --cover-xml-file="/scratch/coverage_py${PYTHON_VERSION}_multiproc.xml"
+    py.test --cov-report xml:/scratch/coverage_py${PYTHON_VERSION}_profiler.xml --cov=nipype nipype/interfaces/tests/test_runtime_profiler.py 
+    py.test --cov-report xml:/scratch/coverage_py${PYTHON_VERSION}_multiproc.xml --cov=nipype nipype/pipeline/plugins/tests/test_multiproc*.py 
 fi
 
 # Copy crashfiles to scratch

docker/circleci/teardown.sh

@@ -6,8 +6,8 @@
 set -u
 set -e
 
-mkdir -p ${CIRCLE_TEST_REPORTS}/nose
-sudo mv ~/scratch/*.xml ${CIRCLE_TEST_REPORTS}/nose
+mkdir -p ${CIRCLE_TEST_REPORTS}/pytest
+sudo mv ~/scratch/*.xml ${CIRCLE_TEST_REPORTS}/pytest
 mkdir -p ~/docs
 sudo mv ~/scratch/docs/* ~/docs/
 mkdir -p ~/logs

docker/nipype_test/Dockerfile_base

@@ -122,6 +122,6 @@ RUN conda config --add channels conda-forge && \
                      nitime \
                      dipy \
                      pandas && \
-    pip install nose-cov doctest-ignore-unicode configparser
+    pip install configparser
 
 CMD ["/bin/bash"]

docker/nipype_test/Dockerfile_py27

@@ -32,7 +32,7 @@ MAINTAINER The nipype developers https://github.com/nipy/nipype
 # Downgrade python to 2.7
 RUN conda update -y conda && \
     conda update --all -y python=2.7 && \
-    pip install nose-cov doctest-ignore-unicode configparser
+    pip install configparser
 
 COPY docker/circleci/run_* /usr/bin/
 RUN chmod +x /usr/bin/run_*

nipype/algorithms/confounds.py

@@ -372,8 +372,8 @@ def _list_outputs(self):
         outputs['components_file'] = os.path.abspath(self.inputs.components_file)
         return outputs
 
-    def _compute_tSTD(self, M, x):
-        stdM = np.std(M, axis=0)
+    def _compute_tSTD(self, M, x, axis=0):
+        stdM = np.std(M, axis=axis)
         # set bad values to x
         stdM[stdM == 0] = x
         stdM[np.isnan(stdM)] = x
@@ -411,6 +411,10 @@ class TCompCorInputSpec(CompCorInputSpec):
                                         'That is, the 2% of voxels '
                                         'with the highest variance are used.')
 
+class TCompCorOutputSpec(CompCorInputSpec):
+    # and all the fields in CompCorInputSpec
+    high_variance_mask = File(exists=True, desc="voxels excedding the variance threshold")
+
 class TCompCor(CompCor):
     '''
     Interface for tCompCor. Computes a ROI mask based on variance of voxels.
@@ -428,7 +432,7 @@ class TCompCor(CompCor):
     '''
 
     input_spec = TCompCorInputSpec
-    output_spec = CompCorOutputSpec
+    output_spec = TCompCorOutputSpec
 
     def _run_interface(self, runtime):
         imgseries = nb.load(self.inputs.realigned_file).get_data()
@@ -438,29 +442,34 @@ def _run_interface(self, runtime):
                              '(shape {})'
                              .format(self.inputs.realigned_file, imgseries.ndim, imgseries.shape))
 
+        if isdefined(self.inputs.mask_file):
+            in_mask_data = nb.load(self.inputs.mask_file).get_data()
+            imgseries = imgseries[in_mask_data != 0, :]
+
         # From the paper:
         # "For each voxel time series, the temporal standard deviation is
         # defined as the standard deviation of the time series after the removal
         # of low-frequency nuisance terms (e.g., linear and quadratic drift)."
         imgseries = regress_poly(2, imgseries)
 
-        time_voxels = imgseries.T
-        num_voxels = np.prod(time_voxels.shape[1:])
-
         # "To construct the tSTD noise ROI, we sorted the voxels by their
         # temporal standard deviation ..."
-        tSTD = self._compute_tSTD(time_voxels, 0)
-        sortSTD = np.sort(tSTD, axis=None) # flattened sorted matrix
+        tSTD = self._compute_tSTD(imgseries, 0, axis=-1)
 
         # use percentile_threshold to pick voxels
-        threshold_index = int(num_voxels * (1. - self.inputs.percentile_threshold))
-        threshold_std = sortSTD[threshold_index]
+        threshold_std = np.percentile(tSTD, 100. * (1. - self.inputs.percentile_threshold))
         mask = tSTD >= threshold_std
-        mask = mask.astype(int).T
+
+        if isdefined(self.inputs.mask_file):
+            mask_data = np.zeros_like(in_mask_data)
+            mask_data[in_mask_data != 0] = mask
+        else:
+            mask_data = mask.astype(int)
 
         # save mask
         mask_file = os.path.abspath('mask.nii')
-        nb.nifti1.save(nb.Nifti1Image(mask, np.eye(4)), mask_file)
+        nb.Nifti1Image(mask_data,
+                       nb.load(self.inputs.realigned_file).affine).to_filename(mask_file)
         IFLOG.debug('tCompcor computed and saved mask of shape {} to mask_file {}'
                    .format(mask.shape, mask_file))
         self.inputs.mask_file = mask_file
@@ -469,6 +478,11 @@ def _run_interface(self, runtime):
         super(TCompCor, self)._run_interface(runtime)
         return runtime
 
+    def _list_outputs(self):
+        outputs = super(TCompCor, self)._list_outputs()
+        outputs['high_variance_mask'] = self.inputs.mask_file
+        return outputs
+
 class TSNRInputSpec(BaseInterfaceInputSpec):
     in_file = InputMultiPath(File(exists=True), mandatory=True,
                              desc='realigned 4D file or a list of 3D files')
@@ -654,7 +668,7 @@ def compute_dvars(in_file, in_mask, remove_zerovariance=False):
         warnings.filterwarnings('error')
 
         # voxelwise standardization
-        diff_vx_stdz = func_diff / np.array([diff_sdhat] * func_diff.shape[-1]).T            
+        diff_vx_stdz = func_diff / np.array([diff_sdhat] * func_diff.shape[-1]).T
         dvars_vx_stdz = diff_vx_stdz.std(axis=0, ddof=1)
 
     return (dvars_stdz, dvars_nstd, dvars_vx_stdz)

nipype/algorithms/tests/test_auto_AddCSVColumn.py

@@ -1,5 +1,4 @@
 # AUTO-GENERATED by tools/checkspecs.py - DO NOT EDIT
-from ...testing import assert_equal
 from ..misc import AddCSVColumn
 
 
@@ -15,7 +14,7 @@ def test_AddCSVColumn_inputs():
 
     for key, metadata in list(input_map.items()):
         for metakey, value in list(metadata.items()):
-            yield assert_equal, getattr(inputs.traits()[key], metakey), value
+            assert getattr(inputs.traits()[key], metakey) == value
 
 
 def test_AddCSVColumn_outputs():
@@ -25,4 +24,4 @@ def test_AddCSVColumn_outputs():
 
     for key, metadata in list(output_map.items()):
         for metakey, value in list(metadata.items()):
-            yield assert_equal, getattr(outputs.traits()[key], metakey), value
+            assert getattr(outputs.traits()[key], metakey) == value