Handle Singularity 3.6 and later /tmp staging via automatic %setup fallback

docs/misc_api.md

@@ -124,6 +124,17 @@ __Arguments__
 - __wd (string)__: working directory path
 
 
+## set_singularity_tmp_fallback
+```python
+set_singularity_tmp_fallback(enable=True)
+```
+Enable or disable the automatic %setup fallback for /tmp and /var/tmp destinations on Singularity 3.6 and later.
+
+__Arguments__
+
+
+- __enable (bool)__: True to enable the fallback (default), False to disable. The default is True.
+
 ## test_cpu_feature_flag
 ```python
 test_cpu_feature_flag(flag)
@@ -138,7 +149,7 @@ __Arguments__
 
 # recipe
 ```python
-recipe(recipe_file, cpu_target=None, ctype=<container_type.DOCKER: 1>, raise_exceptions=False, single_stage=False, singularity_version=u'2.6', userarg=None, working_directory=u'/var/tmp')
+recipe(recipe_file, cpu_target=None, ctype=<container_type.DOCKER: 1>, raise_exceptions=False, single_stage=False, singularity_version=u'2.6', userarg=None, working_directory=u'/var/tmp', singularity_tmp_fallback=True)
 ```
 Recipe builder
 
@@ -169,6 +180,12 @@ as the `USERARG` dictionary.
 - __working_directory__: path to use as the working directory in the
 container specification
 
+- __singularity_tmp_fallback__: If True (default), automatically handle
+copy destinations under /tmp or /var/tmp using a %setup block when
+targeting Singularity >= 3.6. If False, such copy operations are
+rejected and will raise an error, requiring the user to modify the
+recipe.
+
 
 # Stage
 ```python

docs/primitives.md

@@ -123,6 +123,17 @@ identifier.  This also causes the Singularity block to named
 
 - __dest__: Path in the container image to copy the file(s)
 
+- ___exclude_from__: String or list of strings.  One or more filenames
+containing rsync-style exclude patterns (e.g., `.apptainerignore`).
+Only used when building for Singularity or Apptainer.  If specified,
+the copy operation is emitted in the `%setup` section using
+`rsync --exclude-from=<file>` rather than the standard `%files`
+copy directive.  This enables selective exclusion of files and
+directories during the image build, for example to omit large data
+files, caches, or temporary artifacts.  Multiple exclusion files may
+be provided as a list or tuple.  The default is an empty list
+(Singularity specific).
+
 - __files__: A dictionary of file pairs, source and destination, to copy
 into the container image.  If specified, has precedence over
 `dest` and `src`.
@@ -145,6 +156,23 @@ specific).
 
 - __src__: A file, or a list of files, to copy
 
+__Notes__
+
+
+On Singularity 3.6 and later, files cannot be staged directly to
+`/tmp` or `/var/tmp` via the `%files` section. When such destinations
+are detected, the `copy` primitive automatically emits a `%setup`
+block to perform the copy safely (enabled by default).
+
+This behavior can be disabled via
+`hpccm.config.set_singularity_tmp_fallback(False)`, in which case a
+runtime error will be raised.
+
+Note that `--working-directory` (or
+`hpccm.config.set_working_directory()`) only affects commands
+executed during `%post` (e.g., downloads or builds) and does not
+apply to `%files` staging.
+
 __Examples__
 
 
@@ -160,6 +188,9 @@ copy(src=['a', 'b', 'c'], dest='/tmp')
 copy(files={'a': '/tmp/a', 'b': '/opt/b'})
 ```
 
+```python
+copy(src='.', dest='/opt/app', _exclude_from='.apptainerignore')
+```
 
 # environment
 ```python

hpccm/cli.py

@@ -65,6 +65,12 @@ def main(): # pragma: no cover
     parser.add_argument('--working-directory', '--wd', type=str,
                         default='/var/tmp',
                         help='set container working directory')
+    parser.add_argument('--no-singularity-tmp-fallback',
+                        dest='singularity_tmp_fallback',
+                        action='store_false',
+                        default=hpccm.config.g_singularity_tmp_fallback,
+                        help='Disable automatic %%setup fallback for /tmp and /var/tmp destinations on Singularity >= 3.6')
+
     args = parser.parse_args()
 
     # configure logger
@@ -77,7 +83,8 @@ def main(): # pragma: no cover
                           single_stage=args.single_stage,
                           singularity_version=args.singularity_version,
                           userarg=args.userarg,
-                          working_directory=args.working_directory)
+                          working_directory=args.working_directory,
+                          singularity_tmp_fallback=args.singularity_tmp_fallback)
     print(recipe)
 
 if __name__ == "__main__": # pragma: no cover

hpccm/config.py

@@ -44,6 +44,7 @@
 g_linux_version = Version('16.04') # Linux distribution version
 g_singularity_version = Version('2.6') # Singularity version
 g_wd = '/var/tmp' # Working directory
+g_singularity_tmp_fallback = True    # Singularity / Apptainer behavior flags
 
 def get_cpu_architecture():
   """Return the architecture string for the currently configured CPU
@@ -257,6 +258,17 @@ def set_working_directory(wd):
   this = sys.modules[__name__]
   this.g_wd = wd
 
+def set_singularity_tmp_fallback(enable=True):
+  """Enable or disable the automatic %setup fallback for /tmp and /var/tmp
+  destinations on Singularity >= 3.6.
+
+  # Arguments
+
+  enable (bool): True to enable the fallback (default), False to disable.
+  """
+  this = sys.modules[__name__]
+  this.g_singularity_tmp_fallback = enable
+
 def test_cpu_feature_flag(flag):
   """Return True or False depending on whether the CPU supports the
   given feature flag

hpccm/primitives/copy.py

@@ -77,6 +77,22 @@ class copy(object):
 
     src: A file, or a list of files, to copy
 
+    # Notes
+
+    On Singularity 3.6 and later, files cannot be staged directly to
+    `/tmp` or `/var/tmp` via the `%files` section. When such destinations
+    are detected, the `copy` primitive automatically emits a `%setup`
+    block to perform the copy safely (enabled by default).
+
+    This behavior can be disabled via
+    `hpccm.config.set_singularity_tmp_fallback(False)`, in which case a
+    runtime error will be raised.
+
+    Note that `--working-directory` (or
+    `hpccm.config.set_working_directory()`) only affects commands
+    executed during `%post` (e.g., downloads or builds) and does not
+    apply to `%files` staging.
+
     # Examples
 
     ```python
@@ -193,9 +209,24 @@ def __str__(self):
             # version.
             # https://github.com/NVIDIA/hpc-container-maker/issues/345
             if (not self.__from and
+
                 any(f['dest'].startswith(('/var/tmp', '/tmp')) for f in files)):
-                msg = 'Singularity 3.6 and later no longer allow a temporary directory to be used to stage files into the container image.  Modify the recipe or, in many cases, use --working-directory or hpccm.config.set_working_directory() to specify another location.'
-                if hpccm.config.g_singularity_version >= Version('3.6'):
+
+                msg = (
+                    'Singularity 3.6 and later do not allow staging files to /tmp or /var/tmp '
+                    'via the %files section. This restriction applies before the container '
+                    'root filesystem exists.\n\n'
+                    'hpc-container-maker can automatically handle this case using a %setup '
+                    'fallback (enabled by default).\n\n'
+                    'Note: --working-directory (or hpccm.config.set_working_directory()) '
+                    'only affects commands executed during %post (e.g., downloads or builds) '
+                    'and does not apply to %files staging.\n\n'
+                    'To disable the automatic fallback, set:\n'
+                    '  hpccm.config.set_singularity_tmp_fallback(False)\n\n'
+                    'Alternatively, modify the recipe to avoid /tmp or /var/tmp destinations.'
+                )
+
+                if hpccm.config.g_singularity_version >= Version('3.6') and not hpccm.config.g_singularity_tmp_fallback:
                     raise RuntimeError(msg)
                 else:
                     logging.warning(msg)
@@ -228,17 +259,40 @@ def __str__(self):
             flat_files = []
             post = [] # post actions if _post is enabled
             pre = [] # pre actions if _mkdir is enabled
+
+            created_dirs = set()
+
             for pair in files:
                 dest = pair['dest']
                 src = pair['src']
 
-                # Use rsync if exclusion file provided and not multi-stage copy
+                is_tmp_dest = dest.startswith(('/tmp', '/var/tmp'))
+                needs_tmp_fallback = (
+                    hpccm.config.g_singularity_tmp_fallback and
+                    not self.__from and
+                    is_tmp_dest and
+                    hpccm.config.g_singularity_version >= Version('3.6')
+                )
+
+
+                # 1) Use rsync if exclusion file provided and not multi-stage copy (_exclude_from --> always %setup + rsync)
                 if self.__exclude_from and not self.__from:
                     excl_opts = ' '.join('--exclude-from={}'.format(x) for x in self.__exclude_from)
                     pre.append('    mkdir -p ${{SINGULARITY_ROOTFS}}{0}'.format(dest))
                     pre.append('    rsync -av {0} {1}/ ${{SINGULARITY_ROOTFS}}{2}/'.format(excl_opts, src, dest))
                     continue
 
+                # 2) /tmp or /var/tmp fallback
+                if needs_tmp_fallback:
+                    parent = posixpath.dirname(dest)
+                    if parent not in created_dirs:
+                        pre.append('    mkdir -p ${{SINGULARITY_ROOTFS}}{0}'.format(parent))
+                        created_dirs.add(parent)
+
+                    pre.append('    cp -a {0} ${{SINGULARITY_ROOTFS}}{1}'.format(src, dest))
+                    continue
+
+                # 3) Original behavior (unchanged)
                 if self._post:
                     dest = '/'
 

hpccm/recipe.py

@@ -91,7 +91,8 @@ def include(recipe_file, _globals=None, _locals=None, prepend_path=True,
 def recipe(recipe_file, cpu_target=None, ctype=container_type.DOCKER,
            raise_exceptions=False, single_stage=False,
            singularity_version='2.6', userarg=None,
-           working_directory='/var/tmp'):
+           working_directory='/var/tmp',
+           singularity_tmp_fallback=True):
     """Recipe builder
 
     # Arguments
@@ -120,6 +121,12 @@ def recipe(recipe_file, cpu_target=None, ctype=container_type.DOCKER,
     working_directory: path to use as the working directory in the
     container specification
 
+    singularity_tmp_fallback: If True (default), automatically handle
+    copy destinations under /tmp or /var/tmp using a %setup block when
+    targeting Singularity >= 3.6. If False, such copy operations are
+    rejected and will raise an error, requiring the user to modify the
+    recipe.
+
     """
 
     # Make user arguments available
@@ -144,6 +151,9 @@ def recipe(recipe_file, cpu_target=None, ctype=container_type.DOCKER,
     # Set the global working directory
     hpccm.config.g_wd = working_directory
 
+    # Set Singularity /tmp fallback behavior
+    hpccm.config.g_singularity_tmp_fallback = singularity_tmp_fallback
+
     # Any included recipes that are specified using relative paths will
     # need to prepend the path to the main recipe in order to be found.
     # Save the path to the main recipe.