Jjardon/xz

docs/dependencies.md

@@ -27,7 +27,7 @@ if you trust the Python Package Index (PyPI) and pip is available on your
 machine, you can install these dependencies with:
 
 ```
-    pip install fs pyyaml sandboxlib requests jsonschema bottle cherrypy riemann-client
+    pip install fs pyyaml sandboxlib requests jsonschema bottle cherrypy riemann-client backports.lzma
 ```
 
 If you need to install pip itself:

install_dependencies.sh

@@ -44,6 +44,6 @@ if [ $? -ne 0 ]; then
     sudo rm get-pip.py
 fi
 
-sudo pip install fs pyyaml sandboxlib requests
+sudo pip install fs pyyaml sandboxlib requests backports.lzma
 sudo pip install jsonschema bottle cherrypy riemann-client
 sudo pip install pep8

scripts/Dockerfile

@@ -10,4 +10,4 @@ RUN wget https://bootstrap.pypa.io/get-pip.py && \
     python get-pip.py && rm get-pip.py
 
 # install python dependencies
-RUN pip install fs pyyaml sandboxlib requests jsonschema bottle cherrypy
+RUN pip install fs pyyaml sandboxlib requests jsonschema bottle cherrypy backports.lzma 

ybd/cache.py

@@ -134,8 +134,8 @@ def cache(defs, this):
         shutil.move('%s.tar' % cachefile, cachefile)
     else:
         utils.set_mtime_recursively(this['install'])
-        utils.make_deterministic_gztar_archive(cachefile, this['install'])
-        shutil.move('%s.tar.gz' % cachefile, cachefile)
+        utils.make_xztar_archive(cachefile, this['install'])
+        shutil.move('%s.tar.xz' % cachefile, cachefile)
 
     app.config['counter'].increment()
 

ybd/utils.py

@@ -14,7 +14,10 @@
 #
 # =*= License: GPL-2 =*=
 
-import gzip
+try:
+    import lzma
+except ImportError:
+    from backports import lzma
 import tarfile
 import contextlib
 import os
@@ -227,24 +230,9 @@ def _process_list(srcdir, destdir, filelist, actionfunc):
                           ' type.' % srcpath)
 
 
-def make_deterministic_gztar_archive(base_name, root_dir, time=1321009871.0):
-    '''Make a gzipped tar archive of contents of 'root_dir'.
-
-    This function takes extra steps to ensure the output is deterministic,
-    compared to shutil.make_archive(). First, it sorts the results of
-    os.listdir() to ensure the ordering of the files in the archive is the
-    same. Second, it sets a fixed timestamp and filename in the gzip header.
-
-    As well as fixing https://bugs.python.org/issue24465, to make this function
-    redundant we would need to patch shutil.make_archive() so we could manually
-    set the timestamp and filename set in the gzip file header.
-
+def make_xztar_archive(base_name, root_dir):
+    '''Make a xz tar archive of contents of 'root_dir'.
     '''
-    # It's hard to implement this function by monkeypatching
-    # shutil.make_archive() because of the way the tarfile module includes the
-    # filename of the tarfile in the gzip header. So we have to reimplement
-    # shutil.make_archive().
-
     def add_directory_to_tarfile(f_tar, dir_name, dir_arcname):
         for filename in sorted(os.listdir(dir_name)):
             name = os.path.join(dir_name, filename)
@@ -255,11 +243,10 @@ def add_directory_to_tarfile(f_tar, dir_name, dir_arcname):
             if os.path.isdir(name) and not os.path.islink(name):
                 add_directory_to_tarfile(f_tar, name, arcname)
 
-    with open(base_name + '.tar.gz', 'wb') as f:
-        gzip_context = gzip.GzipFile(
-            filename='', mode='wb', fileobj=f, mtime=time)
-        with gzip_context as f_gzip:
-            with tarfile.TarFile(mode='w', fileobj=f_gzip) as f_tar:
+    with open(base_name + '.tar.xz', 'wb') as f:
+        xz_context = lzma.LZMAFile(filename=f, mode='wb', preset=9)
+        with xz_context as f_xz:
+            with tarfile.TarFile(mode='w', fileobj=f_xz) as f_tar:
                 add_directory_to_tarfile(f_tar, root_dir, '.')