diff --git a/docs/dependencies.md b/docs/dependencies.md index 9001b00..8245344 100644 --- a/docs/dependencies.md +++ b/docs/dependencies.md @@ -27,7 +27,7 @@ if you trust the Python Package Index (PyPI) and pip is available on your machine, you can install these dependencies with: ``` - pip install fs pyyaml sandboxlib requests jsonschema bottle cherrypy riemann-client + pip install fs pyyaml sandboxlib requests jsonschema bottle cherrypy riemann-client backports.lzma ``` If you need to install pip itself: diff --git a/install_dependencies.sh b/install_dependencies.sh index 2286536..79e106f 100755 --- a/install_dependencies.sh +++ b/install_dependencies.sh @@ -44,6 +44,6 @@ if [ $? -ne 0 ]; then sudo rm get-pip.py fi -sudo pip install fs pyyaml sandboxlib requests +sudo pip install fs pyyaml sandboxlib requests backports.lzma sudo pip install jsonschema bottle cherrypy riemann-client sudo pip install pep8 diff --git a/scripts/Dockerfile b/scripts/Dockerfile index 757b523..d3c07c5 100644 --- a/scripts/Dockerfile +++ b/scripts/Dockerfile @@ -10,4 +10,4 @@ RUN wget https://bootstrap.pypa.io/get-pip.py && \ python get-pip.py && rm get-pip.py # install python dependencies -RUN pip install fs pyyaml sandboxlib requests jsonschema bottle cherrypy +RUN pip install fs pyyaml sandboxlib requests jsonschema bottle cherrypy backports.lzma diff --git a/ybd/cache.py b/ybd/cache.py index 84ece24..2f831b0 100644 --- a/ybd/cache.py +++ b/ybd/cache.py @@ -134,8 +134,8 @@ def cache(defs, this): shutil.move('%s.tar' % cachefile, cachefile) else: utils.set_mtime_recursively(this['install']) - utils.make_deterministic_gztar_archive(cachefile, this['install']) - shutil.move('%s.tar.gz' % cachefile, cachefile) + utils.make_xztar_archive(cachefile, this['install']) + shutil.move('%s.tar.xz' % cachefile, cachefile) app.config['counter'].increment() diff --git a/ybd/utils.py b/ybd/utils.py index 32815b2..0eac61d 100644 --- a/ybd/utils.py +++ b/ybd/utils.py @@ -14,7 +14,10 @@ # # =*= License: GPL-2 =*= -import gzip +try: + import lzma +except ImportError: + from backports import lzma import tarfile import contextlib import os @@ -227,24 +230,9 @@ def _process_list(srcdir, destdir, filelist, actionfunc): ' type.' % srcpath) -def make_deterministic_gztar_archive(base_name, root_dir, time=1321009871.0): - '''Make a gzipped tar archive of contents of 'root_dir'. - - This function takes extra steps to ensure the output is deterministic, - compared to shutil.make_archive(). First, it sorts the results of - os.listdir() to ensure the ordering of the files in the archive is the - same. Second, it sets a fixed timestamp and filename in the gzip header. - - As well as fixing https://bugs.python.org/issue24465, to make this function - redundant we would need to patch shutil.make_archive() so we could manually - set the timestamp and filename set in the gzip file header. - +def make_xztar_archive(base_name, root_dir): + '''Make a xz tar archive of contents of 'root_dir'. ''' - # It's hard to implement this function by monkeypatching - # shutil.make_archive() because of the way the tarfile module includes the - # filename of the tarfile in the gzip header. So we have to reimplement - # shutil.make_archive(). - def add_directory_to_tarfile(f_tar, dir_name, dir_arcname): for filename in sorted(os.listdir(dir_name)): name = os.path.join(dir_name, filename) @@ -255,11 +243,10 @@ def add_directory_to_tarfile(f_tar, dir_name, dir_arcname): if os.path.isdir(name) and not os.path.islink(name): add_directory_to_tarfile(f_tar, name, arcname) - with open(base_name + '.tar.gz', 'wb') as f: - gzip_context = gzip.GzipFile( - filename='', mode='wb', fileobj=f, mtime=time) - with gzip_context as f_gzip: - with tarfile.TarFile(mode='w', fileobj=f_gzip) as f_tar: + with open(base_name + '.tar.xz', 'wb') as f: + xz_context = lzma.LZMAFile(filename=f, mode='wb', preset=9) + with xz_context as f_xz: + with tarfile.TarFile(mode='w', fileobj=f_xz) as f_tar: add_directory_to_tarfile(f_tar, root_dir, '.')