Add support for ZStandard compressed tar files

autobuild/autobuild_tool_edit.py

@@ -259,7 +259,7 @@ class Archive(InteractiveCommand):
 
     ARGUMENTS = ['format', 'hash_algorithm', 'platform']
 
-    ARG_DICT = {'format':         {'help': 'Archive format (e.g zip, tbz2, tgz, txz)'},
+    ARG_DICT = {'format':         {'help': 'Archive format (e.g zip, tbz2, tgz, txz, tzst)'},
                 'hash_algorithm': {'help': 'The algorithm for computing the archive hash (e.g. md5, sha256, sha3_256, sha3_384, blake2b)'},
                 'platform':       {'help': 'The name of the platform archive to be configured'}
                 }

autobuild/autobuild_tool_install.py

@@ -318,7 +318,7 @@ def _install_package(archive_path, install_dir, exclude=[]):
         logger.error("cannot extract non-existing package: %s" % archive_path)
         return False
     logger.info("extracting from %s" % os.path.basename(archive_path))
-    if tarfile.is_tarfile(archive_path):
+    if ".tar.zst" in archive_path or ".tar.xz" in archive_path or ".tar.bz2" in archive_path or ".tar.gz" in archive_path:
         return __extract_tar_file(archive_path, install_dir, exclude=exclude)
     elif zipfile.is_zipfile(archive_path):
         return __extract_zip_archive(archive_path, install_dir, exclude=exclude)
@@ -336,7 +336,10 @@ def extract_metadata_from_package(archive_path, metadata_file_name):
         logger.error("no package found at: %s" % archive_path)
     else:
         logger.debug("extracting metadata from %s" % os.path.basename(archive_path))
-        if tarfile.is_tarfile(archive_path):
+        if ".tar.zst" in archive_path or ".tar.xz" in archive_path or ".tar.bz2" in archive_path or ".tar.gz" in archive_path:
+            if ".tar.zst" in archive_path:
+                tar = common.ZstdTarFile(archive_path, 'r')
+            else:
                 tar = tarfile.open(archive_path, 'r')
             try:
                 metadata_file = tar.extractfile(metadata_file_name)
@@ -356,6 +359,9 @@ def extract_metadata_from_package(archive_path, metadata_file_name):
 
 def __extract_tar_file(cachename, install_dir, exclude=[]):
     # Attempt to extract the package from the install cache
+    if ".tar.zst" in cachename:
+        tar = common.ZstdTarFile(cachename, 'r')
+    else:
         tar = tarfile.open(cachename, 'r')
     extract = [member for member in tar.getmembers() if member.name not in exclude]
     conflicts = [member.name for member in extract

autobuild/autobuild_tool_package.py

@@ -91,7 +91,7 @@ class AutobuildTool(autobuild_base.AutobuildBase):
         parser.add_argument('--archive-format',
                             default=None,
                             dest='archive_format',
-                            help='the format of the archive (txz, tgz, tbz2 or zip)')
+                            help='the format of the archive (tzst, txz, tgz, tbz2 or zip)')
         parser.add_argument('--build-dir',
                             default=None,
                             dest='select_dir',  # see common.select_directories()
@@ -259,7 +259,7 @@ def package(config, build_directory, platform_name, archive_filename=None, archi
     else:
         archive_description = platform_description.archive
         format = _determine_archive_format(archive_format, archive_description)
-        if format == 'txz' or format == 'tbz2' or format == 'tgz':
+        if format == 'txz' or format == 'tbz2' or format == 'tgz' or format == 'tzst':
             _create_tarfile(tarfilename, format, build_directory, files, results, results_dict)
         elif format == 'zip':
             _create_zip_archive(tarfilename + '.zip', build_directory, files, results, results_dict)
@@ -273,7 +273,7 @@ def _determine_archive_format(archive_format_argument, archive_description):
     if archive_format_argument is not None:
         return archive_format_argument
     elif archive_description is None or archive_description.format is None:
-        return 'txz'
+        return 'tzst'
     else:
         return archive_description.format
 
@@ -318,17 +318,26 @@ def _create_tarfile(tarfilename, format, build_directory, filelist, results, res
     try:
         if format == 'txz':
             tarfilename = tarfilename + '.tar.xz'
-            mode = 'w:xz'
+            tfile = tarfile.open(tarfilename, 'w:xz')
         elif format == 'tbz2':
             tarfilename = tarfilename + '.tar.bz2'
-            mode = 'w:bz2'
+            tfile = tarfile.open(tarfilename, 'w:bz2')
         elif format == 'tgz':
             tarfilename = tarfilename + '.tar.gz'
-            mode = 'w:gz'
+            tfile = tarfile.open(tarfilename, 'w:gz')
+        elif format == 'tzst':
+            tarfilename = tarfilename + '.tar.zst'
+
+            import multiprocessing
+            from pyzstd import CParameter
+            zstdoption = {CParameter.compressionLevel : 22,
+                    CParameter.nbWorkers : 4,
+                    CParameter.checksumFlag : 1}
+            
+            tfile = common.ZstdTarFile(tarfilename, 'w', level_or_option=zstdoption)
         else:
             raise PackageError("unknown tar archive format: %s" % format)
 
-        tfile = tarfile.open(tarfilename, mode)
         for file in filelist:
             try:
                 # Make sure permissions are set on Windows.

autobuild/common.py

@@ -47,6 +47,8 @@ import shutil
 import sys
 import tempfile
 import time
+import tarfile
+from pyzstd import ZstdFile
 
 from .version import AUTOBUILD_VERSION_STRING
 
@@ -624,3 +626,19 @@ def establish_build_id(build_id_arg):
     return str(build_id)
 
 
+class ZstdTarFile(tarfile.TarFile):
+    def __init__(self, name, mode='r', *, level_or_option=None, zstd_dict=None, **kwargs):
+        self.zstd_file = ZstdFile(name, mode,
+                                level_or_option=level_or_option,
+                                zstd_dict=zstd_dict)
+        try:
+            super().__init__(fileobj=self.zstd_file, mode=mode, **kwargs)
+        except:
+            self.zstd_file.close()
+            raise
+
+    def close(self):
+        try:
+            super().close()
+        finally:
+            self.zstd_file.close()

autobuild/tests/test_package.py

@@ -78,7 +78,7 @@ class TestPackaging(BaseTest):
         self.config = configfile.ConfigurationDescription(self.config_path)
         self.platform = 'common'
         self.tar_basename = os.path.join(self.data_dir, "test1-1.0-common-123456")
-        self.tar_name = self.tar_basename + ".tar.xz"
+        self.tar_name = self.tar_basename + ".tar.zst"
         self.zip_name = self.tar_basename + ".zip"
         self.expected_files=['include/file1','LICENSES/test1.txt','autobuild-package.xml']
         self.expected_files.sort()

autobuild/version.py

@@ -43,4 +43,4 @@ $/LicenseInfo$
 # will wreak havoc upon new users of your package, as they will not be able to
 # install your package without manually installing the dependencies first."
 
-AUTOBUILD_VERSION_STRING = "2.0.9"
+AUTOBUILD_VERSION_STRING = "2.1.0"

setup.py

@@ -73,7 +73,7 @@ setup(
     scripts=[],
     license='MIT',
     classifiers=[line for line in CLASSIFIERS.split("\n") if line],
-    install_requires=['llbase', 'pydot'],
+    install_requires=['llbase', 'pydot', 'pyzstd'],
     python_requires=">=3.6",
     #ext_modules=ext_modules,
     )