Commit 0a0414bc authored by Rye Mutt's avatar Rye Mutt 🍞
Browse files

Add support for ZStandard compressed tar files

parent 9ae09e4b
Pipeline #1154 passed with stages
in 28 seconds
...@@ -259,7 +259,7 @@ class Archive(InteractiveCommand): ...@@ -259,7 +259,7 @@ class Archive(InteractiveCommand):
ARGUMENTS = ['format', 'hash_algorithm', 'platform'] ARGUMENTS = ['format', 'hash_algorithm', 'platform']
ARG_DICT = {'format': {'help': 'Archive format (e.g zip, tbz2, tgz, txz)'}, ARG_DICT = {'format': {'help': 'Archive format (e.g zip, tbz2, tgz, txz, tzst)'},
'hash_algorithm': {'help': 'The algorithm for computing the archive hash (e.g. md5, sha256, sha3_256, sha3_384, blake2b)'}, 'hash_algorithm': {'help': 'The algorithm for computing the archive hash (e.g. md5, sha256, sha3_256, sha3_384, blake2b)'},
'platform': {'help': 'The name of the platform archive to be configured'} 'platform': {'help': 'The name of the platform archive to be configured'}
} }
......
...@@ -318,7 +318,7 @@ def _install_package(archive_path, install_dir, exclude=[]): ...@@ -318,7 +318,7 @@ def _install_package(archive_path, install_dir, exclude=[]):
logger.error("cannot extract non-existing package: %s" % archive_path) logger.error("cannot extract non-existing package: %s" % archive_path)
return False return False
logger.info("extracting from %s" % os.path.basename(archive_path)) logger.info("extracting from %s" % os.path.basename(archive_path))
if tarfile.is_tarfile(archive_path): if ".tar.zst" in archive_path or ".tar.xz" in archive_path or ".tar.bz2" in archive_path or ".tar.gz" in archive_path:
return __extract_tar_file(archive_path, install_dir, exclude=exclude) return __extract_tar_file(archive_path, install_dir, exclude=exclude)
elif zipfile.is_zipfile(archive_path): elif zipfile.is_zipfile(archive_path):
return __extract_zip_archive(archive_path, install_dir, exclude=exclude) return __extract_zip_archive(archive_path, install_dir, exclude=exclude)
...@@ -336,8 +336,11 @@ def extract_metadata_from_package(archive_path, metadata_file_name): ...@@ -336,8 +336,11 @@ def extract_metadata_from_package(archive_path, metadata_file_name):
logger.error("no package found at: %s" % archive_path) logger.error("no package found at: %s" % archive_path)
else: else:
logger.debug("extracting metadata from %s" % os.path.basename(archive_path)) logger.debug("extracting metadata from %s" % os.path.basename(archive_path))
if tarfile.is_tarfile(archive_path): if ".tar.zst" in archive_path or ".tar.xz" in archive_path or ".tar.bz2" in archive_path or ".tar.gz" in archive_path:
tar = tarfile.open(archive_path, 'r') if ".tar.zst" in archive_path:
tar = common.ZstdTarFile(archive_path, 'r')
else:
tar = tarfile.open(archive_path, 'r')
try: try:
metadata_file = tar.extractfile(metadata_file_name) metadata_file = tar.extractfile(metadata_file_name)
except KeyError as err: except KeyError as err:
...@@ -356,7 +359,10 @@ def extract_metadata_from_package(archive_path, metadata_file_name): ...@@ -356,7 +359,10 @@ def extract_metadata_from_package(archive_path, metadata_file_name):
def __extract_tar_file(cachename, install_dir, exclude=[]): def __extract_tar_file(cachename, install_dir, exclude=[]):
# Attempt to extract the package from the install cache # Attempt to extract the package from the install cache
tar = tarfile.open(cachename, 'r') if ".tar.zst" in cachename:
tar = common.ZstdTarFile(cachename, 'r')
else:
tar = tarfile.open(cachename, 'r')
extract = [member for member in tar.getmembers() if member.name not in exclude] extract = [member for member in tar.getmembers() if member.name not in exclude]
conflicts = [member.name for member in extract conflicts = [member.name for member in extract
if os.path.exists(os.path.join(install_dir, member.name)) if os.path.exists(os.path.join(install_dir, member.name))
......
...@@ -91,7 +91,7 @@ class AutobuildTool(autobuild_base.AutobuildBase): ...@@ -91,7 +91,7 @@ class AutobuildTool(autobuild_base.AutobuildBase):
parser.add_argument('--archive-format', parser.add_argument('--archive-format',
default=None, default=None,
dest='archive_format', dest='archive_format',
help='the format of the archive (txz, tgz, tbz2 or zip)') help='the format of the archive (tzst, txz, tgz, tbz2 or zip)')
parser.add_argument('--build-dir', parser.add_argument('--build-dir',
default=None, default=None,
dest='select_dir', # see common.select_directories() dest='select_dir', # see common.select_directories()
...@@ -259,7 +259,7 @@ def package(config, build_directory, platform_name, archive_filename=None, archi ...@@ -259,7 +259,7 @@ def package(config, build_directory, platform_name, archive_filename=None, archi
else: else:
archive_description = platform_description.archive archive_description = platform_description.archive
format = _determine_archive_format(archive_format, archive_description) format = _determine_archive_format(archive_format, archive_description)
if format == 'txz' or format == 'tbz2' or format == 'tgz': if format == 'txz' or format == 'tbz2' or format == 'tgz' or format == 'tzst':
_create_tarfile(tarfilename, format, build_directory, files, results, results_dict) _create_tarfile(tarfilename, format, build_directory, files, results, results_dict)
elif format == 'zip': elif format == 'zip':
_create_zip_archive(tarfilename + '.zip', build_directory, files, results, results_dict) _create_zip_archive(tarfilename + '.zip', build_directory, files, results, results_dict)
...@@ -273,7 +273,7 @@ def _determine_archive_format(archive_format_argument, archive_description): ...@@ -273,7 +273,7 @@ def _determine_archive_format(archive_format_argument, archive_description):
if archive_format_argument is not None: if archive_format_argument is not None:
return archive_format_argument return archive_format_argument
elif archive_description is None or archive_description.format is None: elif archive_description is None or archive_description.format is None:
return 'txz' return 'tzst'
else: else:
return archive_description.format return archive_description.format
...@@ -318,17 +318,26 @@ def _create_tarfile(tarfilename, format, build_directory, filelist, results, res ...@@ -318,17 +318,26 @@ def _create_tarfile(tarfilename, format, build_directory, filelist, results, res
try: try:
if format == 'txz': if format == 'txz':
tarfilename = tarfilename + '.tar.xz' tarfilename = tarfilename + '.tar.xz'
mode = 'w:xz' tfile = tarfile.open(tarfilename, 'w:xz')
elif format == 'tbz2': elif format == 'tbz2':
tarfilename = tarfilename + '.tar.bz2' tarfilename = tarfilename + '.tar.bz2'
mode = 'w:bz2' tfile = tarfile.open(tarfilename, 'w:bz2')
elif format == 'tgz': elif format == 'tgz':
tarfilename = tarfilename + '.tar.gz' tarfilename = tarfilename + '.tar.gz'
mode = 'w:gz' tfile = tarfile.open(tarfilename, 'w:gz')
elif format == 'tzst':
tarfilename = tarfilename + '.tar.zst'
import multiprocessing
from pyzstd import CParameter
zstdoption = {CParameter.compressionLevel : 22,
CParameter.nbWorkers : 4,
CParameter.checksumFlag : 1}
tfile = common.ZstdTarFile(tarfilename, 'w', level_or_option=zstdoption)
else: else:
raise PackageError("unknown tar archive format: %s" % format) raise PackageError("unknown tar archive format: %s" % format)
tfile = tarfile.open(tarfilename, mode)
for file in filelist: for file in filelist:
try: try:
# Make sure permissions are set on Windows. # Make sure permissions are set on Windows.
......
...@@ -47,6 +47,8 @@ import shutil ...@@ -47,6 +47,8 @@ import shutil
import sys import sys
import tempfile import tempfile
import time import time
import tarfile
from pyzstd import ZstdFile
from .version import AUTOBUILD_VERSION_STRING from .version import AUTOBUILD_VERSION_STRING
...@@ -624,3 +626,19 @@ def establish_build_id(build_id_arg): ...@@ -624,3 +626,19 @@ def establish_build_id(build_id_arg):
return str(build_id) return str(build_id)
class ZstdTarFile(tarfile.TarFile):
def __init__(self, name, mode='r', *, level_or_option=None, zstd_dict=None, **kwargs):
self.zstd_file = ZstdFile(name, mode,
level_or_option=level_or_option,
zstd_dict=zstd_dict)
try:
super().__init__(fileobj=self.zstd_file, mode=mode, **kwargs)
except:
self.zstd_file.close()
raise
def close(self):
try:
super().close()
finally:
self.zstd_file.close()
...@@ -78,7 +78,7 @@ class TestPackaging(BaseTest): ...@@ -78,7 +78,7 @@ class TestPackaging(BaseTest):
self.config = configfile.ConfigurationDescription(self.config_path) self.config = configfile.ConfigurationDescription(self.config_path)
self.platform = 'common' self.platform = 'common'
self.tar_basename = os.path.join(self.data_dir, "test1-1.0-common-123456") self.tar_basename = os.path.join(self.data_dir, "test1-1.0-common-123456")
self.tar_name = self.tar_basename + ".tar.xz" self.tar_name = self.tar_basename + ".tar.zst"
self.zip_name = self.tar_basename + ".zip" self.zip_name = self.tar_basename + ".zip"
self.expected_files=['include/file1','LICENSES/test1.txt','autobuild-package.xml'] self.expected_files=['include/file1','LICENSES/test1.txt','autobuild-package.xml']
self.expected_files.sort() self.expected_files.sort()
......
...@@ -43,4 +43,4 @@ $/LicenseInfo$ ...@@ -43,4 +43,4 @@ $/LicenseInfo$
# will wreak havoc upon new users of your package, as they will not be able to # will wreak havoc upon new users of your package, as they will not be able to
# install your package without manually installing the dependencies first." # install your package without manually installing the dependencies first."
AUTOBUILD_VERSION_STRING = "2.0.9" AUTOBUILD_VERSION_STRING = "2.1.0"
...@@ -73,7 +73,7 @@ setup( ...@@ -73,7 +73,7 @@ setup(
scripts=[], scripts=[],
license='MIT', license='MIT',
classifiers=[line for line in CLASSIFIERS.split("\n") if line], classifiers=[line for line in CLASSIFIERS.split("\n") if line],
install_requires=['llbase', 'pydot'], install_requires=['llbase', 'pydot', 'pyzstd'],
python_requires=">=3.6", python_requires=">=3.6",
#ext_modules=ext_modules, #ext_modules=ext_modules,
) )
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment