From df1827410b1acfd687bc1ac425c3895ee4e9f39e Mon Sep 17 00:00:00 2001 From: Mark Gomersbach Date: Fri, 17 Apr 2020 10:43:11 +0200 Subject: [PATCH] Add compression types and multithread if possible. (#39) * Add compression types and multithread if possible. Signed-off-by: Mark Gomersbach * Bump shell-linter version Signed-off-by: Mark Gomersbach --- .github/workflows/CI.yml | 4 +- README.md | 70 +++++++++++++++++++++++++++------ mkstage4.sh | 85 +++++++++++++++++++++++++++++----------- 3 files changed, 122 insertions(+), 37 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c788675..3451f27 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -9,10 +9,10 @@ jobs: uses: actions/checkout@v1 - name: "Run Shellcheck" - uses: azohra/shell-linter@v0.2.0 + uses: azohra/shell-linter@v0.3.0 - name: "Run Shellcheck on BATS files" - uses: azohra/shell-linter@v0.2.0 + uses: azohra/shell-linter@v0.3.0 with: path: "tests/*.bash,tests/*.bats" diff --git a/README.md b/README.md index 627a209..ebfe481 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ -![CI](https://github.com/TheChymera/mkstage4/workflows/CI/badge.svg) - # mkstage4 +![CI](https://github.com/TheChymera/mkstage4/workflows/CI/badge.svg) + This is a Bash script to create stage 4 tarballs either for the running system, or a system at a specified mount point. -The script was inspired by an earlier [mkstage4 script](https://github.com/gregf/bin/blob/master/mkstage4) by Greg Fitzgerald (unmaintained as of 2012) which itself was a revamped edition of the [original mkstage4](http://blinkeye.ch/dokuwiki/doku.php/projects/mkstage4) by Reto Glauser (unmaintaied as of 2009). +The script was inspired by an earlier [mkstage4 script](https://github.com/gregf/bin/blob/master/mkstage4) by Greg Fitzgerald (unmaintained as of 2012) which itself was a revamped edition of the [original mkstage4](http://blinkeye.ch/dokuwiki/doku.php/projects/mkstage4) by Reto Glauser (unmaintained as of 2009). More information on mkstage4 can be found on the following blogs, though instructions may be outdated compared to the current version, best documented by this `README` file: -* English: [mkstage4 - Stage 4 Tarballs Made Easy](http://tutorials.chymera.eu/blog/2014/05/18/mkstage4-stage4-tarballs-made-easy/). +* English: [mkstage4 - Stage 4 Tarballs Made Easy](http://tutorials.chymera.eu/blog/2014/05/18/mkstage4-stage4-tarballs-made-easy/). * Chinese: [中文说明](http://liuk.io/blog/gentoo-stage4) ## Installation @@ -23,7 +23,7 @@ chmod +x mkstage4.sh For [Gentoo Linux](http://en.wikipedia.org/wiki/Gentoo_linux) and [Derivatives](http://en.wikipedia.org/wiki/Category:Gentoo_Linux_derivatives), mkstage4 is also available in [Portage](http://en.wikipedia.org/wiki/Portage_(software)) via the base Gentoo overlay. On any Gentoo system, just run the following command: -``` +```bash emerge app-backup/mkstage4 ``` @@ -45,7 +45,7 @@ mkstage4 -t /custom/mount/point archive_name Command line arguments: -``` +```bash mkstage4.sh [-q -c -b -l -k -p] [-s || -t ] [-e ] [custom-tar-options] -q: activates quiet mode (no confirmation). -c: excludes connman network lists. @@ -56,6 +56,7 @@ Command line arguments: -s: makes tarball of current system. -k: separately save current kernel modules and src (smaller & save decompression time). -t: makes tarball of system located at the . + -C: specify tar compression (shows available on runtime and default is bz2) -h: displays help message. ``` @@ -80,22 +81,65 @@ tar xvjpf archive_name.tar.bz2.kmod tar xvjpf archive_name.tar.bz2.ksrc ``` -If you have pbzip2 installed, you can extract using parallelization, with: +If you have one of parallel (de-)compressors installed, you can extract with: + +In case of pbzip2: ```bash -tar -I pbzip2 -xvf archive_name.tar.bz2 +tar -I pbzip2 -xvf archive_name.tar.bz2 --xattrs-include='*.*' --numeric-owner +``` + +Or xz: + +```bash +tar -I 'xz -T0' -xvf archive_name.tar.xz --xattrs-include='*.*' --numeric-owner +``` + +Some compressors have separate binaries for the decompression, like gzip: + +```bash +tar -I unpigz -xvf archive_name.tar.gz --xattrs-include='*.*' --numeric-owner ``` ## Dependencies -* **[Bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell))** - in [Portage](http://en.wikipedia.org/wiki/Portage_(software)) as **app-shells/bash** -* **[tar](https://en.wikipedia.org/wiki/Tar_(computing))** - in Portage as **app-arch/tar** +*Please note that these are very basic dependencies and should already be included in any Linux system.* -*Please note that these are very basic dependencies and should already be included in any Linux system. Additionally, the scrip can use:* +* **[Bash](https://en.wikipedia.org/wiki/Bash_(Unix_shell))** - in [Portage](http://en.wikipedia.org/wiki/Portage_(software)) as **[app-shells/bash](https://packages.gentoo.org/packages/app-shells/bash)** +* **[tar](https://en.wikipedia.org/wiki/Tar_(computing))** - in Portage as **[app-arch/tar](https://packages.gentoo.org/packages/app-arch/tar)** +* **[bzip2](https://gitlab.com/federicomenaquintero/bzip2)** - in Portage as **[app-arch/bzip2](https://packages.gentoo.org/packages/app-arch/bzip2)** (single thread, default compression) -* **[pbzip2](https://launchpad.net/pbzip2)** (optional, if it is installed the archive can be compressed using multiple parallel threads) - in Portage as -**app-arch/pbzip2** +**Optionals**: +*If one the following is installed the archive will be compressed using multiple parallel threads when available, in order of succession:* + +* `-C xz`: + * **[xz](https://tukaani.org/xz/)** - in Portage as **[app-arch/xz](https://packages.gentoo.org/packages/app-arch/xz-utils)**, (parallel) + * **[pixz](https://github.com/vasi/pixz)** - in Portage as **[app-arch/pixz](https://packages.gentoo.org/packages/app-arch/pixz)**, (parallel, indexed) + +* `-C bz2`: + * **[pbzip2](https://launchpad.net/pbzip2/)** - in Portage as **[app-arch/pbzip2](https://packages.gentoo.org/packages/app-arch/pbzip2)**, (parallel) + * **[lbzip2](https://github.com/kjn/lbzip2/)** - in Portage as **[app-arch/lbzip2](https://packages.gentoo.org/packages/app-arch/lbzip2)**, (parallel, faster and more efficient) + +* `-C gz`: + * **[gzip](https://www.gnu.org/software/gzip/)** - in Portage as **[app-arch/gzip](https://packages.gentoo.org/packages/app-arch/gzip)**, (single thread) + * **[pigz](https://www.zlib.net/pigz/)** - in Portage as **[app-arch/pigz](https://packages.gentoo.org/packages/app-arch/pigz)**, (parallel) + +* `-C lrz`: + * **[lrzip](https://github.com/ckolivas/lrzip/)** - in Portage as **[app-arch/lrzip](https://packages.gentoo.org/packages/app-arch/lrzip)**, (parallel) + +* `-C lz`: + * **[lzip](https://www.nongnu.org/lzip/)** - in Portage as **[app-arch/lzip](https://packages.gentoo.org/packages/app-arch/lzip)**, (single thread) + * **[plzip](https://www.nongnu.org/lzip/plzip.html)** - in Portage as **[app-arch/plzip](https://packages.gentoo.org/packages/app-arch/plzip)**, (parallel) + +* `-C lz4`: + * **[lz4](https://github.com/lz4/lz4)** - in Portage as **[app-arch/lz4](https://packages.gentoo.org/packages/app-arch/lz4)**, (parallel) + +* `-C lzo`: + * **[lzop](https://www.lzop.org/)** - in Portage as **[app-arch/lzop](https://packages.gentoo.org/packages/app-arch/lzop)**, (parallel) + +* `-C zstd`: + * **[zstd](https://facebook.github.io/zstd/)** - in Portage as **[app-arch/zstd](https://packages.gentoo.org/packages/app-arch/zstd)**, (parallel) --- Released under the GPLv3 license. diff --git a/mkstage4.sh b/mkstage4.sh index 5e6b67c..6d5cc26 100755 --- a/mkstage4.sh +++ b/mkstage4.sh @@ -6,7 +6,29 @@ then exit 1 fi -#set flag variables to null +# get available compression types +declare -A COMPRESS_TYPES +COMPRESS_TYPES=( + ["bz2"]="bzip2 pbzip2 lbzip2" + ["gz"]="gzip pigz" + ["lrz"]="lrzip" + ["lz"]="lzip plzip" + ["lz4"]="lz4" + ["lzo"]="lzop" + ["xz"]="xz pixz" + ["zstd"]="zstd" + ) +declare -A COMPRESS_AVAILABLE +for ext in "${!COMPRESS_TYPES[@]}"; do + for exe in ${COMPRESS_TYPES[${ext}]}; do + BIN=$(command -v "${exe}") + if [ "${BIN}" != "" ]; then + COMPRESS_AVAILABLE+=(["${ext}"]="${BIN}") + fi + done +done + +# set flag variables to null/default EXCLUDE_BOOT=0 EXCLUDE_CONFIDENTIAL=0 EXCLUDE_LOST=0 @@ -14,8 +36,8 @@ QUIET=0 USER_EXCL=() USER_INCL=() S_KERNEL=0 -PARALLEL=0 HAS_PORTAGEQ=0 +COMPRESS_TYPE="bz2" if command -v portageq &>/dev/null then @@ -28,16 +50,16 @@ USAGE="usage:\n\ -c: excludes some confidential files (currently only .bash_history and connman network lists).\n\ -b: excludes boot directory.\n\ -l: excludes lost+found directory.\n\ - -p: compresses parallelly using pbzip2.\n\ -e: an additional excludes directory (one dir one -e, donot use it with *).\n\ -i: an additional target to include. This has higher precedence than -e, -t, and -s.\n\ -s: makes tarball of current system.\n\ -k: separately save current kernel modules and src (smaller & save decompression time).\n\ -t: makes tarball of system located at the .\n\ + -C: specify tar compression (available: ${!COMPRESS_AVAILABLE[*]}).\n\ -h: displays help message." # reads options: -while getopts ":t:e:i:skqcblph" flag +while getopts ":t:C:e:i:skqcblh" flag do case "$flag" in t) @@ -46,6 +68,9 @@ do s) TARGET="/" ;; + C) + COMPRESS_TYPE="$OPTARG" + ;; q) QUIET=1 ;; @@ -67,9 +92,6 @@ do i) USER_INCL+=("${OPTARG}") ;; - p) - PARALLEL=1 - ;; h) echo -e "$USAGE" exit 0 @@ -119,12 +141,30 @@ fi # determines if filename was given with relative or absolute path if (($(grep -c '^/' <<< "$ARCHIVE") > 0)) then - STAGE4_FILENAME="${ARCHIVE}.tar.bz2" + STAGE4_FILENAME="${ARCHIVE}.tar" else - STAGE4_FILENAME="$(pwd)/${ARCHIVE}.tar.bz2" + STAGE4_FILENAME="$(pwd)/${ARCHIVE}.tar" fi -#Shifts pointer to read custom tar options +# Check if compression in option and filename +if [ -z "$COMPRESS_TYPE" ] +then + echo "$(basename "$0"): no archive compression type specified." + echo -e "$USAGE" + exit 1 +else + STAGE4_FILENAME="${STAGE4_FILENAME}.${COMPRESS_TYPE}" +fi + +# Check if specified type is available +if [ -z "${COMPRESS_AVAILABLE[$COMPRESS_TYPE]}" ] +then + echo "$(basename "$0"): specified archive compression type not supported." + echo "Supported: ${COMPRESS_AVAILABLE[*]}" + exit 1 +fi + +# Shifts pointer to read custom tar options shift mapfile -t OPTIONS <<< "$@" # Handle when no options are passed @@ -197,21 +237,22 @@ then EXCLUDES+=("--exclude=lost+found") fi -# Generic tar options: -TAR_OPTIONS=(-cpP --ignore-failed-read "--xattrs-include='*.*'" --numeric-owner) - -if ((PARALLEL)) +# Compression options +COMP_OPTIONS=("${COMPRESS_AVAILABLE[$COMPRESS_TYPE]}") +if [[ "${COMPRESS_AVAILABLE[$COMPRESS_TYPE]}" == *"/xz" ]] then - if command -v pbzip2 &>/dev/null; then - TAR_OPTIONS+=("--use-compress-prog=pbzip2") - else - echo "WARING: pbzip2 isn't installed, single-threaded compressing is used." >&2 - TAR_OPTIONS+=("-j") - fi -else - TAR_OPTIONS+=("-j") + COMP_OPTIONS+=("-T0") fi +# Generic tar options: +TAR_OPTIONS=( + -cpP + --ignore-failed-read + "--xattrs-include='*.*'" + --numeric-owner + "--use-compress-prog=${COMP_OPTIONS[@]}" + ) + # if not in quiet mode, this message will be displayed: if [[ "$AGREE" != 'yes' ]] then