Merge pull request sukria#142 from cmilanf/devel

Added support for zstd compression

README.md

@@ -39,7 +39,7 @@ Features
 ### Backup Methods
 
 - Full backup only or Full + Incremental backup.
-- Archives formats: tar, tar.gz, tar.bz2, tar.xz, tar.lzma, dar, zip.
+- Archives formats: tar, tar.gz, tar.zst, tar.bz2, tar.xz, tar.lzma, dar, zip.
 - Backup to an attached disk, LAN or Internet.
 - Burns backup to CD/DVD with MD5 checksum verification.
 - Slice archives to 2 GB if using dar archives format.

backup-manager.conf.tpl

@@ -115,9 +115,9 @@ export BM_TARBALL_NAMEFORMAT="long"
 
 # Type of archives
 # Available types are:
-#     tar, tar.gz, tar.bz2, tar.xz, tar.lzma, dar, zip.
+#     tar, tar.gz, tar.bz2, tar.xz, tar.lzma, tar.zst, dar, zip.
 # Make sure to satisfy the appropriate dependencies 
-# (bzip2, dar, xz, lzma, ...).
+# (bzip2, dar, xz, lzma, zstd...).
 export BM_TARBALL_FILETYPE="tar.gz"
 
 # You can choose to build archives remotely over SSH.
@@ -218,7 +218,7 @@ export BM_MYSQL_HOST="localhost"
 # the port where MySQL listen to on the host
 export BM_MYSQL_PORT="3306"
 
-# which compression format to use? (gzip or bzip2)
+# which compression format to use? (gzip, bzip2 or zstd)
 export BM_MYSQL_FILETYPE="bzip2"
 
 # Extra options to append to mysqldump
@@ -255,7 +255,7 @@ export BM_PGSQL_HOST="localhost"
 # the port where PostgreSQL listen to on the host
 export BM_PGSQL_PORT="5432"
 
-# which compression format to use? (gzip or bzip2)
+# which compression format to use? (gzip, bzip2 or zstd)
 export BM_PGSQL_FILETYPE="bzip2"
 
 # Extra options to append to pg_dump

doc/user-guide.sgml

@@ -485,7 +485,7 @@ Suggested value: <tt>long</tt>.
 <sect2 id="BM_TARBALL_FILETYPE"><tt>BM_TARBALL_FILETYPE</tt>
 
 <p>
-<em>Type: enum(tar, tar.gz, tar.bz2, tar.xz, tar.lzma, zip, dar), default: <tt>tar.gz</tt>.</em>
+<em>Type: enum(tar, tar.gz, tar.bz2, tar.xz, tar.lzma, tar.zst, zip, dar), default: <tt>tar.gz</tt>.</em>
 
 <p>
 Basically, this configuration key defines the filetype of the resulting archive.
@@ -498,6 +498,9 @@ make sure you have the corresponding compressor installed.
 <p>
 For the best compression rate, choose <tt>tar.bz2</tt> or <tt>tar.xz</tt>.
 
+<p>
+For the best compression-performance balance, choose <tt>tar.zst</tt>.
+
 <p>
 Since version 0.7.1, &bmngr; supports <em>dar</em> archives. This archiver 
 provides some interesting features like the archive slicing.
@@ -511,6 +514,7 @@ Make sure to statisfy dependencies according to the filetype you choose:
 <item> tar.bz2 : needs "bzip2".
 <item> tar.xz : needs "xz".
 <item> tar.lzma : needs "lzma".
+<item> tar.zst : needs "zstd".
 <item> dar : needs "dar".
 <item> zip : needs "zip".
 </list>

lib/actions.sh

@@ -1,4 +1,4 @@
-# Copyright © 2005-2018 The Backup Manager Authors
+# Copyright � 2005-2018 The Backup Manager Authors
 #
 # See the AUTHORS file for details.
 #
@@ -187,16 +187,21 @@ function check_filetypes()
                 error "The BM_TARBALL_FILETYPE conf key is set to \"tar.bz2\" but bzip2 is not installed."
             fi
         ;;
-         "tar.xz" )
+        "tar.xz" )
             if [[ ! -x "$xz" ]]; then
                 error "The BM_TARBALL_FILETYPE conf key is set to \"tar.xz\" but xz is not installed."
             fi
         ;;
-         "tar.lzma" )
+        "tar.lzma" )
             if [[ ! -x "$lzma" ]]; then
                 error "The BM_TARBALL_FILETYPE conf key is set to \"tar.lzma\" but lzma is not installed."
             fi
         ;;
+        "tar.zst" )
+            if [[ ! -x "$zstd" ]]; then
+                error "The BM_TARBALL_FILETYPE conf key is set to \"tar.zst\" but zstd is not installed."
+            fi
+        ;;
         "dar" )
             if [[ ! -x "$dar" ]]; then
                 error "The BM_TARBALL_FILETYPE conf key is set to \"dar\" but dar is not installed."

lib/backup-methods.sh

@@ -196,6 +196,42 @@ function __exec_meta_command()
                 error "Compressor \$compress is needed."
             fi
         ;;
+        "zstd"|"zst")
+            if [[ "$compress" = "zstd" ]] ||
+               [[ "$compress" = "zst" ]]; then
+               compress_bin=$zstd
+               if [[ -z "$compress_bin" ]]; then
+                   error "zstd is not installed but zstd compression needed."
+               fi
+               ext="zst"
+            fi
+            if [[ -n "$compress_bin" ]] && [[ -x "$compress_bin" ]]; then
+                debug "$command > $file_to_create 2> $logfile"
+                tail_logfile "$logfile"
+                if [[ "$BM_ENCRYPTION_METHOD" = "gpg" ]]; then
+                    warning "Encryption with gpg is not supported with zstd compression at this release."
+                else
+                    $command 2> $logfile | $nice $compress_bin -f -q --rm > $file_to_create.$ext 2> $logfile
+                    cmdpipestatus=${PIPESTATUS[0]}
+                    debug "$command 2> $logfile | $nice $compress_bin -f -q --rm > $file_to_create.$ext 2> $logfile"
+                    file_to_create="$file_to_create.$ext"
+                fi
+
+                if [[ $? -gt 0 ]]; then
+                    warning "Unable to exec \$command; check \$logfile"
+                    rm -f $file_to_create
+                else
+                    if [[ $cmdpipestatus -gt 0 ]]; then
+                        warning "Unable to exec first piped command \$command; check \$logfile"
+                        rm -f $file_to_create
+                    else
+                        rm -f $logfile
+                    fi
+                fi
+            else
+                error "Compressor \$compress is needed."
+            fi
+        ;;
         ""|"uncompressed"|"none")
             if [[ "$verbosedebug" == "true" ]]; then
                 tail -f $logfile &
@@ -512,6 +548,10 @@ function __get_backup_tarball_remote_command()
             __get_flags_tar_blacklist "$target"
             command="$tar $blacklist $dumpsymlinks $BM_TARBALL_EXTRA_OPTIONS -p -c --lzma "$target""
         ;;
+        tar.zst)
+            __get_flags_tar_blacklist "$target"
+            command="$tar $blacklist $dumpsymlinks $BM_TARBALL_EXTRA_OPTIONS -p -c --zstd "$target""
+        ;;
         *)
             error "Remote tarball building is not possible with this archive filetype: \"$BM_TARBALL_FILETYPE\"."
         ;;
@@ -609,6 +649,13 @@ function __get_backup_tarball_command()
             __get_flags_tar_blacklist "$target"
             command="$tar $incremental $blacklist $dumpsymlinks $BM_TARBALL_EXTRA_OPTIONS -p -c --lzma -f"
         ;;
+        tar.zst)
+            if [[ ! -x $zstd ]]; then
+                error "The archive type \"tar.zst\" depends on the tool \"\$zstd\"."
+            fi
+            __get_flags_tar_blacklist "$target"
+            command="$tar $incremental $blacklist $dumpsymlinks $BM_TARBALL_EXTRA_OPTIONS -p -c --zstd -f"
+        ;;
         zip)
             if [[ ! -x $zip ]]; then
                 error "The archive type \"zip\" depends on the tool \"\$zip\"."
@@ -676,6 +723,7 @@ function build_encrypted_archive
 
     if [[ "$BM_TARBALL_FILETYPE" = "tar.xz" ]] ||
        [[ "$BM_TARBALL_FILETYPE" = "tar.lzma" ]] ||
+       [[ "$BM_TARBALL_FILETYPE" = "tar.zst" ]] ||
        [[ "$BM_TARBALL_FILETYPE" = "zip" ]] ||
        [[ "$BM_TARBALL_FILETYPE" = "dar" ]]; then
         error "The encryption is not yet possible with \"\$BM_TARBALL_FILETYPE\" archives."
@@ -813,7 +861,7 @@ function __make_local_tarball_token
             "dar")
                 __get_flags_dar_incremental "$dir_name"
             ;;
-            "tar"|"tar.gz"|"tar.bz2"|"tar.xz"|"tar.lzma")
+            "tar"|"tar.gz"|"tar.bz2"|"tar.xz"|"tar.lzma"|"tar.zst")
                 __get_flags_tar_incremental "$dir_name"
             ;;
             esac

lib/externals.sh

@@ -5,6 +5,7 @@ pbzip2=$(which pbzip2 2> /dev/null) || true
 gzip=$(which gzip 2> /dev/null) || true
 gpg=$(which gpg 2> /dev/null) || true
 xz=$(which xz 2> /dev/null) || true
+zstd=$(which zstd 2> /dev/null) || true
 lzma=$(which lzma 2> /dev/null) || true
 dar=$(which dar 2> /dev/null) || true
 tar=$(which tar 2> /dev/null) || true

po/backup-manager.pot

@@ -95,6 +95,12 @@ msgid ""
 "installed."
 msgstr ""
 
+#: ../lib/actions.sh:193
+msgid ""
+"The BM_TARBALL_FILETYPE conf key is set to \"tar.zst\" but zstd is not "
+"installed."
+msgstr ""
+
 #: ../lib/actions.sh:193
 msgid ""
 "The BM_TARBALL_FILETYPE conf key is set to \"tar.lzma\" but lzma is not "
@@ -217,6 +223,11 @@ msgstr ""
 msgid "The archive type \"tar.xz\" depends on the tool \"$xz\"."
 msgstr ""
 
+#: ../lib/backup-methods.sh:576
+#, sh-format
+msgid "The archive type \"tar.zst\" depends on the tool \"$zstd\"."
+msgstr ""
+
 #: ../lib/backup-methods.sh:576
 #, sh-format
 msgid "The archive type \"tar.lzma\" depends on the tool \"$lzma\"."

po/cs.po

@@ -199,6 +199,11 @@ msgstr "Typ archivu \"tar.bz2\" závisí na nástroji \"$bzip\"."
 msgid "The archive type \"tar.xz\" depends on the tool \"$xz\"."
 msgstr "Typ archivu \"tar.xz\" závisí na nástroji \"$xz\"."
 
+#: ../lib/backup-methods.sh:571
+#, sh-format
+msgid "The archive type \"tar.zst\" depends on the tool \"$zstd\"."
+msgstr "Typ archivu \"tar.zst\" závisí na nástroji \"$zstd\"."
+
 #: ../lib/backup-methods.sh:571
 #, sh-format
 msgid "The archive type \"tar.lzma\" depends on the tool \"$lzma\"."

po/de.po

@@ -217,6 +217,11 @@ msgstr "Der Archivtyp »tar.bz2« hängt vom Werkzeug »$bzip« ab."
 msgid "The archive type \"tar.xz\" depends on the tool \"$xz\"."
 msgstr "Der Archivtyp »tar.xz« hängt vom Werkzeug »$xz« ab."
 
+#: ../lib/backup-methods.sh:571
+#, sh-format
+msgid "The archive type \"tar.zst\" depends on the tool \"$zstd\"."
+msgstr "Der Archivtyp »tar.zst« hängt vom Werkzeug »$zstd« ab."
+
 #: ../lib/backup-methods.sh:571
 #, sh-format
 msgid "The archive type \"tar.lzma\" depends on the tool \"$lzma\"."

po/es.po

@@ -237,6 +237,11 @@ msgstr "El tipo de fichero \"tar.bz2\" depende de la herramienta \"$bzip\"."
 msgid "The archive type \"tar.xz\" depends on the tool \"$xz\"."
 msgstr "El tipo de fichero \"tar.xz\" depende de la herramienta \"$xz\"."
 
+#: ../lib/backup-methods.sh:571
+#, sh-format
+msgid "The archive type \"tar.zst\" depends on the tool \"$zstd\"."
+msgstr "El tipo de fichero \"tar.zst\" depende de la herramienta \"$zstd\"."
+
 #: ../lib/backup-methods.sh:571
 #, sh-format
 msgid "The archive type \"tar.lzma\" depends on the tool \"$lzma\"."

po/fr.po

@@ -86,6 +86,10 @@ msgstr "La clef BM_TARBALL_FILETYPE vaut « tar.bz2 » mais bzip2 n'est pas in
 msgid "The BM_TARBALL_FILETYPE conf key is set to \"tar.xz\" but xz is not installed."
 msgstr "La clef BM_TARBALL_FILETYPE vaut « tar.xz » mais xz n'est pas installé."
 
+#: ../lib/actions.sh:193
+msgid "The BM_TARBALL_FILETYPE conf key is set to \"tar.zst\" but zstd is not installed."
+msgstr "La clef BM_TARBALL_FILETYPE vaut « tar.zst » mais zstd n'est pas installé."
+
 #: ../lib/actions.sh:193
 msgid "The BM_TARBALL_FILETYPE conf key is set to \"tar.lzma\" but lzma is not installed."
 msgstr "La clef BM_TARBALL_FILETYPE vaut « tar.lzma » mais lzma n'est pas installé."
@@ -204,6 +208,11 @@ msgstr "Le type d'archive « tar.bz2 » dépend de l'outil « $bzip »."
 msgid "The archive type \"tar.xz\" depends on the tool \"$xz\"."
 msgstr "Le type d'archive « tar.xz » dépend de l'outil « $xz »."
 
+#: ../lib/backup-methods.sh:576
+#, sh-format
+msgid "The archive type \"tar.zst\" depends on the tool \"$zstd\"."
+msgstr "Le type d'archive « tar.zst » dépend de l'outil « $zstd »."
+
 #: ../lib/backup-methods.sh:576
 #, sh-format
 msgid "The archive type \"tar.lzma\" depends on the tool \"$lzma\"."

po/it.po

@@ -201,6 +201,11 @@ msgstr "Il tipo di archivio \"tar.bz2\" dipende dal tool \"$bzip\"."
 msgid "The archive type \"tar.xz\" depends on the tool \"$xz\"."
 msgstr "Il tipo di archivio \"tar.xz\" dipende dal tool \"$xz\"."
 
+#: ../lib/backup-methods.sh:571
+#, sh-format
+msgid "The archive type \"tar.zst\" depends on the tool \"$zstd\"."
+msgstr "Il tipo di archivio \"tar.zst\" dipende dal tool \"$zstd\"."
+
 #: ../lib/backup-methods.sh:571
 #, sh-format
 msgid "The archive type \"tar.lzma\" depends on the tool \"$lzma\"."

po/nl.po

@@ -199,6 +199,11 @@ msgstr "Het archieftype \"tar.bz2\" is afhankelijk van de tool \"$bzip\"."
 msgid "The archive type \"tar.xz\" depends on the tool \"$xz\"."
 msgstr "Het archieftype \"tar.xz\" is afhankelijk van de tool \"$xz\"."
 
+#: ../lib/backup-methods.sh:571
+#, sh-format
+msgid "The archive type \"tar.zst\" depends on the tool \"$zstd\"."
+msgstr "Het archieftype \"tar.zst\" is afhankelijk van de tool \"$zstd\"."
+
 #: ../lib/backup-methods.sh:571
 #, sh-format
 msgid "The archive type \"tar.lzma\" depends on the tool \"$lzma\"."

po/vi.po

@@ -200,6 +200,11 @@ msgstr "Kiểu kho nén « tar.bz2 » phụ thuộc vào công cụ « $bzip ».
 msgid "The archive type \"tar.xz\" depends on the tool \"$xz\"."
 msgstr "Kiểu kho nén « tar.xz » phụ thuộc vào công cụ « $xz »."
 
+#: ../lib/backup-methods.sh:571
+#, sh-format
+msgid "The archive type \"tar.zst\" depends on the tool \"$zstd\"."
+msgstr "Kiểu kho nén « tar.zst » phụ thuộc vào công cụ « $zstd »."
+
 #: ../lib/backup-methods.sh:571
 #, sh-format
 msgid "The archive type \"tar.lzma\" depends on the tool \"$lzma\"."

t/t22-tarball-zstd.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+
+set -e
+
+# Each test script should include testlib.sh
+source testlib.sh
+# When the test is ready, set this to false for nice outputs.
+# if you want to see what happens, use those flags
+#verbose="true"
+#warnings="true"
+#verbosedebug="true"
+
+# The conffile part of the test, see confs/* for details.
+source confs/base.conf
+source confs/tarball.conf
+
+export BM_ARCHIVE_ROOT="repository"
+export BM_ARCHIVE_METHOD="tarball"
+export BM_TARBALL_DIRECTORIES="$PWD"
+export BM_TARBALL_FILETYPE="tar.zst"
+source $locallib/sanitize.sh
+
+# The test actions
+
+if [[ -e $BM_ARCHIVE_ROOT ]]; then
+    rm -f $BM_ARCHIVE_ROOT/*
+fi
+
+bm_init_env
+bm_init_today
+
+create_directories
+make_archives
+
+name=$(get_dir_name $PWD long)
+if [[ -e "$BM_ARCHIVE_ROOT/$BM_ARCHIVE_PREFIX$name.$TODAY.master.tar.zst" ]]; then
+    rm -rf $BM_ARCHIVE_ROOT
+    exit 0
+else
+    exit 1
+fi

t/testlib.sh

@@ -22,6 +22,7 @@ bzip=$(which bzip2) || true
 gzip=$(which gzip) || true
 gpg=$(which gpg) || true
 xz=$(which xz) || true
+zstd=$(which zstd) || true
 lzma=$(which lzma) || true
 dar=$(which dar) || true
 tar=$(which tar) || true