From 4274a52812887cae479684edf29a77a660e108d6 Mon Sep 17 00:00:00 2001 From: Eric Renfro Date: Sun, 7 Jul 2024 14:59:30 -0400 Subject: [PATCH 01/10] Update restore script to use apt-mark, apt-get, and comm --- scripts/os_debian | 101 ++++++++++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 43 deletions(-) diff --git a/scripts/os_debian b/scripts/os_debian index 8ea880d..8c3a693 100755 --- a/scripts/os_debian +++ b/scripts/os_debian @@ -7,10 +7,10 @@ function is_bin_in_path { } hook_before() { - if ! is_bin_in_path aptitude; then - echo "aptitude needs to be installed for backups to work properly." - exit 1 - fi + #if ! is_bin_in_path aptitude; then + # echo "aptitude needs to be installed for backups to work properly." + # exit 1 + #fi if ! is_bin_in_path rsync; then echo "rsync needs to be installed for backups to work properly." @@ -21,7 +21,8 @@ hook_before() { pushd "$restoreDir" || exit 2 dpkg --get-selections > Package.list - aptitude search --disable-columns -F%p '~i!~M!~v' > InstallOnly.list + #aptitude search --disable-columns -F%p '~i!~M!~v' > InstallOnly.list + apt-mark showmanual | sort > InstallOnly.list #apt-key exportall > /etc/restore/Repo.keys cp -a /etc/apt/sources.list "$restoreDir/" rsync -avhHi /etc/apt/sources.list.d "$restoreDir/" @@ -40,6 +41,13 @@ if [[ ! -d "trusted.gpg.d" ]]; then exit 1 fi +TMPDIR="\$(mktemp -d -t restore-XXXX)" + +cleanup() { + [[ -n "\$TMPDIR" && -d "\$TMPDIR" ]] && rm -rf "\$TMPDIR" +} +trap cleanup EXIT + #apt-key add /etc/restore/Repo.keys #dpkg --set-selections < /etc/restore/Package.list #apt-get dselect-upgrade @@ -48,9 +56,8 @@ install="" echo " * Installing required packages" dpkg-query -s 'rsync' &>/dev/null || install+=" rsync" -dpkg-query -s 'aptitude' &>/dev/null || install+=" aptitude" -dpkg-query -s 'borgbackup' &>/dev/null || install+=" borgbackup" -dpkg-query -s 'borgmatic' &>/dev/null || install+=" borgmatic" +#dpkg-query -s 'borgbackup' &>/dev/null || install+=" borgbackup" +#dpkg-query -s 'borgmatic' &>/dev/null || install+=" borgmatic" dpkg-query -s 'apt-transport-https' &>/dev/null || install+=" apt-transport-https" if [[ -n "\$install" ]]; then @@ -60,50 +67,58 @@ fi echo " * Enabling 32-bit packages" grep ':i386' InstallOnly.list &>/dev/null && dpkg --add-architecture i386 -echo " * Restoring repositories and keys" -cp -a sources.list /etc/apt/sources.list -rsync --ignore-existing -raz sources.list.d/ /etc/apt/sources.list.d/ -rsync --ignore-existing -raz trusted.gpg.d/ /etc/apt/trusted.gpg.d/ -[[ -d keyrings ]] && rsync --ignore-existing -raz keyrings/ /etc/apt/keyrings/ +echo " * Checking for flatpak" -apt update +if [[ -r "sources.list" ]]; then + echo + echo "==============================" + echo "INSTALL REPOS" + echo "==============================" + echo + read -p "Do you want to continue to install backup repositories? [Y/n] " -n 1 -sr promptRepos + if [[ "\$promptRepos" =~ ^[Yy]$ ]]; then + echo "Yes" + cp -a sources.list /etc/apt/sources.list + rsync --ignore-existing -raz sources.list.d/ /etc/apt/sources.list.d/ + rsync --ignore-existing -raz trusted.gpg.d/ /etc/apt/trusted.gpg.d/ + [[ -d keyrings ]] && rsync --ignore-existing -raz keyrings/ /etc/apt/keyrings/ + apt update + else + echo -e "Cancelled\n" + fi +fi -failinstall="" flatpak=0 - -echo " * Gathering installed packages" -#xargs aptitude --schedule-only install < InstallOnly.list -while read p; do - if [[ "\$p" = "flatpak" ]]; then - flatpak=1 - fi - - dpkg-query -s "\$p" &>/dev/null - if [[ "\$?" -eq 1 ]]; then - aptitude --schedule-only install "\$p" - if [[ "\$?" -ne 0 ]]; then - if [[ -z "\$failinstall" ]]; then - failinstall+="\$p" - else - failinstall+=", \$p" - fi - fi - fi -done < InstallOnly.list - -echo "Packages that failed to schedule for install:" -echo "\$failinstall" - -echo " * Restoring installed packages (please confirm)" -aptitude install - +grep 'flatpak' InstallOnly.list &>/dev/null && flatpak=1 if [[ "\$flatpak" -eq 1 ]]; then echo " * Adding flatpak repo: Flathub" flatpak remote-add --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo fi +echo +echo "==============================" +echo "INSTALL PACKAGES" +echo "==============================" +echo +read -p "About to install the system packages per the backup. Do you want to continue? [Y/n] " -n 1 -sr promptPkgs +if [[ "\$promptPkgs" =~ ^[Yy] ]]; then + echo "Yes" + comm --nocheck-order -23 InstallOnly.list <(apt-mark showmanual|sort) grep -Ev 'linux-image|linux-headers' > "\${TMPDIR}/diff.list" + apt-get --simulate install \$(cat diff.list) |& awk '/^E: Unable to locate package / {print \$NF}' | sort > "\${TMPDIR}/diff.fail" + comm --nocheck-order -23 "\${TMPDIR}/diff.list" "\${TMPDIR}/diff.fail" | xargs apt-get install + + read -p "Did everything above look okay and do you want to proceed? [Y/n] " -n 1 -sr promptPkgsDo + if [[ "\$promptPkgsDo" =~ ^[Yy] ]]; then + comm --nocheck-order -23 "\${TMPDIR}/diff.list" "\${TMPDIR}/diff.fail" | xargs apt-get install + else + echo -e "Cancelled\n" + fi +else + echo -e "Cancelled\n" +fi + echo "Packages that failed to schedule for install:" -echo "\$failinstall" +cat "\${TMPDIR}/diff.fail" EOF chmod ug+rx restore.sh popd || exit 2 From 5cf85362dc369e78499d7575c067cde737f56753 Mon Sep 17 00:00:00 2001 From: Eric Renfro Date: Sun, 7 Jul 2024 15:51:24 -0400 Subject: [PATCH 02/10] Correctly detect distro for Ubuntu --- scripts/os | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/os b/scripts/os index 8f8f520..16a3204 100755 --- a/scripts/os +++ b/scripts/os @@ -19,7 +19,7 @@ createRestoreDir() { } checkOS() { - if [[ "$kernel" == "Darin" ]]; then + if [[ "$kernel" == "Darwin" ]]; then return 0 elif [[ "$kernel" == "Linux" ]]; then if [[ -f /etc/os-release ]]; then @@ -27,7 +27,7 @@ checkOS() { DISTRO="$ID" DISTRO_CURRENT="$DISTRO" if [[ -n "$ID_LIKE" ]]; then - DISTRO="$ID_LIKE" + DISTRO="${ID_LIKE# *}" fi return 0 else @@ -45,6 +45,7 @@ runOsHook() { "${scriptPath}/os_macos" "$hook" elif [[ "$kernel" == "Linux" ]]; then case "$DISTRO" in + ubuntu) "${scriptPath}/os_debian" "$hook";; debian) "${scriptPath}/os_debian" "$hook";; fedora) "${scriptPath}/os_fedora" "$hook";; garuda) "${scriptPath}/os_garuda" "$hook";; From cf3831c13ba9cb0ca8779b776a2771fd6e000824 Mon Sep 17 00:00:00 2001 From: Eric Renfro Date: Sun, 7 Jul 2024 15:57:40 -0400 Subject: [PATCH 03/10] Fix distro detection proper --- scripts/os | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/os b/scripts/os index 16a3204..8e7d90a 100755 --- a/scripts/os +++ b/scripts/os @@ -27,7 +27,7 @@ checkOS() { DISTRO="$ID" DISTRO_CURRENT="$DISTRO" if [[ -n "$ID_LIKE" ]]; then - DISTRO="${ID_LIKE# *}" + DISTRO="${ID_LIKE%% *}" fi return 0 else From 4a52a1d74d1d7d14067d8e19f99a05065a5ce51f Mon Sep 17 00:00:00 2001 From: Eric Renfro Date: Sun, 7 Jul 2024 16:01:25 -0400 Subject: [PATCH 04/10] Fix typo in debian agent --- scripts/os_debian | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/os_debian b/scripts/os_debian index 8c3a693..c415359 100755 --- a/scripts/os_debian +++ b/scripts/os_debian @@ -103,7 +103,7 @@ echo read -p "About to install the system packages per the backup. Do you want to continue? [Y/n] " -n 1 -sr promptPkgs if [[ "\$promptPkgs" =~ ^[Yy] ]]; then echo "Yes" - comm --nocheck-order -23 InstallOnly.list <(apt-mark showmanual|sort) grep -Ev 'linux-image|linux-headers' > "\${TMPDIR}/diff.list" + comm --nocheck-order -23 InstallOnly.list <(apt-mark showmanual|sort) | grep -Ev 'linux-image|linux-headers' > "\${TMPDIR}/diff.list" apt-get --simulate install \$(cat diff.list) |& awk '/^E: Unable to locate package / {print \$NF}' | sort > "\${TMPDIR}/diff.fail" comm --nocheck-order -23 "\${TMPDIR}/diff.list" "\${TMPDIR}/diff.fail" | xargs apt-get install From a69d966c2cafe490d79cf63be1c5cb4de9a37475 Mon Sep 17 00:00:00 2001 From: Eric Renfro Date: Sun, 7 Jul 2024 16:04:55 -0400 Subject: [PATCH 05/10] Fix diff.list usage in tmpdir --- scripts/os_debian | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/os_debian b/scripts/os_debian index c415359..adfeaf7 100755 --- a/scripts/os_debian +++ b/scripts/os_debian @@ -104,7 +104,7 @@ read -p "About to install the system packages per the backup. Do you want to con if [[ "\$promptPkgs" =~ ^[Yy] ]]; then echo "Yes" comm --nocheck-order -23 InstallOnly.list <(apt-mark showmanual|sort) | grep -Ev 'linux-image|linux-headers' > "\${TMPDIR}/diff.list" - apt-get --simulate install \$(cat diff.list) |& awk '/^E: Unable to locate package / {print \$NF}' | sort > "\${TMPDIR}/diff.fail" + apt-get --simulate install \$(cat "\${TMPDIR}/diff.list") |& awk '/^E: Unable to locate package / {print \$NF}' | sort > "\${TMPDIR}/diff.fail" comm --nocheck-order -23 "\${TMPDIR}/diff.list" "\${TMPDIR}/diff.fail" | xargs apt-get install read -p "Did everything above look okay and do you want to proceed? [Y/n] " -n 1 -sr promptPkgsDo From f1efcf62740eae3a323fc76d597af7c7fc9736a7 Mon Sep 17 00:00:00 2001 From: Eric Renfro Date: Sun, 7 Jul 2024 16:09:24 -0400 Subject: [PATCH 06/10] Add show of failed packages for review before install and after --- scripts/os_debian | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/os_debian b/scripts/os_debian index adfeaf7..1ac8bb0 100755 --- a/scripts/os_debian +++ b/scripts/os_debian @@ -107,6 +107,11 @@ if [[ "\$promptPkgs" =~ ^[Yy] ]]; then apt-get --simulate install \$(cat "\${TMPDIR}/diff.list") |& awk '/^E: Unable to locate package / {print \$NF}' | sort > "\${TMPDIR}/diff.fail" comm --nocheck-order -23 "\${TMPDIR}/diff.list" "\${TMPDIR}/diff.fail" | xargs apt-get install + echo + echo "Packages that were omitted because they could not be found:" + cat "\${TMPDIR}/diff.fail" | tr '\n' ' ' | fold -s + echo + read -p "Did everything above look okay and do you want to proceed? [Y/n] " -n 1 -sr promptPkgsDo if [[ "\$promptPkgsDo" =~ ^[Yy] ]]; then comm --nocheck-order -23 "\${TMPDIR}/diff.list" "\${TMPDIR}/diff.fail" | xargs apt-get install @@ -118,7 +123,7 @@ else fi echo "Packages that failed to schedule for install:" -cat "\${TMPDIR}/diff.fail" +cat "\${TMPDIR}/diff.fail" | tr '\n' ' ' | fold -s EOF chmod ug+rx restore.sh popd || exit 2 From 3356bab676d63a032cf67695c08bc18acd9de4d3 Mon Sep 17 00:00:00 2001 From: Eric Renfro Date: Sun, 7 Jul 2024 16:17:01 -0400 Subject: [PATCH 07/10] Stylize output with bold --- scripts/os_debian | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/scripts/os_debian b/scripts/os_debian index 1ac8bb0..9450143 100755 --- a/scripts/os_debian +++ b/scripts/os_debian @@ -42,6 +42,8 @@ if [[ ! -d "trusted.gpg.d" ]]; then fi TMPDIR="\$(mktemp -d -t restore-XXXX)" +bold="$(tput bold)" +normal="$(tput sgr0)" cleanup() { [[ -n "\$TMPDIR" && -d "\$TMPDIR" ]] && rm -rf "\$TMPDIR" @@ -54,7 +56,7 @@ trap cleanup EXIT install="" -echo " * Installing required packages" +echo "\${bold} * Installing required packages\${normal}" dpkg-query -s 'rsync' &>/dev/null || install+=" rsync" #dpkg-query -s 'borgbackup' &>/dev/null || install+=" borgbackup" #dpkg-query -s 'borgmatic' &>/dev/null || install+=" borgmatic" @@ -64,18 +66,25 @@ if [[ -n "\$install" ]]; then apt -y install \$install fi -echo " * Enabling 32-bit packages" +echo "\${bold} * Enabling 32-bit packages\${normal}" grep ':i386' InstallOnly.list &>/dev/null && dpkg --add-architecture i386 -echo " * Checking for flatpak" +echo "\${bold} * Checking for flatpak\${normal}" +flatpak=0 +grep 'flatpak' InstallOnly.list &>/dev/null && flatpak=1 +if [[ "\$flatpak" -eq 1 ]]; then + echo " * Adding flatpak repo: Flathub" + flatpak remote-add --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo +fi if [[ -r "sources.list" ]]; then - echo + echo "\${bold}" echo "==============================" echo "INSTALL REPOS" echo "==============================" - echo - read -p "Do you want to continue to install backup repositories? [Y/n] " -n 1 -sr promptRepos + echo "\${normal} + + read -p "\${bold}Do you want to continue to install backup repositories? [Y/n]\${normal} " -n 1 -sr promptRepos if [[ "\$promptRepos" =~ ^[Yy]$ ]]; then echo "Yes" cp -a sources.list /etc/apt/sources.list @@ -88,18 +97,12 @@ if [[ -r "sources.list" ]]; then fi fi -flatpak=0 -grep 'flatpak' InstallOnly.list &>/dev/null && flatpak=1 -if [[ "\$flatpak" -eq 1 ]]; then - echo " * Adding flatpak repo: Flathub" - flatpak remote-add --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo -fi - -echo +echo "\${bold}" echo "==============================" echo "INSTALL PACKAGES" echo "==============================" -echo +echo "\${normal}" + read -p "About to install the system packages per the backup. Do you want to continue? [Y/n] " -n 1 -sr promptPkgs if [[ "\$promptPkgs" =~ ^[Yy] ]]; then echo "Yes" @@ -111,8 +114,9 @@ if [[ "\$promptPkgs" =~ ^[Yy] ]]; then echo "Packages that were omitted because they could not be found:" cat "\${TMPDIR}/diff.fail" | tr '\n' ' ' | fold -s echo + echo - read -p "Did everything above look okay and do you want to proceed? [Y/n] " -n 1 -sr promptPkgsDo + read -p "\${bold}Did everything above look okay and do you want to proceed? [Y/n]\${normal} " -n 1 -sr promptPkgsDo if [[ "\$promptPkgsDo" =~ ^[Yy] ]]; then comm --nocheck-order -23 "\${TMPDIR}/diff.list" "\${TMPDIR}/diff.fail" | xargs apt-get install else @@ -122,7 +126,7 @@ else echo -e "Cancelled\n" fi -echo "Packages that failed to schedule for install:" +echo "\${bold}Packages that failed to schedule for install:\${normal}" cat "\${TMPDIR}/diff.fail" | tr '\n' ' ' | fold -s EOF chmod ug+rx restore.sh From 7f4c3572d782fb5d56527c3994303c099ce2af06 Mon Sep 17 00:00:00 2001 From: Eric Renfro Date: Sun, 7 Jul 2024 16:17:50 -0400 Subject: [PATCH 08/10] Fix typo missing quote --- scripts/os_debian | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/os_debian b/scripts/os_debian index 9450143..8f719f8 100755 --- a/scripts/os_debian +++ b/scripts/os_debian @@ -82,7 +82,7 @@ if [[ -r "sources.list" ]]; then echo "==============================" echo "INSTALL REPOS" echo "==============================" - echo "\${normal} + echo "\${normal}" read -p "\${bold}Do you want to continue to install backup repositories? [Y/n]\${normal} " -n 1 -sr promptRepos if [[ "\$promptRepos" =~ ^[Yy]$ ]]; then From 6545a487e6815248a6a6cc960a3ca7fdba8dca6c Mon Sep 17 00:00:00 2001 From: Eric Renfro Date: Sun, 7 Jul 2024 16:21:42 -0400 Subject: [PATCH 09/10] Fix missing bold prompt and cleanup further --- scripts/os_debian | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/scripts/os_debian b/scripts/os_debian index 8f719f8..3dfe0f1 100755 --- a/scripts/os_debian +++ b/scripts/os_debian @@ -7,11 +7,6 @@ function is_bin_in_path { } hook_before() { - #if ! is_bin_in_path aptitude; then - # echo "aptitude needs to be installed for backups to work properly." - # exit 1 - #fi - if ! is_bin_in_path rsync; then echo "rsync needs to be installed for backups to work properly." exit 1 @@ -20,10 +15,8 @@ hook_before() { mkdir -p "$restoreDir" || exit 1 pushd "$restoreDir" || exit 2 - dpkg --get-selections > Package.list - #aptitude search --disable-columns -F%p '~i!~M!~v' > InstallOnly.list + dpkg --get-selections | sort > Package.list apt-mark showmanual | sort > InstallOnly.list - #apt-key exportall > /etc/restore/Repo.keys cp -a /etc/apt/sources.list "$restoreDir/" rsync -avhHi /etc/apt/sources.list.d "$restoreDir/" rsync -avhHi /etc/apt/trusted.gpg.d "$restoreDir/" @@ -103,7 +96,7 @@ echo "INSTALL PACKAGES" echo "==============================" echo "\${normal}" -read -p "About to install the system packages per the backup. Do you want to continue? [Y/n] " -n 1 -sr promptPkgs +read -p "\${bold}About to install the system packages per the backup. Do you want to continue? [Y/n]\${normal} " -n 1 -sr promptPkgs if [[ "\$promptPkgs" =~ ^[Yy] ]]; then echo "Yes" comm --nocheck-order -23 InstallOnly.list <(apt-mark showmanual|sort) | grep -Ev 'linux-image|linux-headers' > "\${TMPDIR}/diff.list" @@ -119,15 +112,19 @@ if [[ "\$promptPkgs" =~ ^[Yy] ]]; then read -p "\${bold}Did everything above look okay and do you want to proceed? [Y/n]\${normal} " -n 1 -sr promptPkgsDo if [[ "\$promptPkgsDo" =~ ^[Yy] ]]; then comm --nocheck-order -23 "\${TMPDIR}/diff.list" "\${TMPDIR}/diff.fail" | xargs apt-get install + + echo + echo + echo "\${bold}Packages that failed to schedule for install:\${normal}" + cat "\${TMPDIR}/diff.fail" | tr '\n' ' ' | fold -s + echo + echo else echo -e "Cancelled\n" fi else echo -e "Cancelled\n" fi - -echo "\${bold}Packages that failed to schedule for install:\${normal}" -cat "\${TMPDIR}/diff.fail" | tr '\n' ' ' | fold -s EOF chmod ug+rx restore.sh popd || exit 2 From 861657f0ad29c84093142aa2bab3b204b601382b Mon Sep 17 00:00:00 2001 From: Eric Renfro Date: Sun, 7 Jul 2024 16:36:14 -0400 Subject: [PATCH 10/10] Cleanup and updates for final push --- scripts/os | 11 +++++------ scripts/{os_garuda => os_arch} | 0 2 files changed, 5 insertions(+), 6 deletions(-) rename scripts/{os_garuda => os_arch} (100%) diff --git a/scripts/os b/scripts/os index 8e7d90a..9df90ad 100755 --- a/scripts/os +++ b/scripts/os @@ -12,7 +12,7 @@ fi createRestoreDir() { if [[ -d "${restoreDir}" ]]; then - rm -rf "${restoreDir}/*" || exit 2 + rm -rf "${restoreDir:?}/*" || exit 2 else mkdir -p "${restoreDir}" || exit 2 fi @@ -45,10 +45,9 @@ runOsHook() { "${scriptPath}/os_macos" "$hook" elif [[ "$kernel" == "Linux" ]]; then case "$DISTRO" in - ubuntu) "${scriptPath}/os_debian" "$hook";; - debian) "${scriptPath}/os_debian" "$hook";; + debian|ubuntu) "${scriptPath}/os_debian" "$hook";; fedora) "${scriptPath}/os_fedora" "$hook";; - garuda) "${scriptPath}/os_garuda" "$hook";; + arch) "${scriptPath}/os_arch" "$hook";; solus) "${scriptPath}/os_solus" "$hook";; opensuse-leap) "${scriptPath}/os_suse" "$hook";; opensuse-tumbleweed) "${scriptPath}/os_suse" "$hook";; @@ -63,10 +62,10 @@ runOsHook() { hook_before() { checkOS || exit 200 - pushd "$scriptPath" &>/dev/null + pushd "$scriptPath" &>/dev/null || exit 201 git checkout -- . git pull - popd &>/dev/null + popd &>/dev/null || exit 201 createRestoreDir || exit $? runOsHook before diff --git a/scripts/os_garuda b/scripts/os_arch similarity index 100% rename from scripts/os_garuda rename to scripts/os_arch