test: add deb coverage and a few more sanity checks to TEST-88-UPGRADE (#43162)

mkosi/mkosi.conf.d/debian-ubuntu/mkosi.prepare

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: LGPL-2.1-or-later
+set -e
+set -o pipefail
+
+if [[ "$1" == "build" ]]; then
+    exit 0
+fi
+
+pkgdir=/usr/host-pkgs
+develver=$(dpkg-query --root="$BUILDROOT" --showformat='${Version}' --show systemd)
+distrover=$(
+    apt-cache madison systemd |
+        awk -F '|' '$3 !~ /file:\/+repository/ && !found { gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2; found=1 }'
+)
+
+if [[ -z $distrover ]]; then
+    echo "Skipping upgrade test, no distro version found"
+    exit 0
+fi
+
+if [[ ${distrover#*:} =~ ^([0-9]+) ]]; then
+    distromajor=${BASH_REMATCH[1]}
+else
+    echo "Skipping upgrade test, unable to parse distro version: $distrover"
+    exit 0
+fi
+
+if [[ ${develver#*:} =~ ^([0-9]+) ]]; then
+    develmajor=${BASH_REMATCH[1]}
+else
+    echo "Skipping upgrade test, unable to parse development version: $develver"
+    exit 0
+fi
+
+deltaver=$((develmajor - distromajor))
+
+if [[ $deltaver -le 0 ]]; then
+    echo "Skipping upgrade test, distro version is newer or equal: $distrover"
+    exit 0
+fi
+
+distro_packages=()
+devel_packages=()
+
+while IFS=$'\t' read -r package source package_architecture; do
+    [[ $source == systemd ]] || continue
+    [[ $package_architecture == "$DISTRIBUTION_ARCHITECTURE" || $package_architecture == all ]] || continue
+    [[ $package != *-dbgsym* ]] || continue
+
+    devel_packages+=("$package=$(dpkg-query --root="$BUILDROOT" --showformat='${Version}' --show "$package")")
+
+    package_versions=$(apt-cache madison "$package")
+    if awk -F '|' -v version="$distrover" '
+            $3 !~ /file:\/+repository/ {
+                gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2)
+                if ($2 == version)
+                    found=1
+            }
+            END {
+                exit !found
+            }
+        ' <<<"$package_versions"; then
+        distro_packages+=("$package=$distrover")
+    fi
+done < <(dpkg-query --root="$BUILDROOT" --show --showformat='${binary:Package}\t${source:Package}\t${Architecture}\n')
+
+if (( ${#distro_packages[@]} == 0 )); then
+    echo "Skipping upgrade test, no distro packages found"
+    exit 0
+fi
+
+if (( ${#devel_packages[@]} == 0 )); then
+    echo "Skipping upgrade test, no development packages found"
+    exit 0
+fi
+
+rm -rf "$BUILDROOT$pkgdir/distro" "$BUILDROOT$pkgdir/devel"
+mkdir -p "$BUILDROOT$pkgdir"/{distro,devel}
+
+# apt-get download writes packages to the current directory
+(
+    cd "$BUILDROOT$pkgdir/distro"
+    apt-get download "${distro_packages[@]}"
+)
+
+(
+    cd "$BUILDROOT$pkgdir/devel"
+    apt-get download "${devel_packages[@]}"
+)

test/units/TEST-88-UPGRADE.sh

@@ -12,15 +12,38 @@ if ! [[ -d $pkgdir ]]; then
     exit 77
 fi
 
-if ! command -v dnf >/dev/null; then
-    echo 'dnf not found, skipping test.' >/skipped
+if command -v dnf >/dev/null; then
+    package_extension=rpm
+    downgrade=(dnf downgrade --nogpgcheck -y --allowerasing --disablerepo '*')
+    upgrade=(dnf -y upgrade --nogpgcheck --disablerepo '*')
+elif command -v apt-get >/dev/null; then
+    package_extension=deb
+    apt_sources=/run/TEST-88-UPGRADE.sources
+    apt_source_parts=/run/TEST-88-UPGRADE.sources.d
+    : >"$apt_sources"
+    mkdir -p "$apt_source_parts"
+    apt_options=(
+        -o "Dir::Etc::sourcelist=$apt_sources"
+        -o "Dir::Etc::sourceparts=$apt_source_parts"
+        --no-install-recommends
+    )
+    downgrade=(apt-get "${apt_options[@]}" install --allow-downgrades -y)
+    upgrade=(apt-get "${apt_options[@]}" install -y)
+else
+    echo 'No supported package manager found, skipping test.' >/skipped
     exit 77
 fi
 
 minor=$(systemctl --version | awk '/^systemd/{print$2}')
 networkd=
+resolved=
+udev=
 unitscmd='systemctl list-units --failed *systemd*'
 
+if command -v udevadm >/dev/null && systemctl is-active --quiet systemd-udevd.service; then
+    udev=1
+fi
+
 if [[ $($unitscmd --output json | jq length) -gt 0 ]]; then
     echo 'Systemd failed units found before the test:'
     $unitscmd
@@ -29,6 +52,22 @@ fi
 
 check_sd() {
     local unit fail=0 timer1_new timer2_new
+
+    if ! systemctl daemon-reload; then
+        echo 'System manager reload failed after the test!'
+        fail=1
+    fi
+
+    if ! systemd-run --quiet --wait --collect --service-type=exec true; then
+        echo 'Transient service failed after the test!'
+        fail=1
+    fi
+
+    if [[ -n $udev ]] && ! udevadm control --ping --timeout=5; then
+        echo 'Udev failed after the test!'
+        fail=1
+    fi
+
     for unit in $($unitscmd --output json | jq -r '.[].unit'); do
         if ! grep -sxqF "$unit" /tmp/failed-units; then
             fail=1
@@ -47,6 +86,13 @@ check_sd() {
         fi
     fi
 
+    if [[ -n $resolved ]]; then
+        if ! resolvectl status; then
+            echo 'Resolved failed after the test!'
+            fail=1
+        fi
+    fi
+
     if ! loginctl list-sessions; then
         echo 'Loginctl failed after the test!'
         fail=1
@@ -84,12 +130,15 @@ timer2=$(systemctl show -P NextElapseUSecRealtime upgrade_timer_test.timer)
 # FIXME: See https://github.com/systemd/systemd/pull/39293
 systemctl stop systemd-networkd-resolve-hook.socket || true
 
-dnf downgrade --nogpgcheck -y --allowerasing --disablerepo '*' "$pkgdir"/distro/*.rpm
+"${downgrade[@]}" "$pkgdir"/distro/*."$package_extension"
 
-# Some distros don't ship networkd, so the test will always fail
+# Some distros don't ship networkd or resolved, so only test them when available
 if command -v networkctl >/dev/null; then
     networkd=1
 fi
+if command -v resolvectl >/dev/null && systemctl is-active --quiet systemd-resolved.service; then
+    resolved=1
+fi
 
 newminor=$(systemctl --version | awk '/^systemd/{print$2}')
 
@@ -105,7 +154,7 @@ fi
 check_sd
 
 # Finally test the upgrade
-dnf -y upgrade --nogpgcheck --disablerepo '*' "$pkgdir"/devel/*.rpm
+"${upgrade[@]}" "$pkgdir"/devel/*."$package_extension"
 
 # TODO: sanity checks
 check_sd