From e2e536224d0623646cc82c88640207784e2e1417 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Thu, 10 Sep 2026 18:15:27 +0000 Subject: [PATCH 1/4] core/automount: do not abort when there is no autofs to talk to Releasing the requests the kernel queued goes through `open_ioctl_fd()`, which asserts on the manager fd for /dev/autofs. That fd stays unset when `open_dev_autofs()` fails while coldplugging an automount, while the deserialized pipe and its tokens outlive it. Report the failure to the caller instead of asserting/crashing PID 1. Signed-off-by: Martin Pitt --- src/core/automount.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/automount.c b/src/core/automount.c index a0529cda5f..29543590e4 100644 --- a/src/core/automount.c +++ b/src/core/automount.c @@ -458,6 +458,11 @@ static int automount_send_ready(Automount *a, Set *tokens, int status) { if (set_isempty(tokens)) return 0; + /* We have nothing to talk to the autofs with, e.g. because open_dev_autofs() failed at coldplug and + * left us with the deserialized pipe and tokens. Report that instead of asserting below. */ + if (UNIT(a)->manager->dev_autofs_fd < 0 || !a->where) + return -EBADF; + ioctl_fd = open_ioctl_fd(UNIT(a)->manager->dev_autofs_fd, a->where, a->dev_id); if (ioctl_fd < 0) return ioctl_fd; From 4558062cadfbe20bbfaf87f476f45cf46acd9d51 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Fri, 11 Sep 2026 01:36:07 +0000 Subject: [PATCH 2/4] core/automount: log when pending requests cannot be released When an automount goes away, answering the requests the kernel already queued is the only thing that unblocks the processes waiting on them, and nothing retries it later. If the `EHOSTDOWN` calls fail, it leaves the requesting processes hung with no trace in the journal. Log the failures to make this easier to debug. Also move that teardown into a helper function, the next commit needs it as well. Signed-off-by: Martin Pitt --- src/core/automount.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/core/automount.c b/src/core/automount.c index 29543590e4..f82c8d0a40 100644 --- a/src/core/automount.c +++ b/src/core/automount.c @@ -59,6 +59,23 @@ static void automount_init(Unit *u) { UNIT(a)->ignore_on_isolate = true; } +/* Nothing else can answer the requests the kernel already queued for us, and it keeps their processes + * blocked until we do. Make a failure to do so at least visible. The status is what the mount requests get: + * an expire request can only ever fail. */ +static void automount_release_requests(Automount *a, int status) { + int r; + + assert(a); + + r = automount_send_ready(a, a->tokens, status); + if (r < 0) + log_unit_warning_errno(UNIT(a), r, "Failed to release pending automount requests, ignoring: %m"); + + r = automount_send_ready(a, a->expire_tokens, -EHOSTDOWN); + if (r < 0) + log_unit_warning_errno(UNIT(a), r, "Failed to release pending automount expire requests, ignoring: %m"); +} + static void unmount_autofs(Automount *a) { int r; @@ -73,8 +90,7 @@ static void unmount_autofs(Automount *a) { /* If we reload/reexecute things we keep the mount point around */ if (!IN_SET(UNIT(a)->manager->objective, MANAGER_RELOAD, MANAGER_REEXECUTE)) { - automount_send_ready(a, a->tokens, -EHOSTDOWN); - automount_send_ready(a, a->expire_tokens, -EHOSTDOWN); + automount_release_requests(a, -EHOSTDOWN); if (a->where) { r = repeat_unmount(a->where, MNT_DETACH|UMOUNT_NOFOLLOW); From f65e67ff0fa484f62ea64e8df9b16655d507e636 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Wed, 9 Sep 2026 08:21:58 +0000 Subject: [PATCH 3/4] core/automount: tear down autofs when the unit vanished across reload When an automount unit ceases to exist across daemon-reload (observed: because a generator no longer emits it), its deserialized state still makes `automount_coldplug()` pick up the autofs pipe and resume the unit as waiting/running. But nothing can serve that mount point anymore. The kernel keeps blocking every process that touches the mount point until PID 1 answers (which it won't), forever. To fix this, fail pending requests and unmount the autofs when coldplugging an automount unit that is not loaded anymore. Fixes #43700 Signed-off-by: Martin Pitt --- src/core/automount.c | 56 ++++++++- test/units/TEST-07-PID1.issue-43700.sh | 160 +++++++++++++++++++++++++ 2 files changed, 210 insertions(+), 6 deletions(-) create mode 100755 test/units/TEST-07-PID1.issue-43700.sh diff --git a/src/core/automount.c b/src/core/automount.c index f82c8d0a40..43b34c211d 100644 --- a/src/core/automount.c +++ b/src/core/automount.c @@ -48,6 +48,7 @@ static int automount_dispatch_io(sd_event_source *s, int fd, uint32_t events, vo static int automount_start_expire(Automount *a); static void automount_stop_expire(Automount *a); static int automount_send_ready(Automount *a, Set *tokens, int status); +static void automount_enter_dead(Automount *a, AutomountResult f); static void automount_init(Unit *u) { Automount *a = ASSERT_PTR(AUTOMOUNT(u)); @@ -76,7 +77,7 @@ static void automount_release_requests(Automount *a, int status) { log_unit_warning_errno(UNIT(a), r, "Failed to release pending automount expire requests, ignoring: %m"); } -static void unmount_autofs(Automount *a) { +static void unmount_autofs(Automount *a, bool keep_mount_point) { int r; assert(a); @@ -87,9 +88,7 @@ static void unmount_autofs(Automount *a) { a->pipe_event_source = sd_event_source_disable_unref(a->pipe_event_source); a->pipe_fd = safe_close(a->pipe_fd); - /* If we reload/reexecute things we keep the mount point around */ - if (!IN_SET(UNIT(a)->manager->objective, MANAGER_RELOAD, MANAGER_REEXECUTE)) { - + if (!keep_mount_point) { automount_release_requests(a, -EHOSTDOWN); if (a->where) { @@ -100,10 +99,17 @@ static void unmount_autofs(Automount *a) { } } +static bool automount_keep_mount_point(Automount *a) { + assert(a); + + /* If we reload/reexecute things we keep the mount point around, the reloaded unit picks it up again. */ + return IN_SET(UNIT(a)->manager->objective, MANAGER_RELOAD, MANAGER_REEXECUTE); +} + static void automount_done(Unit *u) { Automount *a = ASSERT_PTR(AUTOMOUNT(u)); - unmount_autofs(a); + unmount_autofs(a, automount_keep_mount_point(a)); a->where = mfree(a->where); a->extra_options = mfree(a->extra_options); @@ -273,7 +279,7 @@ static void automount_set_state(Automount *a, AutomountState state) { automount_stop_expire(a); if (!IN_SET(state, AUTOMOUNT_WAITING, AUTOMOUNT_RUNNING)) - unmount_autofs(a); + unmount_autofs(a, automount_keep_mount_point(a)); if (state != old_state) log_unit_debug(UNIT(a), "Changed %s -> %s", automount_state_to_string(old_state), automount_state_to_string(state)); @@ -281,6 +287,41 @@ static void automount_set_state(Automount *a, AutomountState state) { unit_notify(UNIT(a), state_translation_table[old_state], state_translation_table[state], /* reload_success= */ true); } +/* This automount unit's file vanished while we were reloading. Nobody can serve the autofs mount point + * anymore, but the kernel keeps blocking every process that is trying to access it while the mount exists. + * If the mount is already active, nothing waits for it: then only let go of the pipe. Otherwise detach the + * automount and fail the unit, which is what automount_enter_running() does with a request that arrives + * while the unit is not loaded. */ +static int automount_coldplug_vanished(Automount *a) { + Unit *u = UNIT(ASSERT_PTR(a)); + struct stat st; + bool exposed; + + if (lstat(a->where, &st) < 0) { + /* Not knowing what is mounted there, giving the autofs up is the safe side */ + log_unit_warning_errno(u, errno, "Failed to stat automount point '%s', assuming it is exposed: %m", a->where); + exposed = true; + } else + exposed = S_ISDIR(st.st_mode) && st.st_dev == a->dev_id; + + log_unit_warning(u, "Unit is not loaded anymore (%s), %s automount point '%s'.", + unit_load_state_to_string(u->load_state), + exposed ? "tearing down" : "abandoning", a->where); + + if (!exposed) + /* Mop up pending requests, nothing else will answer any more. The file system on top is + * what they were waiting for, so tell them it is there. */ + automount_release_requests(a, 0); + + unmount_autofs(a, /* keep_mount_point= */ !exposed); + + /* Nobody can act on a failed unit without a file, so do not keep it around as one, like the orphans + * manager_deserialize() creates for vanished unit files. */ + u->collect_mode = COLLECT_INACTIVE_OR_FAILED; + automount_enter_dead(a, AUTOMOUNT_FAILURE_RESOURCES); + return 0; +} + static int automount_coldplug(Unit *u) { Automount *a = ASSERT_PTR(AUTOMOUNT(u)); int r; @@ -302,6 +343,9 @@ static int automount_coldplug(Unit *u) { assert(a->pipe_fd >= 0); + if (u->load_state != UNIT_LOADED) + return automount_coldplug_vanished(a); + r = sd_event_add_io(u->manager->event, &a->pipe_event_source, a->pipe_fd, EPOLLIN, automount_dispatch_io, u); if (r < 0) return r; diff --git a/test/units/TEST-07-PID1.issue-43700.sh b/test/units/TEST-07-PID1.issue-43700.sh new file mode 100755 index 0000000000..63604cf40e --- /dev/null +++ b/test/units/TEST-07-PID1.issue-43700.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: LGPL-2.1-or-later +set -eux +set -o pipefail + +# An automount unit that vanishes across daemon-reload must not leave its autofs mount point armed, +# there's nothing left to answer it. +# See https://github.com/systemd/systemd/issues/43700 + +# PID 1 declares automount units unsupported without this device, e.g. in the nspawn containers CI runs in +if [[ ! -e /dev/autofs ]]; then + echo "autofs is not available, skipping automount tests" + exit 77 +fi + +# The errno text asserted below is strerror()'s +export LC_ALL=C + +MOUNT_POINT=/tmp/TEST-07-PID1-issue-43700 +UNIT=$(systemd-escape --path "$MOUNT_POINT") +STAT_ERR=/tmp/TEST-07-PID1-issue-43700.stderr +CURSOR=/tmp/TEST-07-PID1-issue-43700.cursor + +at_exit() { + set +e + [[ -n "${WAITER_PID:-}" ]] && kill "$WAITER_PID" + systemctl stop "$UNIT".automount "$UNIT".mount + # A file system may sit on top of an armed autofs; detach both without stepping into either + while [[ -n "$(fstype "$MOUNT_POINT")" ]]; do + umount -l "$MOUNT_POINT" || break + done + rm -f /run/systemd/system/"$UNIT".{auto,}mount "$STAT_ERR" "$CURSOR" + systemctl unmask "$UNIT".automount + systemctl daemon-reload + systemctl reset-failed "$UNIT".automount "$UNIT".mount + rmdir "$MOUNT_POINT" +} + +trap at_exit EXIT + +# The file system type of the topmost mount at a path, read from mountinfo so that the check itself does not +# trigger the automount: statfs() would, as its path lookup asks for automounting. +fstype() { + awk -v where="$1" '$5 == where { for (i = 7; $i != "-"; i++) {}; type = $(i + 1) } END { print type }' /proc/self/mountinfo +} + +write_automount() { + cat >/run/systemd/system/"$UNIT".automount </run/systemd/system/"$UNIT".mount </run/systemd/system/"$UNIT".mount <"$STAT_ERR" & + WAITER_PID=$! + timeout 10 bash -c "until systemctl list-jobs | grep -F '$UNIT.mount' >/dev/null; do sleep .5; done" + + # Both units vanish while the mount job is still waiting for its device + rm /run/systemd/system/"$UNIT".{auto,}mount + systemctl "$reload" + + # The waiter must fail promptly with the error the teardown sends, rather than hang until its timeout + # kills it (rc=137) or fail for some other reason + rc=0 + wait "$WAITER_PID" || rc=$? + unset WAITER_PID + [[ $rc -ne 0 && $rc -ne 137 ]] + grep -F 'Host is down' "$STAT_ERR" + journalctl --sync + journalctl --cursor-file="$CURSOR" --grep "tearing down automount point '$MOUNT_POINT'" + (! systemctl is-active "$UNIT".automount) + (! systemctl is-failed "$UNIT".automount) + + # The autofs mount point is gone as well + [[ -z "$(fstype "$MOUNT_POINT")" ]] +done + +# Masking counts as vanishing too: the unit stops being loaded, so nothing serves the mount point anymore +write_automount +write_hanging_mount +systemctl daemon-reload +systemctl start "$UNIT".automount +[[ "$(fstype "$MOUNT_POINT")" == autofs ]] +systemctl mask "$UNIT".automount +systemctl daemon-reload +journalctl --sync +journalctl --cursor-file="$CURSOR" --grep "tearing down automount point '$MOUNT_POINT'" +(! systemctl is-active "$UNIT".automount) +[[ -z "$(fstype "$MOUNT_POINT")" ]] +systemctl unmask "$UNIT".automount +rm /run/systemd/system/"$UNIT".{auto,}mount +systemctl daemon-reload From bdb302ffe614e4f50cca8e660309bb9f14505b8b Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Thu, 10 Sep 2026 10:06:32 +0000 Subject: [PATCH 4/4] core/automount: fail unread queued requests when giving up When an automount goes away, the tokens we answer are only those we have read from the pipe. Requests the kernel queued after our last read stayed blocked: closing the pipe does not wake them, the kernel only notices the closed pipe and turns the autofs catatonic when yet another request fails to reach us, and after the lazy unmount no request reaches it anymore. In the normal stop path that window is one event loop iteration; for an automount torn down at coldplug it spans the whole reload. Set the autofs to catatonic state before letting go of it, which fails everything still queued with ENOENT. Answer the known tokens first: closing our end of the pipe before that made a racing request flip the kernel into catatonic mode on its own, which discards the queue and turns our answers into "token not found" errors. Signed-off-by: Martin Pitt --- src/core/automount.c | 64 +++++++++++++++++++++----- test/units/TEST-07-PID1.issue-43700.sh | 62 ++++++++++++++++++++++++- 2 files changed, 113 insertions(+), 13 deletions(-) diff --git a/src/core/automount.c b/src/core/automount.c index 43b34c211d..ec42d7fb0d 100644 --- a/src/core/automount.c +++ b/src/core/automount.c @@ -49,6 +49,7 @@ static int automount_start_expire(Automount *a); static void automount_stop_expire(Automount *a); static int automount_send_ready(Automount *a, Set *tokens, int status); static void automount_enter_dead(Automount *a, AutomountResult f); +static int open_ioctl_fd(int dev_autofs_fd, const char *where, dev_t devid); static void automount_init(Unit *u) { Automount *a = ASSERT_PTR(AUTOMOUNT(u)); @@ -60,6 +61,43 @@ static void automount_init(Unit *u) { UNIT(a)->ignore_on_isolate = true; } +static int autofs_set_catatonic(int dev_autofs_fd, int ioctl_fd) { + struct autofs_dev_ioctl param; + + assert(dev_autofs_fd >= 0); + assert(ioctl_fd >= 0); + + init_autofs_dev_ioctl(¶m); + param.ioctlfd = ioctl_fd; + + return RET_NERRNO(ioctl(dev_autofs_fd, AUTOFS_DEV_IOCTL_CATATONIC, ¶m)); +} + +/* Requests the kernel queued since we last read the pipe are in no token set of ours. Turning the autofs + * catatonic fails them with ENOENT; merely closing the pipe would get the kernel there only once yet another + * request fails to reach us, which under a file system mounted on top never happens. Call this after answering + * the known tokens, so that the kernel does not discard them first. */ +static void automount_set_catatonic(Automount *a) { + _cleanup_close_ int ioctl_fd = -EBADF; + int r; + + assert(a); + + /* No way to talk to the autofs, e.g. when open_dev_autofs() failed at coldplug */ + if (UNIT(a)->manager->dev_autofs_fd < 0 || !a->where) + return; + + ioctl_fd = open_ioctl_fd(UNIT(a)->manager->dev_autofs_fd, a->where, a->dev_id); + if (ioctl_fd < 0) { + log_unit_warning_errno(UNIT(a), ioctl_fd, "Failed to open automount ioctl fd for '%s', ignoring: %m", a->where); + return; + } + + r = autofs_set_catatonic(UNIT(a)->manager->dev_autofs_fd, ioctl_fd); + if (r < 0) + log_unit_warning_errno(UNIT(a), r, "Failed to make automount point '%s' catatonic, ignoring: %m", a->where); +} + /* Nothing else can answer the requests the kernel already queued for us, and it keeps their processes * blocked until we do. Make a failure to do so at least visible. The status is what the mount requests get: * an expire request can only ever fail. */ @@ -75,6 +113,8 @@ static void automount_release_requests(Automount *a, int status) { r = automount_send_ready(a, a->expire_tokens, -EHOSTDOWN); if (r < 0) log_unit_warning_errno(UNIT(a), r, "Failed to release pending automount expire requests, ignoring: %m"); + + automount_set_catatonic(a); } static void unmount_autofs(Automount *a, bool keep_mount_point) { @@ -85,17 +125,16 @@ static void unmount_autofs(Automount *a, bool keep_mount_point) { if (a->pipe_fd < 0) return; + if (!keep_mount_point) + automount_release_requests(a, -EHOSTDOWN); + a->pipe_event_source = sd_event_source_disable_unref(a->pipe_event_source); a->pipe_fd = safe_close(a->pipe_fd); - if (!keep_mount_point) { - automount_release_requests(a, -EHOSTDOWN); - - if (a->where) { - r = repeat_unmount(a->where, MNT_DETACH|UMOUNT_NOFOLLOW); - if (r < 0) - log_unit_error_errno(UNIT(a), r, "Failed to unmount: %m"); - } + if (!keep_mount_point && a->where) { + r = repeat_unmount(a->where, MNT_DETACH|UMOUNT_NOFOLLOW); + if (r < 0) + log_unit_error_errno(UNIT(a), r, "Failed to unmount: %m"); } } @@ -289,9 +328,8 @@ static void automount_set_state(Automount *a, AutomountState state) { /* This automount unit's file vanished while we were reloading. Nobody can serve the autofs mount point * anymore, but the kernel keeps blocking every process that is trying to access it while the mount exists. - * If the mount is already active, nothing waits for it: then only let go of the pipe. Otherwise detach the - * automount and fail the unit, which is what automount_enter_running() does with a request that arrives - * while the unit is not loaded. */ + * Answer whatever is queued and detach the automount, which is what automount_enter_running() does with a + * request that arrives while the unit is not loaded. */ static int automount_coldplug_vanished(Automount *a) { Unit *u = UNIT(ASSERT_PTR(a)); struct stat st; @@ -310,7 +348,9 @@ static int automount_coldplug_vanished(Automount *a) { if (!exposed) /* Mop up pending requests, nothing else will answer any more. The file system on top is - * what they were waiting for, so tell them it is there. */ + * what they were waiting for, so tell them it is there. The autofs below it cannot be + * unmounted and stays behind catatonic: recovering that path takes a manual umount, and + * until then automount_start() refuses to use it again. */ automount_release_requests(a, 0); unmount_autofs(a, /* keep_mount_point= */ !exposed); diff --git a/test/units/TEST-07-PID1.issue-43700.sh b/test/units/TEST-07-PID1.issue-43700.sh index 63604cf40e..e620f536bc 100755 --- a/test/units/TEST-07-PID1.issue-43700.sh +++ b/test/units/TEST-07-PID1.issue-43700.sh @@ -20,6 +20,10 @@ MOUNT_POINT=/tmp/TEST-07-PID1-issue-43700 UNIT=$(systemd-escape --path "$MOUNT_POINT") STAT_ERR=/tmp/TEST-07-PID1-issue-43700.stderr CURSOR=/tmp/TEST-07-PID1-issue-43700.cursor +GENERATOR=/run/systemd/system-generators/TEST-07-PID1-issue-43700 +# Not below /tmp: PID 1 runs generators with a private one, so we would never see these +MARKER=/run/TEST-07-PID1-issue-43700.reloading +RELEASE=/run/TEST-07-PID1-issue-43700.release at_exit() { set +e @@ -29,7 +33,7 @@ at_exit() { while [[ -n "$(fstype "$MOUNT_POINT")" ]]; do umount -l "$MOUNT_POINT" || break done - rm -f /run/systemd/system/"$UNIT".{auto,}mount "$STAT_ERR" "$CURSOR" + rm -f /run/systemd/system/"$UNIT".{auto,}mount "$STAT_ERR" "$CURSOR" "$GENERATOR" "$MARKER" "$RELEASE" systemctl unmask "$UNIT".automount systemctl daemon-reload systemctl reset-failed "$UNIT".automount "$UNIT".mount @@ -158,3 +162,59 @@ journalctl --cursor-file="$CURSOR" --grep "tearing down automount point '$MOUNT_ systemctl unmask "$UNIT".automount rm /run/systemd/system/"$UNIT".{auto,}mount systemctl daemon-reload + +# A request arriving while PID 1 reloads sits unread in the pipe when the vanished unit is torn down, so +# its token is unknown to PID 1. It must still be failed rather than left blocked. Generators run +# synchronously, so one that waits for us holds the reload open for as long as it takes to get a request +# into that window. +write_automount +write_hanging_mount +systemctl daemon-reload +systemctl start "$UNIT".automount + +mkdir -p "$(dirname "$GENERATOR")" +cat >"$GENERATOR" <"$STAT_ERR" & +WAITER_PID=$! +# The request is parked in the kernel while the autofs is still armed, i.e. before PID 1 got to the +# teardown, so it was made inside the reload and PID 1 never read it. These waits are unhurried because +# the generator holds the reload until they are through. timeout(1) has to fork stat(1) first, and +# /proc/PID/wchan needs a kernel with CONFIG_KALLSYMS to name the frame. +timeout 10 bash -c "until pgrep -P $WAITER_PID >/dev/null; do sleep .1; done" +STAT_PID=$(pgrep -P "$WAITER_PID") +timeout 10 bash -c "until grep autofs /proc/$STAT_PID/wchan >/dev/null; do sleep .2; done" +[[ "$(fstype "$MOUNT_POINT")" == autofs ]] +touch "$RELEASE" +wait "$RELOAD_PID" +rm "$GENERATOR" "$MARKER" "$RELEASE" + +# The kernel fails unread requests with ENOENT once the autofs is catatonic +rc=0 +wait "$WAITER_PID" || rc=$? +unset WAITER_PID +[[ $rc -ne 0 && $rc -ne 137 ]] +grep -F 'No such file or directory' "$STAT_ERR" +journalctl --sync +journalctl --cursor-file="$CURSOR" --grep "tearing down automount point '$MOUNT_POINT'" +(! systemctl is-active "$UNIT".automount) +(! systemctl is-failed "$UNIT".automount) +[[ -z "$(fstype "$MOUNT_POINT")" ]]