510 lines
28 KiB
Bash
Executable File
510 lines
28 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# SPDX-License-Identifier: LGPL-2.1-or-later
|
||
set -eux
|
||
set -o pipefail
|
||
shopt -s nullglob
|
||
|
||
# Unset $PAGER so we don't have to use --no-pager everywhere
|
||
export PAGER=
|
||
|
||
REPORT=/usr/lib/systemd/systemd-report
|
||
|
||
"$REPORT" --help
|
||
"$REPORT" help
|
||
"$REPORT" --version
|
||
"$REPORT" --json=help
|
||
|
||
"$REPORT" metrics
|
||
"$REPORT" metrics -j
|
||
"$REPORT" metrics --no-legend
|
||
"$REPORT" describe
|
||
"$REPORT" describe -j
|
||
"$REPORT" describe --no-legend
|
||
"$REPORT" list-sources
|
||
"$REPORT" list-sources -j
|
||
"$REPORT" list-sources --no-legend
|
||
|
||
"$REPORT" metrics io
|
||
"$REPORT" metrics io.systemd piff
|
||
"$REPORT" metrics piff
|
||
"$REPORT" describe io
|
||
"$REPORT" describe io.systemd piff
|
||
"$REPORT" describe piff
|
||
|
||
# test io.systemd.CGroup Metrics
|
||
systemctl start systemd-report-cgroup.socket
|
||
varlinkctl info /run/systemd/report/io.systemd.CGroup
|
||
varlinkctl list-methods /run/systemd/report/io.systemd.CGroup
|
||
varlinkctl --more call /run/systemd/report/io.systemd.CGroup io.systemd.Metrics.List {}
|
||
varlinkctl --more call /run/systemd/report/io.systemd.CGroup io.systemd.Metrics.Describe {}
|
||
|
||
# CpuUsage emits one row per (cgroup, type) where type is total, user, or system.
|
||
# Confirm all three are present.
|
||
cgroup_metrics=$(varlinkctl --more --json=short call /run/systemd/report/io.systemd.CGroup io.systemd.Metrics.List {})
|
||
echo "$cgroup_metrics" | grep '"name":"io.systemd.CGroup.CpuUsage"' | grep '"type":"total"' >/dev/null
|
||
echo "$cgroup_metrics" | grep '"name":"io.systemd.CGroup.CpuUsage"' | grep '"type":"user"' >/dev/null
|
||
echo "$cgroup_metrics" | grep '"name":"io.systemd.CGroup.CpuUsage"' | grep '"type":"system"' >/dev/null
|
||
|
||
# test io.systemd.Network Metrics
|
||
varlinkctl info /run/systemd/report/io.systemd.Network
|
||
varlinkctl list-methods /run/systemd/report/io.systemd.Network
|
||
varlinkctl --more call /run/systemd/report/io.systemd.Network io.systemd.Metrics.List {}
|
||
varlinkctl --more call /run/systemd/report/io.systemd.Network io.systemd.Metrics.Describe {}
|
||
|
||
varlinkctl --more --json=short call /run/systemd/report/io.systemd.Network io.systemd.Metrics.Describe {} | grep '"name":"io.systemd.Network.Address"' >/dev/null
|
||
# we do not send "lo"
|
||
net_metrics="$(varlinkctl call --more --json=short /run/systemd/report/io.systemd.Network io.systemd.Metrics.List {})"
|
||
(! echo "$net_metrics" | grep '"name":"io.systemd.Network.Address"' | grep '"object":"lo"' >/dev/null)
|
||
# add a scratch address and check that it shows up.
|
||
ip address add 192.0.2.1/32 dev lo
|
||
timeout 30 bash -c 'until varlinkctl call --more --json=short /run/systemd/report/io.systemd.Network io.systemd.Metrics.List {} | grep -F "192.0.2.1/32" >/dev/null; do sleep .5; done'
|
||
net_metrics="$(varlinkctl call --more --json=short /run/systemd/report/io.systemd.Network io.systemd.Metrics.List {})"
|
||
echo "$net_metrics" | grep '"name":"io.systemd.Network.Address"' | grep '"object":"lo"' | grep '"value":"192.0.2.1/32"' | grep '"family":"ipv4"' >/dev/null
|
||
ip address del 192.0.2.1/32 dev lo
|
||
|
||
# test io.systemd.HWMon Metrics
|
||
systemctl start systemd-report-hwmon.socket
|
||
varlinkctl info /run/systemd/report/io.systemd.HWMon
|
||
varlinkctl list-methods /run/systemd/report/io.systemd.HWMon
|
||
hwmon_describe="$(varlinkctl call --more /run/systemd/report/io.systemd.HWMon io.systemd.Metrics.Describe {})"
|
||
echo "$hwmon_describe" | jq --seq -r 'select(.name == "io.systemd.HWMon.TemperatureCelsius") | .type' | grep -wx gauge >/dev/null
|
||
# An empty result set is signalled through the io.systemd.Metrics.NoSuchMetric sentinel error, which is what
|
||
# VMs without any hwmon devices produce, hence treat it as success.
|
||
hwmon_metrics="$(varlinkctl call --more --graceful=io.systemd.Metrics.NoSuchMetric /run/systemd/report/io.systemd.HWMon io.systemd.Metrics.List {})"
|
||
# At most one row per readable, non-faulted temp<N>_input attribute in sysfs. A sensor may become
|
||
# transiently unreadable between the two samples, hence only check the bound. VMs typically expose no hwmon
|
||
# devices at all, in which case both counts are zero.
|
||
hwmon_expected=0
|
||
for f in /sys/class/hwmon/hwmon*/temp*_input; do
|
||
hwmon_fault="${f%_input}_fault"
|
||
if [[ -f "$hwmon_fault" ]] && [[ "$(cat "$hwmon_fault")" == 1 ]]; then
|
||
continue
|
||
fi
|
||
if cat "$f" >/dev/null 2>&1; then
|
||
hwmon_expected=$((hwmon_expected + 1))
|
||
fi
|
||
done
|
||
hwmon_reported="$(echo "$hwmon_metrics" | jq --seq -r 'select(.name == "io.systemd.HWMon.TemperatureCelsius") | .fields.sensor' | wc -l)"
|
||
[[ "$hwmon_reported" -le "$hwmon_expected" ]]
|
||
if [[ "$hwmon_reported" -gt 0 ]]; then
|
||
# Every row must carry a numeric value, an object and a sensor field of the form temp<N>
|
||
[[ "$(echo "$hwmon_metrics" | jq --seq -r 'select(.name == "io.systemd.HWMon.TemperatureCelsius") | ((.value | type) == "number") and ((.object | type) == "string") and (.fields.sensor | test("^temp[0-9]+$")) | tostring' | sort -u)" == "true" ]]
|
||
fi
|
||
"$REPORT" describe io.systemd.HWMon
|
||
"$REPORT" metrics io.systemd.HWMon
|
||
|
||
# test io.systemd.DiskSpace Metrics
|
||
|
||
# Mount a scratch ext4 file system and check that it is reported with plausible numbers, that a tmpfs next to
|
||
# it is not, that additional bind mounts of it do not result in additional rows but the shortest writable
|
||
# mount path wins regardless of mount order, that it disappears from the report once remounted read-only,
|
||
# and that a device mapper device stacked on top of it is reported with the loop device as originating
|
||
# source.
|
||
diskspace_mount_tests() {
|
||
# Loop devices and device mapper are not available in containers
|
||
if systemd-detect-virt --quiet --container; then
|
||
echo "Skipping io.systemd.DiskSpace mount tests in container."
|
||
return
|
||
fi
|
||
|
||
DISKSPACE_WORK="$(mktemp -d /var/tmp/report-diskspace.XXXXXX)"
|
||
diskspace_cleanup() {
|
||
set +e
|
||
umount "$DISKSPACE_WORK/tmpfs" "$DISKSPACE_WORK/a" "$DISKSPACE_WORK/b" "$DISKSPACE_WORK/longer" "$DISKSPACE_WORK/ext4" "$DISKSPACE_WORK/dm"
|
||
dmsetup remove report-diskspace
|
||
systemd-dissect --detach "$DISKSPACE_WORK/scratch.raw"
|
||
rm -rf "$DISKSPACE_WORK"
|
||
set -e
|
||
}
|
||
trap diskspace_cleanup EXIT
|
||
mkdir "$DISKSPACE_WORK/ext4" "$DISKSPACE_WORK/a" "$DISKSPACE_WORK/b" "$DISKSPACE_WORK/longer" "$DISKSPACE_WORK/dm" "$DISKSPACE_WORK/tmpfs"
|
||
truncate -s 64M "$DISKSPACE_WORK/scratch.raw"
|
||
mkfs.ext4 -q "$DISKSPACE_WORK/scratch.raw"
|
||
DISKSPACE_LOOP="$(systemd-dissect --attach --loop-ref=report-diskspace "$DISKSPACE_WORK/scratch.raw")"
|
||
mount "$DISKSPACE_LOOP" "$DISKSPACE_WORK/ext4"
|
||
# "a" sorts before "b" and is just as short, but is read-only, hence "b" must be picked
|
||
mount --bind -o ro "$DISKSPACE_WORK/ext4" "$DISKSPACE_WORK/a"
|
||
mount --bind "$DISKSPACE_WORK/ext4" "$DISKSPACE_WORK/b"
|
||
# "longer" is writable too, but has a longer path, hence must not win even though it is mounted last
|
||
mount --bind "$DISKSPACE_WORK/ext4" "$DISKSPACE_WORK/longer"
|
||
mount -t tmpfs tmpfs "$DISKSPACE_WORK/tmpfs"
|
||
dd if=/dev/urandom of="$DISKSPACE_WORK/ext4/fill" bs=1M count=8
|
||
sync -f "$DISKSPACE_WORK/ext4"
|
||
|
||
diskspace_metrics="$(varlinkctl call --more /run/systemd/report/io.systemd.DiskSpace io.systemd.Metrics.List {} | tr -d '\036')"
|
||
# Exactly one row per metric for the scratch file system, reported under the shortest writable mount path
|
||
[[ "$(echo "$diskspace_metrics" | jq -r --arg s "$DISKSPACE_LOOP" 'select(.name == "io.systemd.DiskSpace.SizeBytes" and .fields.source == $s) | .object')" == "$DISKSPACE_WORK/b" ]]
|
||
[[ "$(echo "$diskspace_metrics" | jq -r --arg s "$DISKSPACE_LOOP" 'select(.name == "io.systemd.DiskSpace.FreeBytes" and .fields.source == $s) | .object')" == "$DISKSPACE_WORK/b" ]]
|
||
diskspace_size="$(echo "$diskspace_metrics" | jq -r --arg o "$DISKSPACE_WORK/b" 'select(.name == "io.systemd.DiskSpace.SizeBytes" and .object == $o) | .value')"
|
||
diskspace_free="$(echo "$diskspace_metrics" | jq -r --arg o "$DISKSPACE_WORK/b" 'select(.name == "io.systemd.DiskSpace.FreeBytes" and .object == $o) | .value')"
|
||
# The file system cannot be larger than the image, and at least the 8M we wrote must be missing from the
|
||
# free space.
|
||
[[ "$diskspace_size" -gt 0 ]]
|
||
[[ "$diskspace_size" -le $((64 * 1024 * 1024)) ]]
|
||
[[ "$diskspace_free" -gt 0 ]]
|
||
[[ $((diskspace_free + 8 * 1024 * 1024)) -le "$diskspace_size" ]]
|
||
# df(1) derives the same two numbers from statfs(), hence they must match exactly.
|
||
read -r diskspace_df_size diskspace_df_avail < <(df -B1 --output=size,avail "$DISKSPACE_WORK/ext4" | sed 1d)
|
||
[[ "$diskspace_size" == "$diskspace_df_size" ]]
|
||
[[ "$diskspace_free" == "$diskspace_df_avail" ]]
|
||
# Both rows must carry the file system type as field, and no originating source, as a loop device is not
|
||
# stacked on anything
|
||
[[ "$(echo "$diskspace_metrics" | jq -r --arg o "$DISKSPACE_WORK/b" 'select(.object == $o) | .fields.fileSystemType' | sort -u)" == "ext4" ]]
|
||
[[ "$(echo "$diskspace_metrics" | jq -r --arg o "$DISKSPACE_WORK/b" 'select(.object == $o) | .fields | has("originatingSource") | tostring' | sort -u)" == "false" ]]
|
||
# The tmpfs may not be reported
|
||
(! echo "$diskspace_metrics" | jq -r '.object' | grep -Fx "$DISKSPACE_WORK/tmpfs" >/dev/null)
|
||
|
||
# Once remounted read-only the file system must no longer be reported, regardless of the mount path
|
||
mount -o remount,ro "$DISKSPACE_WORK/ext4"
|
||
diskspace_metrics="$(varlinkctl call --more --graceful=io.systemd.Metrics.NoSuchMetric /run/systemd/report/io.systemd.DiskSpace io.systemd.Metrics.List {} | tr -d '\036')"
|
||
(! echo "$diskspace_metrics" | jq -r '.fields.source' | grep -Fx "$DISKSPACE_LOOP" >/dev/null)
|
||
|
||
# Now stack a (linear) device mapper device on top of the loop device, and mount the file system through
|
||
# that: the DM device must be reported as source, and the loop device it sits on as originating source. The
|
||
# loop device must be unmounted first, as DM needs exclusive access to it.
|
||
umount "$DISKSPACE_WORK/a" "$DISKSPACE_WORK/b" "$DISKSPACE_WORK/longer" "$DISKSPACE_WORK/ext4"
|
||
dmsetup create report-diskspace --table "0 $(blockdev --getsz "$DISKSPACE_LOOP") linear $DISKSPACE_LOOP 0"
|
||
udevadm wait --timeout=30 --settle /dev/mapper/report-diskspace
|
||
mount /dev/mapper/report-diskspace "$DISKSPACE_WORK/dm"
|
||
diskspace_metrics="$(varlinkctl call --more /run/systemd/report/io.systemd.DiskSpace io.systemd.Metrics.List {} | tr -d '\036')"
|
||
[[ "$(echo "$diskspace_metrics" | jq -r --arg o "$DISKSPACE_WORK/dm" 'select(.object == $o) | .fields.source' | sort -u)" == "/dev/mapper/report-diskspace" ]]
|
||
[[ "$(echo "$diskspace_metrics" | jq -r --arg o "$DISKSPACE_WORK/dm" 'select(.object == $o) | .fields.originatingSource' | sort -u)" == "$DISKSPACE_LOOP" ]]
|
||
|
||
diskspace_cleanup
|
||
trap - EXIT
|
||
}
|
||
|
||
# The socket and service are only built if libmount is available
|
||
if ! systemctl cat systemd-report-diskspace.socket &>/dev/null; then
|
||
echo "systemd-report-diskspace.socket is not installed, skipping io.systemd.DiskSpace tests."
|
||
else
|
||
systemctl start systemd-report-diskspace.socket
|
||
varlinkctl info /run/systemd/report/io.systemd.DiskSpace
|
||
varlinkctl list-methods /run/systemd/report/io.systemd.DiskSpace
|
||
# varlinkctl emits a JSON-SEQ stream for --more; strip the record separators so that plain jq can consume it
|
||
diskspace_describe="$(varlinkctl call --more /run/systemd/report/io.systemd.DiskSpace io.systemd.Metrics.Describe {} | tr -d '\036')"
|
||
echo "$diskspace_describe" | jq -r 'select(.name == "io.systemd.DiskSpace.FreeBytes") | .type' | grep -wx gauge >/dev/null
|
||
echo "$diskspace_describe" | jq -r 'select(.name == "io.systemd.DiskSpace.SizeBytes") | .type' | grep -wx gauge >/dev/null
|
||
|
||
diskspace_mount_tests
|
||
|
||
"$REPORT" describe io.systemd.DiskSpace
|
||
"$REPORT" metrics io.systemd.DiskSpace
|
||
fi
|
||
|
||
# test io.systemd.Basic Metrics
|
||
# ensure the socket is running, as some distros don't enable it by default
|
||
systemctl start systemd-report-basic.socket
|
||
varlinkctl info /run/systemd/report/io.systemd.Basic
|
||
varlinkctl list-methods /run/systemd/report/io.systemd.Basic
|
||
varlinkctl --more call /run/systemd/report/io.systemd.Basic io.systemd.Metrics.List {}
|
||
varlinkctl --more call /run/systemd/report/io.systemd.Basic io.systemd.Metrics.Describe {}
|
||
|
||
id1="$(varlinkctl call --more /run/systemd/report/io.systemd.Basic io.systemd.Metrics.List {} | jq --seq -r 'select(.name == "io.systemd.Basic.OSRelease.ID") | .value')"
|
||
id2="$(. /etc/os-release; echo "$ID")"
|
||
[ "$id1" = "$id2" ]
|
||
|
||
# test io.systemd.Basic load average and swap metrics
|
||
basic_metrics="$(varlinkctl call --more /run/systemd/report/io.systemd.Basic io.systemd.Metrics.List {})"
|
||
# NB: '| tostring' turns the numeric value into a raw string, so 'jq -r' prints it cleanly
|
||
# (numbers, unlike strings, are otherwise still emitted with the json-seq record separator).
|
||
basic_value() { echo "$basic_metrics" | jq --seq -r "select(.name == \"io.systemd.Basic.$1\") | .value | tostring"; }
|
||
# Like basic_value, but the value must be a JSON number (jq's 'numbers' filter emits the value
|
||
# only if it is one, so a non-empty result confirms both presence and type), and an optional
|
||
# second argument gives a jq condition further restricting the match, e.g. on .object or .fields.
|
||
basic_number() { echo "$basic_metrics" | jq --seq -r "select(.name == \"io.systemd.Basic.$1\" and ${2:-true}) | .value | numbers | tostring"; }
|
||
|
||
# The three classic load average fields must be present and numeric.
|
||
for field in LoadAverage1Min LoadAverage5Min LoadAverage15Min; do
|
||
test -n "$(basic_number "$field")"
|
||
done
|
||
|
||
# SwapBytes must match the total the kernel reports in /proc/meminfo (which is in kB).
|
||
swap_reported="$(basic_value SwapBytes)"
|
||
swap_expected=$(( $(awk '/^SwapTotal:/ { print $2; found=1 } END { if (!found) print 0 }' /proc/meminfo) * 1024 ))
|
||
[ "$swap_reported" = "$swap_expected" ]
|
||
|
||
# SwapUsedBytes must be numeric and cannot exceed the configured total.
|
||
swap_used="$(basic_number SwapUsedBytes)"
|
||
test -n "$swap_used"
|
||
[ "$swap_used" -le "$swap_reported" ]
|
||
|
||
# Those fields must be reported with a numeric nanoseconds value.
|
||
for type in user system; do
|
||
test -n "$(basic_number CPUUsage ".fields.type == \"$type\"")"
|
||
done
|
||
|
||
# The values are USER_HZ ticks converted to nanoseconds. The metrics were sampled before the
|
||
# current /proc/stat state, and the idle counter mostly grows, so the reported idle time
|
||
# should be positive and no larger than what /proc/stat shows now with some margin.
|
||
cpu_idle_reported="$(basic_number CPUWait '.fields.type == "idle"')"
|
||
awk -v hz="$(getconf CLK_TCK)" -v sampled="$cpu_idle_reported" \
|
||
'/^cpu / { exit !(sampled > 0 && sampled <= ($5 + 1) / hz * 1e9 ) }' /proc/stat
|
||
|
||
# iowait/idle/steal must be valid nanoseconds values, but can go up and down.
|
||
for type in idle iowait steal; do
|
||
test -n "$(basic_number CPUWait ".fields.type == \"$type\"")"
|
||
done
|
||
|
||
# MemoryUsedBytes is MemTotal − MemAvailable, so it cannot exceed the MemTotal reported by the
|
||
# same file. (PhysicalMemoryBytes is not a valid bound here: it is capped by the root cgroup's
|
||
# memory limit, so it can legitimately be smaller than the value computed from /proc/meminfo.)
|
||
mem_used="$(basic_number MemoryUsedBytes)"
|
||
mem_total=$(( $(awk '/^MemTotal:/ { print $2 }' /proc/meminfo) * 1024 ))
|
||
test -n "$mem_used"
|
||
[ "$mem_used" -le "$mem_total" ]
|
||
|
||
# Pressure metrics are only reported if the kernel supports PSI.
|
||
if cat /proc/pressure/cpu >/dev/null 2>&1; then
|
||
for family in PressureAvg10 PressureStallSeconds; do
|
||
for resource in cpu memory io; do
|
||
test -n "$(basic_number "$family" ".fields.resource == \"$resource\" and .fields.type == \"some\"")"
|
||
# "full" is deliberately not emitted for cpu, where the kernel always reports 0.
|
||
full="$(basic_number "$family" ".fields.resource == \"$resource\" and .fields.type == \"full\"")"
|
||
if [ "$resource" = cpu ]; then
|
||
test -z "$full"
|
||
else
|
||
test -n "$full"
|
||
fi
|
||
done
|
||
done
|
||
|
||
# The avg10 values are percentages, so they must lie in the 0…100 range. This would catch a
|
||
# wrong fixed-point conversion, which would inflate the value 2048x.
|
||
echo "$basic_metrics" | jq --seq -r 'select(.name == "io.systemd.Basic.PressureAvg10") | .value | tostring' |
|
||
awk '{ if ($1 < 0 || $1 > 100) exit 1 }'
|
||
fi
|
||
|
||
# Every whole physical block device (not partitions, not virtual or stacked devices) must be
|
||
# reported, with byte values derived from the 512-byte sector counters in sysfs. Those counters
|
||
# only grow, so the values sampled above cannot exceed what sysfs shows now.
|
||
for dev in /sys/block/*; do
|
||
devpath="$(udevadm info --query=property --value --property=DEVPATH "$dev" || :)"
|
||
devname="$(udevadm info --query=property --value --property=DEVNAME "$dev" || :)"
|
||
[[ -n "$devpath" && -n "$devname" ]] || continue
|
||
|
||
[[ "$devpath" =~ "/devices/virtual/" ]] && continue
|
||
|
||
read_bytes="$(basic_number DiskReadBytes ".object == \"${devname}\"")"
|
||
write_bytes="$(basic_number DiskWriteBytes ".object == \"${devname}\"")"
|
||
test -n "$read_bytes"
|
||
test -n "$write_bytes"
|
||
|
||
read -r _ _ read_sectors _ _ _ write_sectors _ <"$dev/stat"
|
||
[ "$read_bytes" -le "$(( read_sectors * 512 ))" ]
|
||
[ "$write_bytes" -le "$(( write_sectors * 512 ))" ]
|
||
done
|
||
|
||
# Every new metric family must be described, even those whose values depend on the environment.
|
||
basic_describe="$(varlinkctl call --more /run/systemd/report/io.systemd.Basic io.systemd.Metrics.Describe {})"
|
||
for name in CPUUsage CPUWait DiskReadBytes DiskWriteBytes MemoryUsedBytes PressureAvg10 PressureStallSeconds SwapUsedBytes; do
|
||
echo "$basic_describe" | grep -F "io.systemd.Basic.$name" >/dev/null
|
||
done
|
||
|
||
# io.systemd.Manager.Version should be non-empty and match what `systemctl --version` reports
|
||
metrics_version="$(varlinkctl call --more /run/systemd/report/io.systemd.Manager io.systemd.Metrics.List {} | jq --seq -r 'select(.name == "io.systemd.Manager.Version") | .value')"
|
||
[ -n "$metrics_version" ]
|
||
systemctl --version | grep -F "($metrics_version)" >/dev/null
|
||
|
||
# Boot timeline timestamps. The kernel CLOCK_MONOTONIC is 0 by definition, so only its realtime is
|
||
# reported; userspace reports both realtime and monotonic. We don't check FinishTimestamp here:
|
||
# the test runs as a service, so manager_check_finished() typically hasn't fired yet and the
|
||
# timestamp is unset (the metric is then suppressed). Likewise KernelTimestamp is cleared when
|
||
# systemd runs inside a container (see main.c), so the metric is suppressed there too.
|
||
manager_metrics="$(varlinkctl call --more /run/systemd/report/io.systemd.Manager io.systemd.Metrics.List {})"
|
||
metric_value() { echo "$manager_metrics" | jq --seq -r "select(.name == \"io.systemd.Manager.$1\") | .value | tostring"; }
|
||
if ! systemd-detect-virt --quiet --container; then
|
||
[ "$(metric_value KernelTimestamp.Realtime)" -gt 0 ]
|
||
fi
|
||
[ "$(metric_value UserspaceTimestamp.Realtime)" -gt 0 ]
|
||
[ "$(metric_value UserspaceTimestamp.Monotonic)" -gt 0 ]
|
||
|
||
# These startup phases are captured unconditionally by manager_startup() on every boot (in containers
|
||
# too, where they are not redirected to the InitRD* variants), so both clocks are always set on the
|
||
# system manager by the time this service runs. We don't check UnitsLoadTimestamp (the last load, only
|
||
# set on the reload/reexec triggered by switch-root) nor the Security/Firmware/Loader/InitRD timestamps,
|
||
# which depend on the boot environment and may legitimately be unset (the metric is then suppressed).
|
||
for phase in GeneratorsStartTimestamp GeneratorsFinishTimestamp \
|
||
UnitsLoadStartTimestamp UnitsLoadFinishTimestamp; do
|
||
[ "$(metric_value "$phase.Realtime")" -gt 0 ]
|
||
[ "$(metric_value "$phase.Monotonic")" -gt 0 ]
|
||
done
|
||
|
||
# test io.systemd.Basic.MachineInfo.* metrics, sourced from /etc/machine-info
|
||
if [ -e /etc/machine-info ]; then
|
||
MACHINE_INFO_BACKUP="$(mktemp)"
|
||
cp /etc/machine-info "$MACHINE_INFO_BACKUP"
|
||
MACHINE_INFO_EXISTED=1
|
||
else
|
||
MACHINE_INFO_EXISTED=0
|
||
fi
|
||
|
||
restore_machine_info() {
|
||
set +e
|
||
if [ "$MACHINE_INFO_EXISTED" = 1 ]; then
|
||
cp "$MACHINE_INFO_BACKUP" /etc/machine-info
|
||
rm -f "$MACHINE_INFO_BACKUP"
|
||
else
|
||
rm -f /etc/machine-info
|
||
fi
|
||
set -e
|
||
}
|
||
trap restore_machine_info EXIT
|
||
|
||
cat >/etc/machine-info <<EOF
|
||
PRETTY_HOSTNAME="Test Machine"
|
||
DEPLOYMENT=test
|
||
LOCATION="Test Lab, Rack 3"
|
||
TAGS=foo:bar:baz
|
||
EOF
|
||
|
||
# The metric source re-reads /etc/machine-info on every call, so no service restart is needed.
|
||
machine_info_metrics="$(varlinkctl call --more /run/systemd/report/io.systemd.Basic io.systemd.Metrics.List {})"
|
||
machine_info_value() { echo "$machine_info_metrics" | jq --seq -r "select(.name == \"$1\") | .value"; }
|
||
|
||
[ "$(machine_info_value io.systemd.Basic.MachineInfo.PRETTY_HOSTNAME)" = "Test Machine" ]
|
||
[ "$(machine_info_value io.systemd.Basic.MachineInfo.DEPLOYMENT)" = "test" ]
|
||
[ "$(machine_info_value io.systemd.Basic.MachineInfo.LOCATION)" = "Test Lab, Rack 3" ]
|
||
[ "$(machine_info_value io.systemd.Basic.MachineInfo.TAGS)" = "foo:bar:baz" ]
|
||
|
||
restore_machine_info
|
||
trap - EXIT
|
||
|
||
# Test HTTP upload (plain http)
|
||
FAKE_SERVER=/usr/lib/systemd/tests/integration-tests/TEST-74-AUX-UTILS/TEST-74-AUX-UTILS.units/fake-report-server.py
|
||
CERTDIR=$(mktemp -d)
|
||
|
||
at_exit() {
|
||
set +e
|
||
systemctl stop fake-report-server fake-report-server-tls
|
||
systemctl stop systemd-report.socket systemd-report-files.socket systemd-report-sign-plain.socket
|
||
rm -f /run/systemd/report.files/testreportfile /run/systemd/report.files/binaryfile
|
||
rm -rf "$CERTDIR" "${SIGN_WORK:-}"
|
||
}
|
||
trap at_exit EXIT
|
||
|
||
systemd-run -p Type=notify --unit=fake-report-server "$FAKE_SERVER"
|
||
systemctl status fake-report-server
|
||
|
||
"$REPORT" generate io.systemd.Manager.UnitsTotal
|
||
|
||
"$REPORT" generate io.systemd.Manager.UnitsTotal | jq .
|
||
|
||
"$REPORT" upload --url=http://localhost:8089/
|
||
|
||
# Test HTTPS upload with generated TLS certificates
|
||
openssl req -x509 -newkey rsa:2048 -keyout "$CERTDIR/server.key" -out "$CERTDIR/server.crt" \
|
||
-days 1 -nodes -subj "/CN=localhost" 2>/dev/null
|
||
|
||
systemd-run -p Type=notify --unit=fake-report-server-tls \
|
||
"$FAKE_SERVER" --cert="$CERTDIR/server.crt" --key="$CERTDIR/server.key" --port=8090
|
||
systemctl status fake-report-server-tls
|
||
|
||
"$REPORT" upload --url=https://localhost:8090/ --key=- --trust="$CERTDIR/server.crt" \
|
||
--extra-header='Authorization: Bearer magic string'
|
||
|
||
# -----------------------------------------------------------------------------
|
||
# Test the systemd-report-files@.service metric provider: files dropped into one
|
||
# of the report.files directories are exposed verbatim as io.systemd.Files.*
|
||
# metrics, keyed by their file name.
|
||
mkdir -p /run/systemd/report.files
|
||
REPORT_FILE_CONTENT="hello from the report files metric provider"
|
||
printf '%s' "$REPORT_FILE_CONTENT" >/run/systemd/report.files/testreportfile
|
||
|
||
systemctl start systemd-report-files.socket
|
||
varlinkctl info /run/systemd/report/io.systemd.Files
|
||
varlinkctl list-methods /run/systemd/report/io.systemd.Files
|
||
|
||
# The metric value must be the exact file contents.
|
||
files_metrics="$(varlinkctl call --more /run/systemd/report/io.systemd.Files io.systemd.Metrics.List {})"
|
||
files_value() { echo "$files_metrics" | jq --seq -r "select(.name == \"$1\") | .value"; }
|
||
[ "$(files_value io.systemd.Files.testreportfile)" = "$REPORT_FILE_CONTENT" ]
|
||
|
||
# Describe must list the very same metric, of type string.
|
||
files_describe="$(varlinkctl call --more /run/systemd/report/io.systemd.Files io.systemd.Metrics.Describe {})"
|
||
echo "$files_describe" | jq --seq -r 'select(.name == "io.systemd.Files.testreportfile") | .type' | grep -wx string >/dev/null
|
||
|
||
# Files that aren't valid UTF-8 text must be skipped silently (the value is a
|
||
# JSON string, so only text files can be reported).
|
||
printf '\xff\xfe binary garbage' >/run/systemd/report.files/binaryfile
|
||
files_metrics="$(varlinkctl call --more /run/systemd/report/io.systemd.Files io.systemd.Metrics.List {})"
|
||
[ -z "$(files_value io.systemd.Files.binaryfile)" ]
|
||
rm -f /run/systemd/report.files/binaryfile
|
||
|
||
# -----------------------------------------------------------------------------
|
||
# Test report signing through the plain software backend, driven entirely via
|
||
# the io.systemd.Report Varlink interface (the GenerateSigned method). The plain
|
||
# backend is only built/installed with OpenSSL support, so skip if unavailable.
|
||
if ! systemctl cat systemd-report-sign-plain.socket &>/dev/null; then
|
||
echo "systemd-report-sign-plain.socket is not installed, skipping report signing test."
|
||
exit 0
|
||
fi
|
||
|
||
systemctl start systemd-report.socket
|
||
systemctl start systemd-report-sign-plain.socket
|
||
|
||
SIGN_WORK="$(mktemp -d)"
|
||
|
||
# Ask systemd-report to generate a *signed* report over Varlink. The reply
|
||
# carries the signed report as base64-encoded JSON-SEQ data.
|
||
varlinkctl call /run/systemd/io.systemd.Report io.systemd.Report.GenerateSigned \
|
||
'{"matches":["io.systemd.Manager.UnitsTotal"]}' | jq -r .reportData | base64 -d >"$SIGN_WORK/report.seq"
|
||
|
||
# The first JSON-SEQ record is the report itself. This is exactly the byte
|
||
# sequence that got signed, including the leading record separator (0x1e) and
|
||
# the trailing newline, so 'head -n1' reproduces it verbatim.
|
||
head -n1 "$SIGN_WORK/report.seq" >"$SIGN_WORK/message.bin"
|
||
tr -d '\036' <"$SIGN_WORK/message.bin" | jq -e '.mediaType == "application/vnd.io.systemd.report"' >/dev/null
|
||
|
||
# The remaining record(s) are signature objects. With only the plain backend
|
||
# enabled there is exactly one, with mechanism "plain".
|
||
sig_json="$(tail -n +2 "$SIGN_WORK/report.seq" | tr -d '\036')"
|
||
[ "$(echo "$sig_json" | jq -r .mechanism)" = "plain" ]
|
||
[ "$(echo "$sig_json" | jq -r .mediaType)" = "application/vnd.io.systemd.report.signature" ]
|
||
|
||
# The sha256 recorded in the signature must match the digest of the report bytes.
|
||
[ "$(echo "$sig_json" | jq -r .sha256)" = "$(sha256sum "$SIGN_WORK/message.bin" | cut -d' ' -f1)" ]
|
||
|
||
# Extract the embedded public key and the raw Ed25519 signature.
|
||
echo "$sig_json" | jq -r .data.key >"$SIGN_WORK/pub.pem"
|
||
echo "$sig_json" | jq -r .data.signature | base64 -d >"$SIGN_WORK/sig.bin"
|
||
|
||
# The signature is made over the SHA256 digest bytes directly (Ed25519, the
|
||
# signer does not pre-hash), so verify the raw digest against the signature.
|
||
openssl dgst -sha256 -binary "$SIGN_WORK/message.bin" >"$SIGN_WORK/digest.bin"
|
||
openssl pkeyutl -verify -pubin -inkey "$SIGN_WORK/pub.pem" -rawin \
|
||
-in "$SIGN_WORK/digest.bin" -sigfile "$SIGN_WORK/sig.bin"
|
||
|
||
# The public key embedded in the report must be the very key the plain backend
|
||
# generated and persisted on disk.
|
||
diff <(openssl pkey -pubin -in "$SIGN_WORK/pub.pem" -pubout) \
|
||
<(openssl pkey -pubin -in /var/lib/systemd/report.sign.plain/local.public -pubout)
|
||
|
||
# A tampered report must no longer verify against the signature.
|
||
printf 'x' >>"$SIGN_WORK/message.bin"
|
||
openssl dgst -sha256 -binary "$SIGN_WORK/message.bin" >"$SIGN_WORK/digest-bad.bin"
|
||
if openssl pkeyutl -verify -pubin -inkey "$SIGN_WORK/pub.pem" -rawin \
|
||
-in "$SIGN_WORK/digest-bad.bin" -sigfile "$SIGN_WORK/sig.bin"; then
|
||
echo "Signature verification unexpectedly succeeded for tampered report" >&2
|
||
exit 1
|
||
fi
|
||
|
||
# -----------------------------------------------------------------------------
|
||
# The plain backend also acts as a metric provider, exposing its public signing
|
||
# key as the io.systemd.Report.SignPlain.PublicKey metric. This is served off the
|
||
# very same socket as the signer, reachable in the metrics directory under
|
||
# /run/systemd/report/ (the signer endpoint is just a symlink into the root-only
|
||
# /run/systemd/report.sign/ directory pointing at the same socket).
|
||
varlinkctl info /run/systemd/report/io.systemd.Report.SignPlain
|
||
varlinkctl list-methods /run/systemd/report/io.systemd.Report.SignPlain
|
||
|
||
# The exposed public key must be the very key persisted on disk.
|
||
sign_metrics="$(varlinkctl call --more /run/systemd/report/io.systemd.Report.SignPlain io.systemd.Metrics.List {})"
|
||
echo "$sign_metrics" | jq --seq -r 'select(.name == "io.systemd.Report.SignPlain.PublicKey") | .value' >"$SIGN_WORK/metric-pub.pem"
|
||
diff <(openssl pkey -pubin -in "$SIGN_WORK/metric-pub.pem" -pubout) \
|
||
<(openssl pkey -pubin -in /var/lib/systemd/report.sign.plain/local.public -pubout)
|
||
|
||
# Describe must list the very same metric, of type string.
|
||
varlinkctl call --more /run/systemd/report/io.systemd.Report.SignPlain io.systemd.Metrics.Describe {} |
|
||
jq --seq -r 'select(.name == "io.systemd.Report.SignPlain.PublicKey") | .type' | grep -wx string >/dev/null
|