utf8: make utf8_char_console_width() reject what utf8_is_valid() rejects

utf8_char_console_width() decoded with utf8_encoded_to_unichar(), which does not
check for overlong encodings, does not check unichar_is_valid(), and still
accepts the obsolete five and six byte forms. Those checks live in the wrapper
utf8_encoded_valid_unichar(), which is what utf8_is_valid() and ellipsize_mem()
use. Over all 4244897280 NUL free byte strings of length one to four, that is the
whole UTF-8 encoding space, the wrapper agrees with utf8_is_valid_n() everywhere,
while the bare decoder disagrees on 8835838 of them, always by accepting more.

So utf8_console_width() assigned a finite width to strings that utf8_is_valid()
refuses. format-table.c measures a cell with utf8_console_width() and, if it does
not fit, hands it to ellipsize(), which validates strictly, returns NULL, and is
reported as -ENOMEM. A surrogate, overlong, noncharacter or five byte sequence in
any cell therefore failed the whole table with a bogus out of memory error, and
only once the column happened to be narrow enough to require truncation, so
whether it happened at all depended on the width of the user's terminal. At other
widths the invalid sequence was printed to the terminal instead.

Decode with utf8_encoded_valid_unichar() so that how wide we think a string is
agrees with whether we consider it valid UTF-8 in the first place. Invalid cells
are now refused consistently with -EINVAL, the way the sequences that
utf8_encoded_to_unichar() already rejected always were.

Note that for tables this makes the refusal total rather than per field:
table_data_requested_width_height() turns a SIZE_MAX width into -EINVAL, which
table_format() propagates, so a cell in one of the newly rejected encodings now
fails the whole table at every width. systemd-dissect and repart --dry-run put
GPT partition names read straight off the image into cells without filtering
them, so a crafted partition name suppresses the whole listing where before it
cost one field, and only at some terminal widths. That is the intended trade:
the alternative is to keep emitting raw invalid bytes to the terminal at
whichever widths happen not to need truncation. Should degrading the cell rather
than the table turn out to be preferable, the place for it is where the text
enters the table, via utf8_escape_invalid() and friends, not in a helper whose
job is to say how wide a string is.

This also makes the assert(w >= 0) in utf8_escape_non_printable_full() hold by
construction rather than by the accident that its own validation was stricter
than the width helper's.

(cherry picked from commit cd51add717)

src/basic/utf8.c

@@ -181,10 +181,16 @@ int utf8_char_console_width(const char *str) {
 
         assert(str);
 
-        r = utf8_encoded_to_unichar(str, &c);
+        /* Validate the same way utf8_is_valid() does. utf8_encoded_to_unichar() on its own accepts
+         * overlong encodings, surrogates, noncharacters and the obsolete 5 and 6 byte forms, which would
+         * make our idea of how wide a string is disagree with our idea of whether it is valid UTF-8 in
+         * the first place. */
+        r = utf8_encoded_valid_unichar(str, SIZE_MAX);
         if (r < 0)
                 return r;
 
+        assert_se(utf8_encoded_to_unichar(str, &c) == r);
+
         return unichar_console_width(c);
 }
 

src/test/test-format-table.c

@@ -8,6 +8,8 @@
 #include "env-util.h"
 #include "format-table.h"
 #include "json-util.h"
+#include "string-util.h"
+#include "strv.h"
 #include "terminal-util.h"
 #include "tests.h"
 #include "time-util.h"
@@ -35,6 +37,36 @@ TEST(issue_9549) {
                      "foooo raw  no 673.6M Wed 2018-07-11 00:10:33 J… Wed 2018-07-11 00:16:00 JST\n");
 }
 
+TEST(invalid_utf8_cell) {
+        /* A cell that is not valid UTF-8 must be refused the same way whatever the column width is.
+         * The width was previously determined with a more permissive decoder than the one ellipsize()
+         * uses, so a surrogate, overlong, noncharacter or obsolete five byte sequence was measured as
+         * if it were fine, and then failed inside ellipsize() as a bogus -ENOMEM, but only once the
+         * column happened to be narrow enough to require truncation. The widths below deliberately go
+         * past the width of the whole table, so that the last iterations are ones where no truncation
+         * is needed at all: those are the ones where the old code did not fail but printed the invalid
+         * sequence to the terminal instead. */
+
+        FOREACH_STRING(x,
+                       "\xed\xa0\x80",          /* UTF-16 surrogate */
+                       "\xc0\xaf",              /* overlong '/' */
+                       "\xef\xb7\x90",          /* noncharacter U+FDD0 */
+                       "\xf8\x88\x80\x80\x80")  /* obsolete five byte form */
+                for (size_t w = 8; w <= 64; w += 8) {
+                        _cleanup_(table_unrefp) Table *table = NULL;
+                        _cleanup_free_ char *formatted = NULL, *cell = NULL;
+
+                        ASSERT_NOT_NULL((table = table_new("name", "value")));
+                        ASSERT_NOT_NULL((cell = strjoin("aaaaaaaaaaaaaaaa", x, "aaaaaaaaaaaaaaaa")));
+                        ASSERT_OK(table_add_many(table,
+                                                 TABLE_STRING, cell,
+                                                 TABLE_STRING, "second"));
+                        table_set_width(table, w);
+
+                        ASSERT_ERROR(table_format(table, &formatted), EINVAL);
+                }
+}
+
 TEST(multiline) {
         _cleanup_(table_unrefp) Table *table = NULL;
         _cleanup_free_ char *formatted = NULL;

src/test/test-utf8.c

@@ -207,13 +207,36 @@ TEST(utf8_n_codepoints) {
         assert_se(utf8_n_codepoints("\xF1") == SIZE_MAX);
 }
 
+TEST(utf8_char_console_width) {
+        ASSERT_EQ(utf8_char_console_width("a"), 1);
+        ASSERT_EQ(utf8_char_console_width("串"), 2);
+        ASSERT_EQ(utf8_char_console_width("\t"), 8);
+
+        /* How wide we think a character is must agree with whether we consider it valid UTF-8 in the
+         * first place. utf8_encoded_to_unichar(), which this used to be based on, accepts everything
+         * below except the bare continuation byte, while utf8_is_valid() rejects all of them. */
+        FOREACH_STRING(x,
+                       "\x80",                       /* bare continuation byte */
+                       "\xed\xa0\x80",               /* UTF-16 surrogate */
+                       "\xc0\xaf",                   /* overlong '/' */
+                       "\xef\xb7\x90",               /* noncharacter U+FDD0 */
+                       "\xef\xbf\xbe",               /* noncharacter U+FFFE */
+                       "\xf8\x88\x80\x80\x80",       /* obsolete five byte form */
+                       "\xfc\x84\x80\x80\x80\x80") { /* obsolete six byte form */
+                ASSERT_FALSE(utf8_is_valid(x));
+                ASSERT_ERROR(utf8_char_console_width(x), EINVAL);
+                ASSERT_EQ(utf8_console_width(x), SIZE_MAX);
+        }
+}
+
 TEST(utf8_console_width) {
-        assert_se(utf8_console_width("abc") == 3);
-        assert_se(utf8_console_width("zażółcić gęślą jaźń") == 19);
-        assert_se(utf8_console_width("串") == 2);
-        assert_se(utf8_console_width("") == 0);
-        assert_se(utf8_console_width("…👊🔪💐…") == 8);
-        assert_se(utf8_console_width("\xF1") == SIZE_MAX);
+        ASSERT_EQ(utf8_console_width("abc"), (size_t) 3);
+        ASSERT_EQ(utf8_console_width("zażółcić gęślą jaźń"), (size_t) 19);
+        ASSERT_EQ(utf8_console_width("串"), (size_t) 2);
+        ASSERT_EQ(utf8_console_width(""), (size_t) 0);
+        ASSERT_EQ(utf8_console_width("…👊🔪💐…"), (size_t) 8);
+        ASSERT_EQ(utf8_console_width("\xF1"), SIZE_MAX);
+        ASSERT_EQ(utf8_console_width("\t"), (size_t) 8);
 }
 
 TEST(utf8_to_utf16) {