Changeset b31323f in mainline for common/str.c

Timestamp:

2025-04-17T14:29:23Z (5 days ago)

Author:

Jiří Zárevúcky <zarevucky.jiri@…>

Branches:

master

Children:

ae787807

Parents:

65bf084

git-author:

Jiří Zárevúcky <zarevucky.jiri@…> (2025-04-17 11:01:00)

git-committer:

Jiří Zárevúcky <zarevucky.jiri@…> (2025-04-17 14:29:23)

Message:

Test, fix and extend string sanitization

File:

• common/str.c (modified) (12 diffs)

common/str.c

@@ -234 +234 @@
 }
 
+static bool _is_surrogate(const mbstate_t *mb, uint8_t b)
+{
+        return (mb->state == 0b1111110000001101 && b >= 0xa0);
+}
+
 #define _likely(expr) __builtin_expect((expr), true)
 #define _unlikely(expr) __builtin_expect((expr), false)
@@ -299 +304 @@
                                         return CHAR_INVALID;
 
+                                /* Reject surrogates */
+                                if (_unlikely(ch >= 0xD800 && ch < 0xE000))
+                                        return CHAR_INVALID;
+
                                 return ch;
                         }
@@ -323 +332 @@
                                         return CHAR_INVALID;
 
+                                /* Reject out-of-range characters. */
+                                if (_unlikely(ch >= 0x110000))
+                                        return CHAR_INVALID;
+
                                 return ch;
                         }
@@ -339 +352 @@
                 uint8_t b = s[*offset];
 
-                if (!_is_continuation(b) || _is_non_shortest(mb, b)) {
+                if (!_is_continuation(b) || _is_non_shortest(mb, b) || _is_surrogate(mb, b)) {
                         mb->state = 0;
                         return CHAR_INVALID;
@@ -523 +536 @@
 }
 
-/* Convert in place any bytes that don't form a valid character into U_SPECIAL. */
-static void _sanitize_string(char *str, size_t n)
+/* Convert in place any bytes that don't form a valid character into replacement. */
+static size_t _str_sanitize(char *str, size_t n, uint8_t replacement)
 {
         uint8_t *b = (uint8_t *) str;
-
-        for (; *b && n > 0; b++, n--) {
+        size_t count = 0;
+
+        for (; n > 0 && b[0]; b++, n--) {
                 int cont = _continuation_bytes(b[0]);
                 if (__builtin_expect(cont, 0) == 0)
@@ -534 +548 @@
 
                 if (cont < 0 || n <= (size_t) cont) {
-                        b[0] = U_SPECIAL;
+                        b[0] = replacement;
+                        count++;
                         continue;
                 }
 
                 /* Check continuation bytes. */
+                bool valid = true;
                 for (int i = 1; i <= cont; i++) {
                         if (!_is_continuation(b[i])) {
-                                b[0] = U_SPECIAL;
-                                continue;
+                                valid = false;
+                                break;
                         }
+                }
+
+                if (!valid) {
+                        b[0] = replacement;
+                        count++;
+                        continue;
                 }
 
@@ -551 +573 @@
                  */
 
-                switch (cont) {
-                case 1:
-                        /* 0b110!!!!x 0b10xxxxxx */
-                        if (!(b[0] & 0b00011110))
-                                b[0] = U_SPECIAL;
-
-                        continue;
-                case 2:
-                        /* 0b1110!!!! 0b10!xxxxx 0b10xxxxxx */
-                        if (!(b[0] & 0b00001111) && !(b[1] & 0b00100000))
-                                b[0] = U_SPECIAL;
-
-                        continue;
-                case 3:
-                        /* 0b11110!!! 0b10!!xxxx 0b10xxxxxx 0b10xxxxxx */
-                        if (!(b[0] & 0b00000111) && !(b[1] & 0b00110000))
-                                b[0] = U_SPECIAL;
-
+                /* 0b110!!!!x 0b10xxxxxx */
+                if (cont == 1 && !(b[0] & 0b00011110)) {
+                        b[0] = replacement;
+                        count++;
                         continue;
                 }
-        }
+
+                /* 0b1110!!!! 0b10!xxxxx 0b10xxxxxx */
+                if (cont == 2 && !(b[0] & 0b00001111) && !(b[1] & 0b00100000)) {
+                        b[0] = replacement;
+                        count++;
+                        continue;
+                }
+
+                /* 0b11110!!! 0b10!!xxxx 0b10xxxxxx 0b10xxxxxx */
+                if (cont == 3 && !(b[0] & 0b00000111) && !(b[1] & 0b00110000)) {
+                        b[0] = replacement;
+                        count++;
+                        continue;
+                }
+
+                /* Check for surrogate character encoding. */
+                if (cont == 2 && b[0] == 0xED && b[1] >= 0xA0) {
+                        b[0] = replacement;
+                        count++;
+                        continue;
+                }
+
+                /* Check for out-of-range code points. */
+                if (cont == 3 && (b[0] > 0xF4 || (b[0] == 0xF4 && b[1] >= 0x90))) {
+                        b[0] = replacement;
+                        count++;
+                        continue;
+                }
+
+                b += cont;
+                n -= cont;
+        }
+
+        return count;
+}
+
+size_t str_sanitize(char *str, size_t n, uint8_t replacement)
+{
+        return _str_sanitize(str, n, replacement);
 }
 
@@ -1130 +1176 @@
 
         /* In-place translate invalid bytes to U_SPECIAL. */
-        _sanitize_string(dest, size);
+        _str_sanitize(dest, size, U_SPECIAL);
 }
 
@@ -1159 +1205 @@
 
         /* In-place translate invalid bytes to U_SPECIAL. */
-        _sanitize_string(dest, size);
+        _str_sanitize(dest, size, U_SPECIAL);
 }
 
@@ -1183 +1229 @@
         if (dstr_size < size) {
                 _str_cpyn(dest + dstr_size, size - dstr_size, src);
-                _sanitize_string(dest + dstr_size, size - dstr_size);
+                _str_sanitize(dest + dstr_size, size - dstr_size, U_SPECIAL);
         }
 }
@@ -1762 +1808 @@
 
         memcpy(dest, src, size);
-        _sanitize_string(dest, size);
+        _str_sanitize(dest, size, U_SPECIAL);
         return dest;
 }
@@ -1795 +1841 @@
 
         memcpy(dest, src, size);
-        _sanitize_string(dest, size);
+        _str_sanitize(dest, size, U_SPECIAL);
         dest[size] = 0;
         return dest;