diff options
| -rw-r--r-- | libbutl/char-scanner.cxx | 126 | ||||
| -rw-r--r-- | libbutl/char-scanner.ixx | 56 | ||||
| -rw-r--r-- | libbutl/char-scanner.mxx | 90 | ||||
| -rw-r--r-- | libbutl/char-scanner.txx | 146 | ||||
| -rw-r--r-- | libbutl/manifest-parser.cxx | 24 | ||||
| -rw-r--r-- | libbutl/manifest-parser.ixx | 31 | ||||
| -rw-r--r-- | libbutl/manifest-parser.mxx | 33 | ||||
| -rw-r--r-- | libbutl/manifest-rewriter.cxx | 22 | ||||
| -rw-r--r-- | libbutl/manifest-serializer.cxx | 84 | ||||
| -rw-r--r-- | libbutl/manifest-serializer.mxx | 16 | ||||
| -rw-r--r-- | libbutl/standard-version.cxx | 2 | ||||
| -rw-r--r-- | libbutl/unicode.cxx | 165 | ||||
| -rw-r--r-- | libbutl/unicode.ixx | 72 | ||||
| -rw-r--r-- | libbutl/unicode.mxx | 82 | ||||
| -rw-r--r-- | libbutl/utf8.cxx | 342 | ||||
| -rw-r--r-- | libbutl/utf8.ixx | 305 | ||||
| -rw-r--r-- | libbutl/utf8.mxx | 130 | ||||
| -rw-r--r-- | libbutl/utility.cxx | 132 | ||||
| -rw-r--r-- | libbutl/utility.ixx | 95 | ||||
| -rw-r--r-- | libbutl/utility.mxx | 70 | ||||
| -rw-r--r-- | tests/manifest-parser/driver.cxx | 40 | ||||
| -rw-r--r-- | tests/manifest-rewriter/driver.cxx | 4 | ||||
| -rw-r--r-- | tests/manifest-serializer/driver.cxx | 18 | ||||
| -rw-r--r-- | tests/utf8/driver.cxx | 207 |
24 files changed, 1696 insertions, 596 deletions
diff --git a/libbutl/char-scanner.cxx b/libbutl/char-scanner.cxx deleted file mode 100644 index 85416e5..0000000 --- a/libbutl/char-scanner.cxx +++ /dev/null @@ -1,126 +0,0 @@ -// file : libbutl/char-scanner.cxx -*- C++ -*- -// license : MIT; see accompanying LICENSE file - -#ifndef __cpp_modules_ts -#include <libbutl/char-scanner.mxx> -#endif - -// C includes. - -#ifndef __cpp_lib_modules_ts -#include <string> // char_traits -#include <cstdint> // uint64_t -#include <istream> -#endif - -// Other includes. - -#ifdef __cpp_modules_ts -module butl.char_scanner; - -// Only imports additional to interface. -#ifdef __clang__ -#ifdef __cpp_lib_modules_ts -import std.core; -import std.io; -#endif -import butl.fdstream; -#endif - -#endif - -using namespace std; - -namespace butl -{ - char_scanner:: - char_scanner (istream& is, bool crlf, uint64_t l, uint64_t p) - : line (l), - column (1), - position (p), - is_ (is), - buf_ (dynamic_cast<fdbuf*> (is.rdbuf ())), - gptr_ (nullptr), - egptr_ (nullptr), - crlf_ (crlf) - { - } - - auto char_scanner:: - peek () -> xchar - { - if (unget_) - return ungetc_; - - if (unpeek_) - return unpeekc_; - - if (eos_) - return xchar (xchar::traits_type::eof (), line, column, position); - - int_type v (peek_ ()); - - if (v == xchar::traits_type::eof ()) - eos_ = true; - else if (crlf_ && v == '\r') - { - int_type v1; - do - { - get_ (); - v1 = peek_ (); - } - while (v1 == '\r'); - - if (v1 != '\n') - { - // We need to make sure subsequent calls to peek() return newline. - // - unpeek_ = true; - unpeekc_ = xchar ('\n', line, column, position); - - if (v1 == xchar::traits_type::eof ()) - eos_ = true; - } - - v = '\n'; - } - - return xchar (v, line, column, position); - } - - void char_scanner:: - get (const xchar& c) - { - if (unget_) - unget_ = false; - else - { - if (unpeek_) - { - unpeek_ = false; - } - // When is_.get () returns eof, the failbit is also set (stupid, - // isn't?) which may trigger an exception. To work around this - // we will call peek() first and only call get() if it is not - // eof. But we can only call peek() on eof once; any subsequent - // calls will spoil the failbit (even more stupid). - // - else if (!eos (c)) - get_ (); - - if (!eos (c)) - { - if (c == '\n') - { - line++; - column = 1; - } - else - column++; - - position = pos_ (); - } - } - } -} diff --git a/libbutl/char-scanner.ixx b/libbutl/char-scanner.ixx index 36cc93d..7e9c4b0 100644 --- a/libbutl/char-scanner.ixx +++ b/libbutl/char-scanner.ixx @@ -3,8 +3,30 @@ namespace butl { - inline auto char_scanner:: - get () -> xchar + template <typename V> + inline char_scanner<V>:: + char_scanner (std::istream& is, bool crlf, std::uint64_t l, std::uint64_t p) + : char_scanner (is, validator_type (), crlf, l, p) + { + } + + template <typename V> + inline auto char_scanner<V>:: + peek (std::string& what) -> xchar + { + return peek (&what); + } + + template <typename V> + inline auto char_scanner<V>:: + peek () -> xchar + { + return peek (nullptr /* what */); + } + + template <typename V> + inline auto char_scanner<V>:: + get (std::string* what) -> xchar { if (unget_) { @@ -13,13 +35,28 @@ namespace butl } else { - xchar c (peek ()); + xchar c (peek (what)); get (c); return c; } } - inline void char_scanner:: + template <typename V> + inline auto char_scanner<V>:: + get (std::string& what) -> xchar + { + return get (&what); + } + + template <typename V> + inline auto char_scanner<V>:: + get () -> xchar + { + return get (nullptr /* what */); + } + + template <typename V> + inline void char_scanner<V>:: unget (const xchar& c) { // Because iostream::unget cannot work once eos is reached, we have to @@ -29,7 +66,8 @@ namespace butl ungetc_ = c; } - inline auto char_scanner:: + template <typename V> + inline auto char_scanner<V>:: peek_ () -> int_type { if (gptr_ != egptr_) @@ -48,7 +86,8 @@ namespace butl return r; } - inline void char_scanner:: + template <typename V> + inline void char_scanner<V>:: get_ () { int_type c; @@ -61,11 +100,14 @@ namespace butl else c = is_.get (); // About as fast as ignore() and way faster than tellg(). + validated_ = false; + if (save_ != nullptr && c != xchar::traits_type::eof ()) save_->push_back (static_cast<char_type> (c)); } - inline std::uint64_t char_scanner:: + template <typename V> + inline std::uint64_t char_scanner<V>:: pos_ () const { return buf_ != nullptr ? buf_->tellg () : 0; diff --git a/libbutl/char-scanner.mxx b/libbutl/char-scanner.mxx index 5ad3d61..e57245b 100644 --- a/libbutl/char-scanner.mxx +++ b/libbutl/char-scanner.mxx @@ -10,6 +10,8 @@ #ifndef __cpp_lib_modules_ts #include <string> // char_traits #include <cstdint> // uint64_t +#include <climits> // INT_* +#include <utility> // pair, make_pair() #include <istream> #endif @@ -30,12 +32,26 @@ import butl.fdstream; LIBBUTL_MODEXPORT namespace butl { + // Refer to utf8_validator for details. + // + struct noop_validator + { + std::pair<bool, bool> + validate (char) {return std::make_pair (true, true);} + + std::pair<bool, bool> + validate (char c, std::string&) {return validate (c);} + }; + // Low-level character stream scanner. Normally used as a base for // higher-level lexers. // - class LIBBUTL_SYMEXPORT char_scanner + template <typename V = noop_validator> + class char_scanner { public: + using validator_type = V; + // If the crlf argument is true, then recognize Windows newlines (0x0D // 0x0A) and convert them to just '\n' (0x0A). Note that a standalone // 0x0D is treated "as if" it was followed by 0x0A and multiple 0x0D @@ -49,7 +65,13 @@ LIBBUTL_MODEXPORT namespace butl // and position in the stream (useful when re-scanning data saved with the // save_* facility). // - char_scanner (std::istream& is, + char_scanner (std::istream&, + bool crlf = true, + std::uint64_t line = 1, + std::uint64_t position = 0); + + char_scanner (std::istream&, + validator_type, bool crlf = true, std::uint64_t line = 1, std::uint64_t position = 0); @@ -62,10 +84,10 @@ LIBBUTL_MODEXPORT namespace butl public: // Extended character. It includes line/column/position information and is - // capable of representing EOF. + // capable of representing EOF and invalid characters. // - // Note that implicit conversion of EOF to char_type results in NUL - // character (which means in most cases it is safe to compare xchar to + // Note that implicit conversion of EOF/invalid to char_type results in + // NUL character (which means in most cases it is safe to compare xchar to // char without checking for EOF). // class xchar @@ -76,6 +98,9 @@ LIBBUTL_MODEXPORT namespace butl using char_type = traits_type::char_type; int_type value; + + // Note that the column is of the codepoint this byte belongs to. + // std::uint64_t line; std::uint64_t column; @@ -84,9 +109,12 @@ LIBBUTL_MODEXPORT namespace butl // std::uint64_t position; + static int_type + invalid () {return traits_type::eof () != INT_MIN ? INT_MIN : INT_MAX;} + operator char_type () const { - return value != traits_type::eof () + return value != traits_type::eof () && value != invalid () ? static_cast<char_type> (value) : char_type (0); } @@ -98,27 +126,44 @@ LIBBUTL_MODEXPORT namespace butl : value (v), line (l), column (c), position (p) {} }; + // Note that if any of the get() or peek() functions return an invalid + // character, then the scanning has failed and none of them should be + // called again. + xchar get (); + // As above but in case of an invalid character also return the + // description of why it is invalid. + // + xchar + get (std::string& what); + void get (const xchar& peeked); // Get previously peeked character (faster). void unget (const xchar&); - // Note that if there is an "ungot" character, peek() will return - // that. + // Note that if there is an "ungot" character, peek() will return that. // xchar peek (); - // Tests. In the future we can add tests line alpha(), alnum(), - // etc. + // As above but in case of an invalid character also return the + // description of why it is invalid. + // + xchar + peek (std::string& what); + + // Tests. In the future we can add tests line alpha(), alnum(), etc. // static bool eos (const xchar& c) {return c.value == xchar::traits_type::eof ();} + static bool + invalid (const xchar& c) {return c.value == xchar::invalid ();} + // Line, column and position of the next character to be extracted from // the stream by peek() or get(). // @@ -159,8 +204,8 @@ LIBBUTL_MODEXPORT namespace butl }; protected: - using int_type = xchar::int_type; - using char_type = xchar::char_type; + using int_type = typename xchar::int_type; + using char_type = typename xchar::char_type; int_type peek_ (); @@ -171,11 +216,27 @@ LIBBUTL_MODEXPORT namespace butl std::uint64_t pos_ () const; + xchar + get (std::string* what); + + xchar + peek (std::string* what); + protected: std::istream& is_; - // Note that if you are reading from the buffer directly, then it is - // also your responsibility to save the data. + validator_type val_; + bool decoded_ = true; // The peeked character is last byte of sequence. + bool validated_ = false; // The peeked character has been validated. + + // Note that if you are reading from the buffer directly, then it is also + // your responsibility to call the validator and save the data (see + // save_*(). + // + // Besides that, make sure that the peek() call preceding the scan is + // followed by the get() call (see validated_, decoded_, and unpeek_ for + // the hairy details; realistically, you would probably only direct-scan + // ASCII fragments). // fdbuf* buf_; // NULL if not ifdstream. const char_type* gptr_; @@ -195,3 +256,4 @@ LIBBUTL_MODEXPORT namespace butl } #include <libbutl/char-scanner.ixx> +#include <libbutl/char-scanner.txx> diff --git a/libbutl/char-scanner.txx b/libbutl/char-scanner.txx new file mode 100644 index 0000000..d4e2082 --- /dev/null +++ b/libbutl/char-scanner.txx @@ -0,0 +1,146 @@ +// file : libbutl/char-scanner.txx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef __cpp_lib_modules_ts +#include <utility> // move +#endif + +namespace butl +{ + template <typename V> + char_scanner<V>:: + char_scanner (std::istream& is, + validator_type v, + bool crlf, + std::uint64_t l, + std::uint64_t p) + : line (l), + column (1), + position (p), + is_ (is), + val_ (std::move (v)), + buf_ (dynamic_cast<fdbuf*> (is.rdbuf ())), + gptr_ (nullptr), + egptr_ (nullptr), + crlf_ (crlf) + { + } + + template <typename V> + auto char_scanner<V>:: + peek (std::string* what) -> xchar + { + if (unget_) + return ungetc_; + + if (unpeek_) + return unpeekc_; + + if (eos_) + return xchar (xchar::traits_type::eof (), line, column, position); + + int_type v (peek_ ()); + + if (v == xchar::traits_type::eof ()) + { + if (!decoded_) + { + if (what != nullptr) + *what = "unexpected end of stream"; + + v = xchar::invalid (); + } + + eos_ = true; + } + else + { + auto valid = [what, this] (int_type v) + { + if (validated_) + return true; + + char c (xchar::traits_type::to_char_type (v)); + std::pair<bool, bool> r (what != nullptr + ? val_.validate (c, *what) + : val_.validate (c)); + + decoded_ = r.second; + validated_ = true; + return r.first; + }; + + if (!valid (v)) + v = xchar::invalid (); + else if (crlf_ && v == '\r') + { + // Note that '\r' is a valid character (otherwise we won't be here), + // so we don't validate it again below. We also postpone the + // validation of the next non-'\r' character (except EOF) until the + // next peek() call. + // + int_type v1; + do + { + get_ (); // Sets validated_ to false. + v1 = peek_ (); + } + while (v1 == '\r'); + + if (v1 != '\n') + { + // We need to make sure subsequent calls to peek() return newline. + // + unpeek_ = true; + unpeekc_ = xchar ('\n', line, column, position); + + // Note that the previous character is decoded ('\r') and so EOF is + // legitimate. + // + if (v1 == xchar::traits_type::eof ()) + eos_ = true; + } + + v = '\n'; + } + } + + return xchar (v, line, column, position); + } + + template <typename V> + void char_scanner<V>:: + get (const xchar& c) + { + if (unget_) + unget_ = false; + else + { + if (unpeek_) + { + unpeek_ = false; + } + // When is_.get () returns eof, the failbit is also set (stupid, + // isn't?) which may trigger an exception. To work around this + // we will call peek() first and only call get() if it is not + // eof. But we can only call peek() on eof once; any subsequent + // calls will spoil the failbit (even more stupid). + // + else if (!eos (c)) + get_ (); + + if (!eos (c)) + { + if (c == '\n') + { + line++; + column = 1; + } + else if (decoded_) // The character is the last in a sequence? + column++; + + position = pos_ (); + } + } + } +} diff --git a/libbutl/manifest-parser.cxx b/libbutl/manifest-parser.cxx index 4de59b7..9514bbd 100644 --- a/libbutl/manifest-parser.cxx +++ b/libbutl/manifest-parser.cxx @@ -89,7 +89,7 @@ namespace butl parse_name (r); skip_spaces (); - c = get (); + c = get ("manifest"); if (eos (c)) { @@ -117,7 +117,7 @@ namespace butl skip_spaces (); parse_value (r); - c = peek (); + c = peek ("manifest"); // The character after the value should be either a newline or eos. // @@ -126,7 +126,7 @@ namespace butl r.end_pos = c.position; if (c == '\n') - get (); + get (c); // Now figure out whether what we've got makes sense, depending // on the state we are in. @@ -217,6 +217,8 @@ namespace butl void manifest_parser:: parse_name (name_value& r) { + auto peek = [this] () {return manifest_parser::peek ("manifest name");}; + xchar c (peek ()); r.name_line = c.line; @@ -228,13 +230,19 @@ namespace butl break; r.name += c; - get (); + get (c); } } void manifest_parser:: parse_value (name_value& r) { + auto peek = [this] () {return manifest_parser::peek ("manifest value");}; + + // Here we don't always track the last peeked character. + // + auto get = [this] () {manifest_parser::get ("manifest value");}; + xchar c (peek ()); r.value_line = c.line; @@ -408,6 +416,8 @@ namespace butl pair<manifest_parser::xchar, uint64_t> manifest_parser:: skip_spaces () { + auto peek = [this] () {return manifest_parser::peek ("manifest");}; + xchar c (peek ()); bool start (c.column == 1); uint64_t lp (c.position); @@ -437,12 +447,12 @@ namespace butl if (!start) return make_pair (c, lp); - get (); + get (c); // Read until newline or eos. // for (c = peek (); !eos (c) && c != '\n'; c = peek ()) - get (); + get (c); continue; } @@ -450,7 +460,7 @@ namespace butl return make_pair (c, lp); // Not a space. } - get (); + get (c); } return make_pair (c, lp); diff --git a/libbutl/manifest-parser.ixx b/libbutl/manifest-parser.ixx index e616ad9..bc5246c 100644 --- a/libbutl/manifest-parser.ixx +++ b/libbutl/manifest-parser.ixx @@ -3,6 +3,37 @@ namespace butl { + + inline auto manifest_parser:: + get (const char* what) -> xchar + { + xchar c (base::get (ebuf_)); + + if (invalid (c)) + throw manifest_parsing (name_, + c.line, c.column, + std::string ("invalid ") + what + ": " + ebuf_); + return c; + } + + inline void manifest_parser:: + get (const xchar& peeked) + { + base::get (peeked); + } + + inline auto manifest_parser:: + peek (const char* what) -> xchar + { + xchar c (base::peek (ebuf_)); + + if (invalid (c)) + throw manifest_parsing (name_, + c.line, c.column, + std::string ("invalid ") + what + ": " + ebuf_); + return c; + } + inline manifest_name_value manifest_parser:: next () { diff --git a/libbutl/manifest-parser.mxx b/libbutl/manifest-parser.mxx index adf6181..77addff 100644 --- a/libbutl/manifest-parser.mxx +++ b/libbutl/manifest-parser.mxx @@ -25,10 +25,12 @@ export module butl.manifest_parser; import std.core; import std.io; #endif +import butl.utf8; import butl.optional; import butl.char_scanner; import butl.manifest_types; #else +#include <libbutl/utf8.mxx> #include <libbutl/optional.mxx> #include <libbutl/char-scanner.mxx> #include <libbutl/manifest-types.mxx> @@ -54,7 +56,8 @@ LIBBUTL_MODEXPORT namespace butl std::string description; }; - class LIBBUTL_SYMEXPORT manifest_parser: protected butl::char_scanner + class LIBBUTL_SYMEXPORT manifest_parser: + protected char_scanner<utf8_validator> { public: // The filter, if specified, is called by next() prior to returning the @@ -69,7 +72,10 @@ LIBBUTL_MODEXPORT namespace butl manifest_parser (std::istream& is, const std::string& name, std::function<filter_function> filter = {}) - : char_scanner (is), name_ (name), filter_ (std::move (filter)) {} + : char_scanner (is, + utf8_validator (codepoint_types::graphic, U"\n\r\t")), + name_ (name), + filter_ (std::move (filter)) {} const std::string& name () const {return name_;} @@ -97,6 +103,8 @@ LIBBUTL_MODEXPORT namespace butl split_comment (const std::string&); private: + using base = char_scanner<utf8_validator>; + void parse_next (manifest_name_value&); @@ -114,12 +122,33 @@ LIBBUTL_MODEXPORT namespace butl std::pair<xchar, std::uint64_t> skip_spaces (); + // As base::get() but in case of an invalid character throws + // manifest_parsing. + // + xchar + get (const char* what); + + // Get previously peeked character (faster). + // + void + get (const xchar&); + + // As base::peek() but in case of an invalid character throws + // manifest_parsing. + // + xchar + peek (const char* what); + private: const std::string name_; const std::function<filter_function> filter_; enum {start, body, end} s_ = start; std::string version_; // Current format version. + + // Buffer for a get()/peek() potential error. + // + std::string ebuf_; }; // Parse and return a single manifest. Throw manifest_parsing in case of an diff --git a/libbutl/manifest-rewriter.cxx b/libbutl/manifest-rewriter.cxx index ba0c866..e38d5f4 100644 --- a/libbutl/manifest-rewriter.cxx +++ b/libbutl/manifest-rewriter.cxx @@ -30,8 +30,10 @@ import butl.fdstream; import butl.manifest_types; #endif +import butl.utility; // utf8_length() import butl.manifest_serializer; #else +#include <libbutl/utility.mxx> #include <libbutl/manifest-serializer.mxx> #endif @@ -101,8 +103,16 @@ namespace butl manifest_serializer s (os, path_.string (), long_lines_); + // Note that the name can be surrounded with the ASCII whitespace + // characters and the start_pos refers to the first character in the + // line. + // + // Also note that we assume the already serialized name to be a valid + // UTF-8 byte string and so utf8_length() may not throw. + // s.write_value (nv.value, - static_cast<size_t> (nv.colon_pos - nv.start_pos + 2)); + static_cast<size_t> (nv.colon_pos - nv.start_pos) - + (nv.name.size () - utf8_length (nv.name)) + 2); } os << suffix; @@ -128,15 +138,21 @@ namespace butl os << '\n'; manifest_serializer s (os, path_.string (), long_lines_); - s.write_name (nv.name); + size_t n (s.write_name (nv.name)); os << ':'; if (!nv.value.empty ()) { os << ' '; + + // Note that the name can be surrounded with the ASCII whitespace + // characters and the start_pos refers to the first character in the + // line. + // s.write_value (nv.value, - static_cast<size_t> (nv.colon_pos - nv.start_pos + 2)); + static_cast<size_t> (nv.colon_pos - nv.start_pos) - + (nv.name.size () - n) + 2); } os << suffix; diff --git a/libbutl/manifest-serializer.cxx b/libbutl/manifest-serializer.cxx index 0a81478..6a26a15 100644 --- a/libbutl/manifest-serializer.cxx +++ b/libbutl/manifest-serializer.cxx @@ -30,6 +30,11 @@ import std.io; import butl.manifest_types; #endif +import butl.utf8; +import butl.utility; +#else +#include <libbutl/utf8.mxx> +#include <libbutl/utility.mxx> #endif using namespace std; @@ -86,13 +91,13 @@ namespace butl break; } - write_name (n); + size_t l (write_name (n)); os_ << ':'; if (!v.empty ()) { os_ << ' '; - write_value (v, n.size () + 2); + write_value (v, l + 2); } os_ << endl; @@ -111,6 +116,10 @@ namespace butl if (s_ == end) throw serialization (name_, "serialization after eos"); + string what; + if (!utf8 (t, what, codepoint_types::graphic, U"\n\r\t")) + throw serialization (name_, "invalid comment: " + what); + os_ << '#'; if (!t.empty ()) @@ -144,7 +153,7 @@ namespace butl return r; } - void manifest_serializer:: + size_t manifest_serializer:: write_name (const string& n) { if (n.empty ()) @@ -153,43 +162,76 @@ namespace butl if (n[0] == '#') throw serialization (name_, "name starts with '#'"); + size_t r (0); + pair<bool, bool> v; + utf8_validator val (codepoint_types::graphic, U"\n\r\t"); + + string what; for (char c: n) { - switch (c) + v = val.validate (c, what); + + if (!v.first) + throw serialization (name_, "invalid name: " + what); + + if (v.second) // Sequence last byte? { - case ' ': - case '\t': - case '\r': - case '\n': throw serialization (name_, "name contains whitespace"); - case ':': throw serialization (name_, "name contains ':'"); - default: break; + // Note: ASCII characters may not be a part of a multi-byte sequence. + // + switch (c) + { + case ' ': + case '\t': + case '\r': + case '\n': throw serialization (name_, "name contains whitespace"); + case ':': throw serialization (name_, "name contains ':'"); + default: break; + } + + ++r; } } + // Make sure that the last UTF-8 sequence is complete. + // + if (!v.second) + throw serialization (name_, "invalid name: incomplete UTF-8 sequence"); + os_ << n; + return r; } void manifest_serializer:: write_value (const char* s, size_t n, size_t cl) { + utf8_validator val (codepoint_types::graphic, U"\n\r\t"); + char c ('\0'); + bool b (true); // Begin of UTF-8 byte sequence. - // The idea is to break on the 77th character (i.e., write it - // on the next line) which means we have written 76 characters + // The idea is to break on the 77th codepoint (i.e., write it + // on the next line) which means we have written 76 codepoints // on this line plus 2 for '\' and '\n', which gives us 78. // - for (const char* e (s + n); s != e; s++, cl++) + string what; + for (const char* e (s + n); s != e; s++) { char pc (c); c = *s; + pair<bool, bool> v (val.validate (c, what)); + + if (!v.first) + throw serialization (name_, "invalid value: " + what); + // Note that even the "hard" break (see below) is not that hard when it // comes to breaking the line right after the backslash. Doing so would // inject the redundant newline character, as the line-terminating // backslash would be escaped. So we delay breaking till the next - // non-backslash character. + // non-backslash character. We also delay until the beginning of a UTF-8 + // sequence. // - if (pc != '\\' && !long_lines_) + if (pc != '\\' && b && !long_lines_) { bool br (false); // Break the line. @@ -237,8 +279,18 @@ namespace butl } os_ << c; + + b = v.second; + + if (b) + ++cl; } + // Make sure that the last UTF-8 sequence is complete. + // + if (!b) + throw serialization (name_, "invalid value: incomplete UTF-8 sequence"); + // What comes next is always a newline. If the last character that // we have written is a backslash, escape it. // @@ -256,7 +308,7 @@ namespace butl // Use the multi-line mode in any of the following cases: // - // - column offset is too large (say greater than 39 (78/2) characters; we + // - column offset is too large (say greater than 39 (78/2) codepoints; we // cannot start on the next line since that would start the multi-line // mode) // - value contains newlines diff --git a/libbutl/manifest-serializer.mxx b/libbutl/manifest-serializer.mxx index f114ffb..b73c255 100644 --- a/libbutl/manifest-serializer.mxx +++ b/libbutl/manifest-serializer.mxx @@ -60,7 +60,7 @@ LIBBUTL_MODEXPORT namespace butl const std::string& value); // Unless long_lines is true, break lines in values (including multi-line) - // so that their length does not exceed 78 characters (including '\n'). + // so that their length does not exceed 78 codepoints (including '\n'). // manifest_serializer (std::ostream& os, const std::string& name, @@ -108,23 +108,23 @@ LIBBUTL_MODEXPORT namespace butl void write_next (const std::string& name, const std::string& value); - // Validate and write a name. + // Validate and write a name and return its length in codepoints. // - void + size_t write_name (const std::string&); // Write a value assuming the current line already has the specified - // offset. If the resulting line length would be too large then the - // multi-line representation will be used. It is assumed that the name, - // followed by the colon, is already written. + // codepoint offset. If the resulting line length would be too large then + // the multi-line representation will be used. It is assumed that the + // name, followed by the colon, is already written. // void write_value (const std::string&, std::size_t offset); // Write the specified number of characters from the specified string // (assuming there are no newlines) split into multiple lines at or near - // the 78 characters boundary. Assume the current line already has the - // specified offset. + // the 78 codepoints boundary. Assume the current line already has the + // specified codepoint offset. // void write_value (const char* s, std::size_t n, std::size_t offset); diff --git a/libbutl/standard-version.cxx b/libbutl/standard-version.cxx index c27b064..a9f5eb8 100644 --- a/libbutl/standard-version.cxx +++ b/libbutl/standard-version.cxx @@ -41,6 +41,8 @@ using namespace std; namespace butl { + using std::to_string; + // Parse uint64_t from the specified string starting at the specified // position and check the min/max constraints. If successful, save the // result, update the position to point to the next character, and return diff --git a/libbutl/unicode.cxx b/libbutl/unicode.cxx new file mode 100644 index 0000000..4219846 --- /dev/null +++ b/libbutl/unicode.cxx @@ -0,0 +1,165 @@ +// file : libbutl/unicode.cxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef __cpp_modules_ts +#include <libbutl/unicode.mxx> +#endif + +#ifndef __cpp_lib_modules_ts +#include <string> +#include <ostream> +#include <cstdint> + +#include <cstddef> // size_t +#include <utility> // pair +#include <algorithm> // lower_bound() +#endif + +#ifdef __cpp_modules_ts +module butl.unicode; + +// Only imports additional to interface. +#ifdef __clang__ +#ifdef __cpp_lib_modules_ts +import std.core; +import std.io; +#endif +#endif + +#endif + +using namespace std; + +namespace butl +{ + // Sorted arrays of the Unicode codepoint ranges corresponding to the + // codepoint types (see the Types of Code Points table in the Unicode 12.0 + // Standard for details). Note that code type range lists (but not ranges + // themselves) may overlap. + // + // Also note that the graphic type codepoints are numerous and scattered. + // Thus, we will consider a codepoint to be of the graphic type if it is not + // of any other type. + // + using codepoint_range = pair<char32_t, char32_t>; + + static const codepoint_range cn_rs[] = // Control. + { + {0x00, 0x1F}, + {0x7F, 0x9F} + }; + + static const codepoint_range fr_rs[] = // Format. + { + {0x000AD, 0x000AD}, + {0x00600, 0x00605}, + {0x0061C, 0x0061C}, + {0x006DD, 0x006DD}, + {0x0070F, 0x0070F}, + {0x008E2, 0x008E2}, + {0x0180E, 0x0180E}, + {0x0200B, 0x0200F}, + {0x0202A, 0x0202E}, + {0x02060, 0x02064}, + {0x02066, 0x0206F}, + {0x0FEFF, 0x0FEFF}, + {0x0FFF9, 0x0FFFB}, + {0x110BD, 0x110BD}, + {0x110CD, 0x110CD}, + {0x13430, 0x13438}, + {0x1BCA0, 0x1BCA3}, + {0x1D173, 0x1D17A}, + {0xE0001, 0xE0001}, + {0xE0020, 0xE007F} + }; + + static const codepoint_range pr_rs[] = // Private-use. + { + {0x00E000, 0x00F8FF}, + {0x0F0000, 0x10FFFF} + }; + + static const codepoint_range nc_rs[] = // Non-character. + { + {0xFDD0, 0xFDEF} + }; + + static const codepoint_range rs_rs[] = // Reserved. + { + {0x30000, 0xE0000}, + {0xE0002, 0xE001F}, + {0xE0080, 0xE00FF}, + {0xE01F0, 0xEFFFF} + }; + + struct codepoint_type_ranges + { + codepoint_types type; + const codepoint_range* begin; + const codepoint_range* end; + }; + + static const codepoint_type_ranges ct_ranges[] = + { + { + codepoint_types::control, + cn_rs, + cn_rs + sizeof (cn_rs) / sizeof (*cn_rs) + }, + { + codepoint_types::format, + fr_rs, + fr_rs + sizeof (fr_rs) / sizeof (*fr_rs) + }, + { + codepoint_types::private_use, + pr_rs, + pr_rs + sizeof (pr_rs) / sizeof (*pr_rs) + }, + { + codepoint_types::non_character, + nc_rs, + nc_rs + sizeof (nc_rs) / sizeof (*nc_rs) + }, + { + codepoint_types::reserved, + rs_rs, + rs_rs + sizeof (rs_rs) / sizeof (*rs_rs) + } + }; + + // Return the codepoint type of a range if the codepoint value falls into + // one and the graphic type otherwise. + // + // Note that this is a type detection fallback (see codepoint_type() for + // details). + // + codepoint_types + codepoint_type_lookup (char32_t c) + { + // Note that the codepoint type range lists may overlap. Thus, we iterate + // over all of them until there is a match. + // + for (size_t i (0); i != sizeof (ct_ranges) / sizeof (*ct_ranges); ++i) + { + const codepoint_type_ranges& rs (ct_ranges[i]); + + // Find the range that either contains the codepoint or lays to the + // right of it. Note that here we assume a range to be less than a + // codepoint value if it lays to the left of the codepoint. + // + const codepoint_range* r ( + lower_bound (rs.begin, rs.end, + c, + [] (const codepoint_range& r, char32_t c) + { + return r.second < c; + })); + + if (r != rs.end && r->first <= c) // Contains the codepoint? + return rs.type; + } + + return codepoint_types::graphic; + } +} diff --git a/libbutl/unicode.ixx b/libbutl/unicode.ixx new file mode 100644 index 0000000..cba4fd2 --- /dev/null +++ b/libbutl/unicode.ixx @@ -0,0 +1,72 @@ +// file : libbutl/unicode.ixx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +namespace butl +{ + inline codepoint_types + operator&= (codepoint_types& x, codepoint_types y) + { + return x = static_cast<codepoint_types> ( + static_cast<std::uint16_t> (x) & + static_cast<std::uint16_t> (y)); + } + + inline codepoint_types + operator|= (codepoint_types& x, codepoint_types y) + { + return x = static_cast<codepoint_types> ( + static_cast<std::uint16_t> (x) | + static_cast<std::uint16_t> (y)); + } + + inline codepoint_types + operator& (codepoint_types x, codepoint_types y) + { + return x &= y; + } + + inline codepoint_types + operator| (codepoint_types x, codepoint_types y) + { + return x |= y; + } + + LIBBUTL_SYMEXPORT codepoint_types + codepoint_type_lookup (char32_t); + + inline codepoint_types + codepoint_type (char32_t c) + { + // Optimize for the common case (printable ASCII characters). + // + if (c >= 0x20 && c <= 0x7E) // Printable ASCII? + return codepoint_types::graphic; + else if (c > 0x10FFFF || (c >= 0xD800 && c <= 0xDFFF)) // Invalid? + return codepoint_types::none; + else if ((c & 0xFFFF) >= 0xFFFE) // Non-range based? + return codepoint_types::non_character; + else + return codepoint_type_lookup (c); + } + + inline std::string + to_string (codepoint_types t) + { + // Note that we use the terms from the Unicode standard ("private-use" + // rather than "private use", "noncharacter" rather than "non-character"). + // + switch (t) + { + case codepoint_types::graphic: return "graphic"; + case codepoint_types::format: return "format"; + case codepoint_types::control: return "control"; + case codepoint_types::private_use: return "private-use"; + case codepoint_types::non_character: return "noncharacter"; // No dash. + case codepoint_types::reserved: return "reserved"; + case codepoint_types::none: + case codepoint_types::any: return ""; + } + + return ""; // Types combination. + } +} diff --git a/libbutl/unicode.mxx b/libbutl/unicode.mxx new file mode 100644 index 0000000..b846476 --- /dev/null +++ b/libbutl/unicode.mxx @@ -0,0 +1,82 @@ +// file : libbutl/unicode.mxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef __cpp_modules_ts +#pragma once +#endif + +// C includes. + +#ifndef __cpp_lib_modules_ts +#include <string> +#include <ostream> +#include <cstdint> // uint16_t +#endif + +// Other includes. + +#ifdef __cpp_modules_ts +export module butl.unicode; +#ifdef __cpp_lib_modules_ts +import std.core; +import std.io; +#endif +#endif + +#include <libbutl/export.hxx> + +LIBBUTL_MODEXPORT namespace butl +{ + // Note that the Unicode Standard requires the surrogates ([D800 DFFF]) to + // only be used in the context of the UTF-16 character encoding form. Thus, + // we omit the surrogate codepoint type and assume surrogates as invalid + // codepoints. + // + enum class codepoint_types: std::uint16_t + { + // Useful to denote invalid codepoints or when building the type set + // incrementally. + // + none = 0x00, + + graphic = 0x01, // L(etter), M(ark), N(number), P(uncturation), + // S(symbol), Zs(separator, space) + format = 0x02, + control = 0x04, + private_use = 0x08, + non_character = 0x10, + reserved = 0x20, + + any = 0x3f + }; + + codepoint_types operator& (codepoint_types, codepoint_types); + codepoint_types operator| (codepoint_types, codepoint_types); + codepoint_types operator&= (codepoint_types&, codepoint_types); + codepoint_types operator|= (codepoint_types&, codepoint_types); + + // Return the codepoint type for a valid codepoint value and none otherwise. + // + // Note that the valid codepoint ranges are [0 D800) and (DFFF 10FFFF]. + // + codepoint_types + codepoint_type (char32_t); + + // Return the type name for a single codepoint type and empty string for + // `none` and `any`. + // + // Potential future improvements: + // - add the none value name parameter ("invalid" by default) + // - produce names for type masks ("graphic, format", "any", etc) + // + std::string + to_string (codepoint_types); + + inline std::ostream& + operator<< (std::ostream& os, codepoint_types ts) + { + return os << to_string (ts); + } +} + +#include <libbutl/unicode.ixx> diff --git a/libbutl/utf8.cxx b/libbutl/utf8.cxx deleted file mode 100644 index 0f24559..0000000 --- a/libbutl/utf8.cxx +++ /dev/null @@ -1,342 +0,0 @@ -// file : libbutl/utf8.cxx -*- C++ -*- -// license : MIT; see accompanying LICENSE file - -#ifndef __cpp_modules_ts -#include <libbutl/utility.mxx> -#endif - -#ifndef __cpp_lib_modules_ts -#include <string> -#include <cstddef> - -#include <algorithm> // lower_bound() -#endif - -#ifdef __cpp_modules_ts -module butl.utility; - -// Only imports additional to interface. -#ifdef __clang__ -#ifdef __cpp_lib_modules_ts -import std.core; -import std.io; -#endif -#endif - -#endif - -namespace butl -{ - using namespace std; - - // Sorted arrays of the Unicode codepoint ranges corresponding to the - // codepoint types. Note that code type range lists (but not ranges - // themselves) may overlap. - // - // Note that the graphic type codepoints are numerous and scattered. Thus, - // we will consider a codepoint to be of the graphic type if it is not of - // any other type. - // - using codepoint_range = pair<char32_t, char32_t>; - - static const codepoint_range cn_rs[] = // Control. - { - {0x00, 0x1F}, - {0x7F, 0x9F} - }; - - static const codepoint_range fr_rs[] = // Format. - { - {0x000AD, 0x000AD}, - {0x00600, 0x00605}, - {0x0061C, 0x0061C}, - {0x006DD, 0x006DD}, - {0x0070F, 0x0070F}, - {0x008E2, 0x008E2}, - {0x0180E, 0x0180E}, - {0x0200B, 0x0200F}, - {0x0202A, 0x0202E}, - {0x02060, 0x02064}, - {0x02066, 0x0206F}, - {0x0FEFF, 0x0FEFF}, - {0x0FFF9, 0x0FFFB}, - {0x110BD, 0x110BD}, - {0x110CD, 0x110CD}, - {0x13430, 0x13438}, - {0x1BCA0, 0x1BCA3}, - {0x1D173, 0x1D17A}, - {0xE0001, 0xE0001}, - {0xE0020, 0xE007F} - }; - - static const codepoint_range pr_rs[] = // Private-use. - { - {0x00E000, 0x00F8FF}, - {0x0F0000, 0x10FFFF} - }; - - static const codepoint_range nc_rs[] = // Non-character. - { - {0xFDD0, 0xFDEF} - }; - - static const codepoint_range rs_rs[] = // Reserved. - { - {0x30000, 0xE0000}, - {0xE0002, 0xE001F}, - {0xE0080, 0xE00FF}, - {0xE01F0, 0xEFFFF} - }; - - struct codepoint_type_ranges - { - codepoint_types type; - const codepoint_range* begin; - const codepoint_range* end; - }; - - static const codepoint_type_ranges ct_ranges[] = - { - { - codepoint_types::control, - cn_rs, - cn_rs + sizeof (cn_rs) / sizeof (*cn_rs) - }, - { - codepoint_types::format, - fr_rs, - fr_rs + sizeof (fr_rs) / sizeof (*fr_rs) - }, - { - codepoint_types::private_use, - pr_rs, - pr_rs + sizeof (pr_rs) / sizeof (*pr_rs) - }, - { - codepoint_types::non_character, - nc_rs, - nc_rs + sizeof (nc_rs) / sizeof (*nc_rs) - }, - { - codepoint_types::reserved, - rs_rs, - rs_rs + sizeof (rs_rs) / sizeof (*rs_rs) - } - }; - - bool - utf8 (const string& s, codepoint_types ts, const char32_t* wl) - { - // A UCS-4 character is encoded as the UTF-8 byte sequence as follows, - // depending on the value range it falls into: - // - // 0x00000000 - 0x0000007F: - // 0xxxxxxx - // - // 0x00000080 - 0x000007FF: - // 110xxxxx 10xxxxxx - // - // 0x00000800 - 0x0000FFFF: - // 1110xxxx 10xxxxxx 10xxxxxx - // - // 0x00010000 - 0x001FFFFF: - // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - // - // 0x00200000 - 0x03FFFFFF: - // 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx - // - // 0x04000000 - 0x7FFFFFFF: - // 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx - // - // Also note that the Unicode Standard (as of 12.1) specifies no - // codepoints above 0x10FFFF, so we consider 5- and 6-byte length UTF-8 - // sequences as invalid (we could have added `unspecified` codepoint type - // except that there are no UTF-8 validation tables defined for these - // sequences). - // - size_t n (s.size ()); - - for (size_t i (0); i != n; ) - { - // Detect the UTF-8 byte sequence length based on its first byte. While - // at it, start calculating the Unicode codepoint value. - // - size_t sn; - char32_t c; - unsigned char b1 (s[i]); - - if (b1 < 0x80) - { - sn = 1; - c = b1; - } - else if (b1 < 0xE0) - { - sn = 2; - c = b1 & 0x1F; // Takes 5 rightmost bits of the first sequence byte. - } - else if (b1 < 0xF0) - { - sn = 3; - c = b1 & 0xF; // Takes 4 rightmost bits of the first sequence byte. - } - else if (b1 < 0xF8) - { - sn = 4; - c = b1 & 0x7; // Takes 3 rightmost bits of the first sequence byte. - } - else - return false; // The byte starts 5- or 6-byte length sequence. - - // Bail out if the string doesn't contain all the requred codepoint - // encoding bytes. - // - if (sn > n - i) - return false; - - // Note that while a codepoint may potentially be encoded with byte - // sequences of different lengths, only the shortest encoding sequence - // is considered well-formed. Also a well-formed sequence may not be - // decoded into a UTF-16 surrogate value ([D800 DFFF]) or a value that - // is greater than the max codepoint value (0x10FFFF). We will check all - // that using the Well-Formed UTF-8 Byte Sequences table (provided by - // the Unicode 12.0 Standard) which also takes care of the missing UTF-8 - // sequence bytes. - // - // Return true if a byte value belongs to the specified range. - // - auto belongs = [] (unsigned char c, unsigned char l, unsigned char r) - { - return c >= l && c <= r; - }; - - switch (sn) - { - case 1: break; // Always well-formed by the definition (see above). - case 2: - { - // [000080 0007FF]: [C2 DF] [80 BF] - // - // Check the first/second bytes combinations: - // - if (!(belongs (b1, 0xC2, 0xDF) && belongs (s[i + 1], 0x80, 0xBF))) - return false; - - break; - } - case 3: - { - // [000800 000FFF]: E0 [A0 BF] [80 BF] - // [001000 00CFFF]: [E1 EC] [80 BF] [80 BF] - // [00D000 00D7FF]: ED [80 9F] [80 BF] ; Excludes surrogates. - // [00E000 00FFFF]: [EE EF] [80 BF] [80 BF] - // - unsigned char b2 (s[i + 1]); - - if (!((b1 == 0xE0 && belongs (b2, 0xA0, 0xBF)) || - (belongs (b1, 0xE1, 0xEC) && belongs (b2, 0x80, 0xBF)) || - (b1 == 0xED && belongs (b2, 0x80, 0x9F)) || - (belongs (b1, 0xEE, 0xEF) && belongs (b2, 0x80, 0xBF))) || - !belongs (s[i + 2], 0x80, 0xBF)) - return false; - - break; - } - case 4: - { - // [010000 03FFFF]: F0 [90 BF] [80 BF] [80 BF] - // [040000 0FFFFF]: [F1 F3] [80 BF] [80 BF] [80 BF] - // [100000 10FFFF]: F4 [80 8F] [80 BF] [80 BF] - // - unsigned char b2 (s[i + 1]); - - if (!((b1 == 0xF0 && belongs (b2, 0x90, 0xBF)) || - (belongs (b1, 0xF1, 0xF3) && belongs (b2, 0x80, 0xBF)) || - (b1 == 0xF4 && belongs (b2, 0x80, 0x8F))) || - !belongs (s[i + 2], 0x80, 0xBF) || - !belongs (s[i + 3], 0x80, 0xBF)) - return false; - - break; - } - } - - // For the remaining sequence bytes, "append" their 6 rightmost bits to - // the resulting codepoint value. - // - --sn; - ++i; - - for (size_t n (i + sn); i != n; ++i) - c = (c << 6) | (s[i] & 0x3F); - - // Check the decoded codepoint, unless any codepoint type is allowed. - // - if (ts == codepoint_types::any) - continue; - - using traits = u32string::traits_type; - - // Check if the decoded codepoint is whitelisted. - // - if (wl != nullptr && - traits::find (wl, traits::length (wl), c) != nullptr) - continue; - - // Match the decoded codepoint type against the specified type set. - // - // Detect the codepoint type (see the Types of Code Points table in the - // Unicode 12.0 Standard for details). - // - codepoint_types ct; - - // Optimize for the common case (printable ASCII characters). - // - if (c >= 0x20 && c <= 0x7E) - ct = codepoint_types::graphic; - else if ((c & 0xFFFF) >= 0xFFFE) // Non-range based detection. - ct = codepoint_types::non_character; - else - { - // Note that we consider a codepoint to be of the graphic type if it - // is not of any other type (see above). - // - ct = codepoint_types::graphic; - - // Note that the codepoint type range lists may overlap. Thus, we - // iterate over all of them until there is a match. - // - for (size_t i (0); i != sizeof (ct_ranges) / sizeof (*ct_ranges); ++i) - { - const codepoint_type_ranges& rs (ct_ranges[i]); - - // Find the range that either contains the codepoint or lays to the - // right of it. Note that here we assume a range to be less than a - // codepoint if it lays to the left of the codepoint. - // - const codepoint_range* r ( - lower_bound (rs.begin, rs.end, - c, - [] (const codepoint_range& r, char32_t c) - { - return r.second < c; - })); - - if (r != rs.end && r->first <= c) // Contains the codepoint? - { - ct = rs.type; - break; - } - } - } - - // Now check if the codepoint type matches the specified set. Note: also - // covers the `ts == codepoint_types::none` case. - // - if ((ct & ts) == codepoint_types::none) - return false; - } - - return true; - } -} diff --git a/libbutl/utf8.ixx b/libbutl/utf8.ixx new file mode 100644 index 0000000..3d2e092 --- /dev/null +++ b/libbutl/utf8.ixx @@ -0,0 +1,305 @@ +// file : libbutl/utf8.ixx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +namespace butl +{ + inline utf8_validator:: + utf8_validator (codepoint_types ts, const char32_t* wl) + : types_ (ts), + whitelist_ (wl) + { + } + + inline std::pair<bool, bool> utf8_validator:: + validate (char c) + { + return validate (c, nullptr /* what */); + } + + inline std::pair<bool, bool> utf8_validator:: + validate (char c, std::string& what) + { + return validate (c, &what); + } + + inline std::pair<bool, bool> utf8_validator:: + validate (char c, std::string* what) + { + using namespace std; + + // A UCS-4 character is encoded as the UTF-8 byte sequence as follows, + // depending on the value range it falls into: + // + // 0x00000000 - 0x0000007F: + // 0xxxxxxx + // + // 0x00000080 - 0x000007FF: + // 110xxxxx 10xxxxxx + // + // 0x00000800 - 0x0000FFFF: + // 1110xxxx 10xxxxxx 10xxxxxx + // + // 0x00010000 - 0x001FFFFF: + // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + // + // 0x00200000 - 0x03FFFFFF: + // 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx + // + // 0x04000000 - 0x7FFFFFFF: + // 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx + // + // Also note that the Unicode Standard (as of 12.1) specifies no + // codepoints above 0x10FFFF, so we consider 5- and 6-byte length UTF-8 + // sequences as invalid (we could have added `unspecified` codepoint type + // except that there are no UTF-8 validation tables defined for these + // sequences). + // + unsigned char b (c); + + // Compose the detailed "invalid UTF-8 sequence byte" error. + // + auto byte_error = [c, b, this] () + { + string s ("invalid UTF-8 sequence "); + + const char* names[] = {"first", "second", "third", "forth"}; + s += names[seq_index_]; + s += " byte (0x"; + + const char digits[] = "0123456789ABCDEF"; + s += digits[(b >> 4) & 0xF]; + s += digits[b & 0xF]; + + // If the byte happens to be a printable ASCII character then let's + // print it as a character as well. This can help a bit with grepping + // through text while troubleshooting. + // + if (b >= 0x20 && b <= 0x7E) + { + s += " '"; + s += c; + s += "'"; + } + + s += ")"; + return s; + }; + + // Detect the byte sequence length based on its first byte. While at it, + // start calculating the resulting Unicode codepoint value. + // + if (seq_index_ == 0) + { + if (b < 0x80) + { + seq_size_ = 1; + codepoint_ = b; + } + else if (b < 0xE0) + { + seq_size_ = 2; + codepoint_ = b & 0x1F; // Takes 5 rightmost bits. + } + else if (b < 0xF0) + { + seq_size_ = 3; + codepoint_ = b & 0xF; // Takes 4 rightmost bits. + } + else if (b < 0xF8) + { + seq_size_ = 4; + codepoint_ = b & 0x7; // Takes 3 rightmost bits. + } + else + { + if (what != nullptr) + { + if (b < 0xFE) + { + *what = b < 0xFC ? "5" : "6"; + *what += "-byte length UTF-8 sequence"; + } + else + *what = byte_error (); + } + + return make_pair (false, false); // Invalid byte. + } + } + + // Note that while a codepoint may potentially be encoded with byte + // sequences of different lengths, only the shortest encoding sequence is + // considered well-formed. Also a well-formed sequence may not be decoded + // into invalid codepoint value (see codepoint_type() for details). We + // will check all that using the Well-Formed UTF-8 Byte Sequences table + // (provided by the Unicode 12.0 Standard) which also takes care of the + // missing UTF-8 sequence bytes. + // + bool valid (false); + + // Return true if a byte value belongs to the specified range. + // + auto belongs = [] (unsigned char c, unsigned char l, unsigned char r) + { + return c >= l && c <= r; + }; + + switch (seq_size_) + { + case 1: valid = true; break; // Well-formed by the definition (see above). + case 2: + { + // [000080 0007FF]: [C2 DF] [80 BF] + // + // Check the first byte and set the second byte range. + // + if (seq_index_ == 0) + { + if ((valid = belongs (b, 0xC2, 0xDF))) + byte2_range_ = make_pair (0x80, 0xBF); + } + else // Check the second byte. + valid = belongs (b, byte2_range_.first, byte2_range_.second); + + break; + } + case 3: + { + // [000800 000FFF]: E0 [A0 BF] [80 BF] + // [001000 00CFFF]: [E1 EC] [80 BF] [80 BF] + // [00D000 00D7FF]: ED [80 9F] [80 BF] ; Excludes surrogates. + // [00E000 00FFFF]: [EE EF] [80 BF] [80 BF] + // + // Check the first byte and set the second byte range. + // + if (seq_index_ == 0) + { + if ((valid = (b == 0xE0))) + byte2_range_ = make_pair (0xA0, 0xBF); + else if ((valid = belongs (b, 0xE1, 0xEC))) + byte2_range_ = make_pair (0x80, 0xBF); + else if ((valid = (b == 0xED))) + byte2_range_ = make_pair (0x80, 0x9F); + else if ((valid = belongs (b, 0xEE, 0xEF))) + byte2_range_ = make_pair (0x80, 0xBF); + } + else if (seq_index_ == 1) // Check the second byte. + valid = belongs (b, byte2_range_.first, byte2_range_.second); + else // Check the third byte. + valid = belongs (b, 0x80, 0xBF); + + break; + } + case 4: + { + // [010000 03FFFF]: F0 [90 BF] [80 BF] [80 BF] + // [040000 0FFFFF]: [F1 F3] [80 BF] [80 BF] [80 BF] + // [100000 10FFFF]: F4 [80 8F] [80 BF] [80 BF] + // + // Check the first byte and set the second byte range. + // + if (seq_index_ == 0) + { + if ((valid = (b == 0xF0))) + byte2_range_ = make_pair (0x90, 0xBF); + else if ((valid = belongs (b, 0xF1, 0xF3))) + byte2_range_ = make_pair (0x80, 0xBF); + else if ((valid = (b == 0xF4))) + byte2_range_ = make_pair (0x80, 0x8F); + } + else if (seq_index_ == 1) // Check the second byte. + valid = belongs (b, byte2_range_.first, byte2_range_.second); + else // Check the third and forth bytes. + valid = belongs (b, 0x80, 0xBF); + + break; + } + } + + // Bail out if the current UTF-8 sequence byte is invalid. + // + if (!valid) + { + // We could probably distinguish "surrogate" and "exceed max value" from + // other ill-formedness cases (amend the well-formedness table, keep + // decoding the sequence, and test the codepoint in the end) and produce + // more specific error messages, but this doesn't seem worth the + // trouble. + // + if (what != nullptr) + *what = byte_error (); + + return make_pair (false, false); // Invalid byte. + } + + // "Append" the sequence byte's 6 rightmost bits to the resulting + // codepoint value, unless this is the first byte (which value is already + // taken into account; see above). + // + if (seq_index_ != 0) + codepoint_ = (codepoint_ << 6) | (b & 0x3F); + + // If we didn't get to the end of the UTF-8 sequence, then we are done + // with this byte. + // + if (++seq_index_ != seq_size_) + return make_pair (true, false); // Valid byte. + + // Prepare for the next UTF-8 sequence validation, regardless of the + // decoded codepoint validity. + // + seq_index_ = 0; + + // Check the decoded codepoint, unless any codepoint type is allowed. + // + // Note that the well-formedness sequence check guarantees that we decoded + // a valid Unicode codepoint (see above). + // + if (types_ == codepoint_types::any) + return make_pair (true, true); // Valid codepoint. + + // Check if the decoded codepoint is whitelisted. + // + using traits = u32string::traits_type; + + if (whitelist_ != nullptr && + traits::find (whitelist_, traits::length (whitelist_), codepoint_) != + nullptr) + return make_pair (true, true); // Valid codepoint. + + // Now check if the codepoint type matches the specified set. Note: also + // covers the `types_ == codepoint_types::none` case. + // + codepoint_types t (codepoint_type (codepoint_)); + + if ((t & types_) != codepoint_types::none) + return make_pair (true, true); // Valid codepoint. + + if (what != nullptr) + *what = "invalid Unicode codepoint (" + to_string (t) + ")"; + + return make_pair (false, true); // Invalid codepoint. + } + + inline std::pair<bool, bool> utf8_validator:: + recover (char c) + { + // We are recovered if the character can be interpreted as a sequence + // leading byte. + // + // As an optimization, bail out if the byte is a sequence trailing byte + // (10xxxxxx). + // + if ((c & 0xC0) == 0x80) + return std::make_pair (false, false); // Invalid byte. + + seq_index_ = 0; + return validate (c); + } + + inline char32_t utf8_validator:: + codepoint () const + { + return codepoint_; + } +} diff --git a/libbutl/utf8.mxx b/libbutl/utf8.mxx new file mode 100644 index 0000000..15e8ded --- /dev/null +++ b/libbutl/utf8.mxx @@ -0,0 +1,130 @@ +// file : libbutl/utf8.mxx -*- C++ -*- +// license : MIT; see accompanying LICENSE file + +#ifndef __cpp_modules_ts +#pragma once +#endif + +// C includes. + +#ifndef __cpp_lib_modules_ts +#include <string> +#include <cstdint> // uint8_t +#include <utility> // pair +#endif + +// Other includes. + +#ifdef __cpp_modules_ts +export module butl.utf8; +#ifdef __cpp_lib_modules_ts +import std.core; +#endif +import butl.unicode; +#else +#include <libbutl/unicode.mxx> +#endif + +#include <libbutl/export.hxx> + +LIBBUTL_MODEXPORT namespace butl +{ + // Here and below we will refer to bytes that encode a singe Unicode + // codepoint as "UTF-8 byte sequence" ("UTF-8 sequence" or "byte sequence" + // for short) and a sequence of such sequences as "UTF-8 encoded byte + // string" ("byte string" for short). + // + + // Validate a UTF-8 encoded byte string one byte at a time. Optionally, also + // validate that its decoded codepoints belong to the specified types or + // codepoint whitelist. + // + class utf8_validator + { + public: + // Note: use whitelist via shallow copy. + // + explicit + utf8_validator (codepoint_types = codepoint_types::any, + const char32_t* whitelist = nullptr); + + // Validate the next byte returning true if it is valid (first) and + // whether it is the last byte of a codepoint (second). The {false, true} + // result indicates a byte sequence decoded into a codepoint of undesired + // type rather than an invalid byte that happens to be the last in the + // sequence (and may well be a valid starting byte of the next sequence). + // + // Note that in case the byte is invalid, calling this function again + // without recovery is illegal. + // + std::pair<bool, bool> + validate (char); + + // As above but in case of an invalid byte also return the description of + // why it is invalid. + // + // Note that the description only contains the reason why the specified + // byte is not part of a valid UTF-8 sequence or the desired codepoint + // type, for example: + // + // "invalid UTF-8 sequence first byte (0xB0)" + // "invalid Unicode codepoint (reserved)" + // + // It can be used to form complete diagnostics along these lines: + // + // cerr << "invalid manifest value " << name << ": " << what << endl; + // + std::pair<bool, bool> + validate (char, std::string& what); + + // As above but decide whether the description is needed at runtime (what + // may be NULL). + // + std::pair<bool, bool> + validate (char, std::string* what); + + // Recover from an invalid byte. + // + // This function must be called with the first invalid and then subsequent + // bytes until it signals that the specified byte is valid. Note that it + // shall not be called if the sequence is decoded into a codepoint of an + // undesired type. + // + // Note also that a byte being invalid in the middle of a UTF-8 sequence + // may be valid as a first byte of the next sequence. + // + std::pair<bool, bool> + recover (char); + + // Return the codepoint of the last byte sequence. + // + // This function can only be legally called after validate() or recover() + // signal that the preceding byte is valid and last. + // + char32_t + codepoint () const; + + private: + codepoint_types types_; + const char32_t* whitelist_; + + // State machine. + // + uint8_t seq_size_; // [1 4]; calculated at the first byte validation. + uint8_t seq_index_ = 0; // [0 3] + + // Last byte sequence decoded codepoint (built incrementally). + // + char32_t codepoint_; + + // The byte range a valid UTF-8 sequence second byte must belong to as + // calculated during the first byte validation. + // + // Note that the subsequent (third and forth) bytes must belong to the + // [80 BF] range regardless to the previous bytes. + // + std::pair<unsigned char, unsigned char> byte2_range_; + }; +} + +#include <libbutl/utf8.ixx> diff --git a/libbutl/utility.cxx b/libbutl/utility.cxx index ce78295..d6a21c6 100644 --- a/libbutl/utility.cxx +++ b/libbutl/utility.cxx @@ -35,6 +35,9 @@ import std.io; #endif #endif +import butl.utf8; +#else +#include <libbutl/utf8.mxx> #endif namespace butl @@ -191,6 +194,135 @@ namespace butl } void + to_utf8 (string& s, char repl, codepoint_types ts, const char32_t* wl) + { + using iterator = string::iterator; + + utf8_validator val (ts, wl); + + iterator i (s.begin ()); // Source current position. + iterator e (s.end ()); // Source end position. + iterator d (i); // Destination current position. + iterator b (d); // Begin of the current destination sequence. + + // Replace the current byte and prepare for the next sequence. + // + auto replace_byte = [&d, &b, repl] () + { + *d++ = repl; + b = d; + }; + + // Replace bytes of the current sequence excluding the current byte and + // prepare for the next sequence. + // + auto replace_sequence = [&d, &b, repl] () + { + for (; b != d; ++b) + *b = repl; + }; + + // Replace sequence bytes with a single replacement byte and prepare for + // the next sequence. + // + auto replace_codepoint = [&d, &b, &replace_byte] () + { + d = b; // Rewind to the beginning of the sequence. + replace_byte (); + }; + + // Iterate over the byte string appending valid bytes, replacing invalid + // bytes/codepoints, and recovering after invalid bytes. + // + for (; i != e; ++i) + { + char c (*i); + pair<bool, bool> v (val.validate (c)); + + // Append a valid byte and prepare for the next sequence if the sequence + // end is reached. + // + auto append_byte = [&d, &b, &v, &c] () + { + *d++ = c; + + if (v.second) // Sequence last byte? + b = d; + }; + + // If this is a valid byte/codepoint, then append the byte and proceed + // to the next string byte. + // + if (v.first) + { + append_byte (); + continue; + } + + // If this is an invalid codepoint, then replace the sequence with a + // single replacement character and proceed to the next byte sequence + // (no recovery is necessary). + // + if (v.second) + { + replace_codepoint (); + continue; + } + + // Now, given this is an invalid byte, replace the current sequence + // bytes and recover. + // + replace_sequence (); + + // Stay in the recovery cycle until a valid byte is encountered. Note + // that we start from where we left off, not from the next byte (see + // utf8_validator::recover() for details). + // + for (; i != e; ++i) + { + c = *i; + v = val.recover (c); + + // End the recovery cycle for a valid byte. + // + if (v.first) + { + append_byte (); + break; + } + + // End the recovery cycle for a decoded but invalid (ASCII-range) + // codepoint. + // + if (v.second) + { + replace_codepoint (); + break; + } + + replace_byte (); + } + + // Bail out if we reached the end of the byte string. Note that while we + // failed to recover (otherwise i != e), all the bytes are already + // replaced. + // + if (i == e) + break; + } + + // If the last byte sequence is incomplete, then replace its bytes. + // + if (b != d) + replace_sequence (); + + // Shrink the byte string if we replaced any invalid codepoints. + // + if (d != e) + s.resize (d - s.begin ()); + } + + void setenv (const string& name, const string& value) { #ifndef _WIN32 diff --git a/libbutl/utility.ixx b/libbutl/utility.ixx index c5fdbac..27ef7fb 100644 --- a/libbutl/utility.ixx +++ b/libbutl/utility.ixx @@ -2,8 +2,11 @@ // license : MIT; see accompanying LICENSE file #ifndef __cpp_lib_modules_ts -#include <cstdlib> // getenv() -#include <algorithm> +#include <cctype> // toupper(), tolower(), is*() +#include <cwctype> // isw*() +#include <cstdlib> // getenv() +#include <algorithm> // for_each() +#include <stdexcept> // invalid_argument #endif namespace butl @@ -216,44 +219,84 @@ namespace butl return sanitize_identifier (std::string (s)); } - inline codepoint_types - operator&= (codepoint_types& x, codepoint_types y) + inline bool + eof (std::istream& is) { - return x = static_cast<codepoint_types> ( - static_cast<std::uint16_t> (x) & - static_cast<std::uint16_t> (y)); + if (!is.fail ()) + return false; + + if (is.eof ()) + return true; + + throw std::istream::failure (""); } - inline codepoint_types - operator|= (codepoint_types& x, codepoint_types y) + inline optional<std::size_t> + utf8_length_impl (const std::string& s, + std::string* what, + codepoint_types ts, + const char32_t* wl) { - return x = static_cast<codepoint_types> ( - static_cast<std::uint16_t> (x) | - static_cast<std::uint16_t> (y)); + using namespace std; + + // Optimize for an empty string. + // + if (s.empty ()) + return 0; + + size_t r (0); + pair<bool, bool> v; + utf8_validator val (ts, wl); + + for (char c: s) + { + v = val.validate (c, what); + + if (!v.first) // Invalid byte? + return nullopt; + + if (v.second) // Last byte in the sequence? + ++r; + } + + // Make sure that the last UTF-8 sequence is complete. + // + if (!v.second) + { + if (what != nullptr) + *what = "incomplete UTF-8 sequence"; + + return nullopt; + } + + return r; } - inline codepoint_types - operator& (codepoint_types x, codepoint_types y) + inline std::size_t + utf8_length (const std::string& s, codepoint_types ts, const char32_t* wl) { - return x &= y; + using namespace std; + + string what; + if (optional<size_t> r = utf8_length_impl (s, &what, ts, wl)) + return *r; + + throw invalid_argument (what); } - inline codepoint_types - operator| (codepoint_types x, codepoint_types y) + inline bool + utf8 (const std::string& s, + std::string& what, + codepoint_types ts, + const char32_t* wl) { - return x |= y; + return utf8_length_impl (s, &what, ts, wl).has_value (); } inline bool - eof (std::istream& is) + utf8 (const std::string& s, codepoint_types ts, const char32_t* wl) { - if (!is.fail ()) - return false; - - if (is.eof ()) - return true; - - throw std::istream::failure (""); + return utf8_length_impl (s, nullptr, ts, wl).has_value (); } inline optional<std::string> diff --git a/libbutl/utility.mxx b/libbutl/utility.mxx index 71c2860..b84e731 100644 --- a/libbutl/utility.mxx +++ b/libbutl/utility.mxx @@ -16,13 +16,10 @@ #include <iosfwd> // ostream #include <istream> #include <cstddef> // size_t -#include <utility> // move(), forward() +#include <utility> // move(), forward(), pair #include <cstring> // strcmp(), strlen() #include <exception> // exception, uncaught_exception[s]() //#include <functional> // hash - -#include <cctype> // toupper(), tolower(), is*() -#include <cwctype> // isw*() #endif #include <libbutl/ft/lang.hxx> // thread_local @@ -34,8 +31,12 @@ export module butl.utility; import std.core; import std.io; #endif +import butl.utf8; +import butl.unicode; import butl.optional; #else +#include <libbutl/utf8.mxx> +#include <libbutl/unicode.mxx> #include <libbutl/optional.mxx> #endif @@ -194,41 +195,44 @@ LIBBUTL_MODEXPORT namespace butl std::string sanitize_identifier (std::string&&); std::string sanitize_identifier (const std::string&); - // Return true if the string is a valid UTF-8 encoded byte sequence and, - // optionally, its decoded codepoints belong to the specified types or to - // the codepoint whitelist. - // - // Note that the Unicode Standard considers a UTF-8 byte sequence decoded - // into a codepoint of the surrogate type as invalid. Thus, the surrogate - // type may not be specified. + // Return true if the string is a valid UTF-8 encoded byte string and, + // optionally, its decoded codepoints belong to the specified types or + // codepoint whitelist. // - enum class codepoint_types: std::uint16_t - { - // Useful to only allow the whitelisted codepoints or when building the - // type set incrementally. - // - none = 0x00, - - graphic = 0x01, // L(etter), M(ark), N(number), P(uncturation), - // S(symbol), Zs(separator, space) - format = 0x02, - control = 0x04, - private_use = 0x08, - non_character = 0x10, - reserved = 0x20, - - any = 0x3f - }; + bool + utf8 (const std::string&, + codepoint_types = codepoint_types::any, + const char32_t* whitelist = nullptr); - LIBBUTL_SYMEXPORT bool + // As above but in case of an invalid sequence also return the description + // of why it is invalid. + // + bool utf8 (const std::string&, + std::string& what, codepoint_types = codepoint_types::any, const char32_t* whitelist = nullptr); - codepoint_types operator& (codepoint_types, codepoint_types); - codepoint_types operator| (codepoint_types, codepoint_types); - codepoint_types operator&= (codepoint_types&, codepoint_types); - codepoint_types operator|= (codepoint_types&, codepoint_types); + // Return UTF-8 byte string length in codepoints. Throw + // std::invalid_argument if this is not a valid UTF-8. + // + std::size_t + utf8_length (const std::string&, + codepoint_types = codepoint_types::any, + const char32_t* whitelist = nullptr); + + // Fixup the specified string (in place) to be valid UTF-8 replacing invalid + // bytes and codepoints with the specified character, for example, '?'. + // + // Potential future improvements: + // - char32_t replacement (will need UTF-8 encoding) + // - different replacement for bytes and codepoints + // + LIBBUTL_SYMEXPORT void + to_utf8 (std::string&, + char replacement, + codepoint_types = codepoint_types::any, + const char32_t* whitelist = nullptr); // If an input stream is in a failed state, then return true if this is // because of the eof and throw istream::failure otherwise. If the stream diff --git a/tests/manifest-parser/driver.cxx b/tests/manifest-parser/driver.cxx index 57674cb..a34f2b7 100644 --- a/tests/manifest-parser/driver.cxx +++ b/tests/manifest-parser/driver.cxx @@ -40,6 +40,9 @@ namespace butl static bool equal (const optional<pairs>& actual, const optional<pairs>& expected); + static pairs + parse (const char* m, manifest_parser::filter_function f = {}); + // Test manifest as it is represented in the stream, including format // version and end-of-manifest values. // @@ -188,6 +191,41 @@ namespace butl assert (p.first == "" && p.second == "comment"); } + // UTF-8. + // + assert (test (":1\n#\xD0\xB0\n\xD0\xB0y\xD0\xB0:\xD0\xB0z\xD0\xB0", + {{"","1"}, + {"\xD0\xB0y\xD0\xB0", "\xD0\xB0z\xD0\xB0"}, + {"",""}, + {"",""}})); + + assert (fail (":1\n#\xD0\n\xD0\xB0y\xD0\xB0:\xD0\xB0z\xD0\xB0")); + assert (fail (":1\n#\xD0\xB0\n\xB0y\xD0\xB0:\xD0\xB0z\xD0\xB0")); + assert (fail (":1\n#\xD0\xB0\n\xD0y\xD0\xB0:\xD0\xB0z\xD0\xB0")); + assert (fail (":1\n#\xD0\xB0\n\xD0\xB0y\xD0:\xD0\xB0z\xD0\xB0")); + assert (fail (":1\n#\xD0\xB0\n\xD0\xB0y\xD0\xB0:\xD0z\xD0\xB0")); + assert (fail (":1\n#\xD0\xB0\n\xD0\xB0y\xD0\xB0:\xD0\xB0z\xD0")); + assert (fail (":1\r\r\xB0#\n\xD0\xB0y\xD0\xB0:\xD0\xB0z\xD0\xB0")); + assert (fail (":1\r\xD0#\n\xD0\xB0y\xD0\xB0:\xD0\xB0z\xD0\xB0")); + assert (fail (":1\n#\xD0\xB0\n\xD0\xB0y\xD0\xB0:\xD0\xB0z\xD0\xB0\r\xD0")); + + // Test parsing failure for manifest with multi-byte UTF-8 sequences + // (the column is properly reported, etc). + // + try + { + parse (":1\na\xD0\xB0\xD0\xB0\xFE"); + assert (false); + } + catch (const manifest_parsing& e) + { + assert (e.line == 2 && + e.column == 4 && + e.description == + "invalid manifest name: " + "invalid UTF-8 sequence first byte (0xFE)"); + } + // Filtering. // assert (test (":1\na: abc\nb: bca\nc: cab", @@ -281,7 +319,7 @@ namespace butl } static pairs - parse (const char* m, manifest_parser::filter_function f = {}) + parse (const char* m, manifest_parser::filter_function f) { istringstream is (m); is.exceptions (istream::failbit | istream::badbit); diff --git a/tests/manifest-rewriter/driver.cxx b/tests/manifest-rewriter/driver.cxx index fd76929..ec73d81 100644 --- a/tests/manifest-rewriter/driver.cxx +++ b/tests/manifest-rewriter/driver.cxx @@ -90,6 +90,10 @@ namespace butl {{"abc", "xyz"}}) == ":1\n abc: \\\nxyz\n\\"); + assert (edit (":1\n a\xD0\xB0g : b", + {{"a\xD0\xB0g", "xyz"}}) == + ":1\n a\xD0\xB0g : \\\nxyz\n\\"); + // Test editing of manifests that contains CR characters. // assert (edit (":1\r\na: b\r\r\n", {{"a", "xyz"}}) == ":1\r\na: xyz\r\r\n"); diff --git a/tests/manifest-serializer/driver.cxx b/tests/manifest-serializer/driver.cxx index 148a281..c818b4a 100644 --- a/tests/manifest-serializer/driver.cxx +++ b/tests/manifest-serializer/driver.cxx @@ -46,6 +46,7 @@ main () assert (test ({{"#", "x"}}, "# x\n")); assert (test ({{"#", "x"},{"#", "y"},{"#", ""}}, "# x\n# y\n#\n")); assert (fail ({{"",""},{"#", "x"}})); // serialization after eos + assert (fail ({{"#", "\xB0"}})); // invalid UTF-8 sequence // Empty manifest stream. // @@ -89,6 +90,12 @@ main () assert (fail ({{"","1"},{"a b",""}})); assert (fail ({{"","1"},{"a\tb",""}})); assert (fail ({{"","1"},{"a\n",""}})); + assert (fail ({{"","1"},{"a\xB0",""}})); // invalid UTF-8 sequence + + // Invalid value. + // + assert (fail ({{"","1"},{"a","\xB0"}})); // invalid UTF-8 sequence + assert (fail ({{"","1"},{"a","\xD0"}})); // incomplete UTF-8 sequence // Simple value. // @@ -172,11 +179,22 @@ main () "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\\Y\\\n" "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy"); + // Hard break after the UTF-8/delayed hard break. + // + string l6 ("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\xF0\x90\x8C\x82" + "\xF0\x90\x8C\x82yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy"); + + string e6 ("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\xF0\x90\x8C\x82\\\n" + "\xF0\x90\x8C\x82yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy"); + assert (test ({{"","1"},{"a",l1},{"",""},{"",""}}, ": 1\na: " + e1 + "\n")); assert (test ({{"","1"},{"a",l2},{"",""},{"",""}}, ": 1\na: " + e2 + "\n")); assert (test ({{"","1"},{"a",l3},{"",""},{"",""}}, ": 1\na: " + e3 + "\n")); assert (test ({{"","1"},{"a",l4},{"",""},{"",""}}, ": 1\na: " + e4 + "\n")); assert (test ({{"","1"},{"a",l5},{"",""},{"",""}}, ": 1\na: " + e5 + "\n")); + assert (test ({{"","1"},{"a",l6},{"",""},{"",""}}, ": 1\na: " + e6 + "\n")); // Multi-line value. // diff --git a/tests/utf8/driver.cxx b/tests/utf8/driver.cxx index 8480dec..f35e65e 100644 --- a/tests/utf8/driver.cxx +++ b/tests/utf8/driver.cxx @@ -13,8 +13,10 @@ #ifdef __cpp_lib_modules_ts import std.core; #endif +import butl.utf8; import butl.utility; #else +#include <libbutl/utf8.mxx> #include <libbutl/utility.mxx> #endif @@ -24,6 +26,17 @@ using namespace butl; int main () { + // utf8() tests. + // + auto utf8_error = [] (const string& s, + codepoint_types ts = codepoint_types::any, + const char32_t* wl = nullptr) + { + string error; + assert (!utf8 (s, error, ts, wl)); + return error; + }; + // Valid sequences. // // Empty. @@ -43,18 +56,36 @@ main () // Ill-formed sequences. // + // Long sequences. + // + assert (!utf8 ("\xF8")); // 5-byte sequence. + assert (!utf8 ("\xFC")); // 6-byte sequence. + + assert (utf8_error ("\xF8") == "5-byte length UTF-8 sequence"); + assert (utf8_error ("\xFC") == "6-byte length UTF-8 sequence"); + assert (utf8_error ("\xFE") == "invalid UTF-8 sequence first byte (0xFE)"); + // 2-byte sequences. // assert (!utf8 ("\xC1\x80")); // Invalid first byte. assert (!utf8 ("\xD0y")); // Invalid second byte. + assert (utf8_error ("\xC1\x80") == + "invalid UTF-8 sequence first byte (0xC1)"); + + assert (utf8_error ("\xD0y") == + "invalid UTF-8 sequence second byte (0x79 'y')"); + // 3-byte sequences. // assert (!utf8 ("\xE2\x70\x80")); // Invalid second byte. assert (!utf8 ("\xE2\x80\x70")); // Invalid third byte. - assert (!utf8 ("\xED\xA0\x80")); // Min UTF-16 surrogate value. - assert (!utf8 ("\xED\xBF\xBF")); // Max UTF-16 surrogate value. + assert (!utf8 ("\xED\xA0\x80")); // Min UTF-16 surrogate. + assert (!utf8 ("\xED\xBF\xBF")); // Max UTF-16 surrogate. + + assert (utf8_error ("\xE2\x80\x70") == + "invalid UTF-8 sequence third byte (0x70 'p')"); // 4-byte sequences. // @@ -63,9 +94,8 @@ main () assert (!utf8 ("\xF0\x90\x70\x80")); // Invalid third byte. assert (!utf8 ("\xF1\x80\x80\xC0")); // Invalid forth byte. - // Out of the codepoint range (0x10ffff + 1). - // - assert (!utf8 ("\xF4\x90\x80\x80")); + assert (utf8_error ("\xF1\x80\x80\xC0") == + "invalid UTF-8 sequence forth byte (0xC0)"); // Incomplete sequences. // @@ -73,14 +103,25 @@ main () assert (!utf8 ("\xE4\xBA")); // 3-byte sequence. assert (!utf8 ("\xF0\x90\x8C")); // 4-byte sequence. + assert (utf8_error ("\xD0") == "incomplete UTF-8 sequence"); + // Missing sequence leading bytes. // - assert (!utf8 ("\xB0xyz")); // 2-byte sequence. - assert (!utf8 ("\xBA\x8Cxyz")); // 3-byte sequence. - assert (!utf8 ("\x8Cxyz")); // 3-byte sequence. - assert (!utf8 ("\x90\x8C\x82xyz")); // 4-byte sequence. - assert (!utf8 ("\x8C\x82xyz")); // 4-byte sequence. - assert (!utf8 ("\x82xyz")); // 4-byte sequence. + assert (!utf8 ("\xB0xyz")); // 2-byte sequence. + assert (!utf8 ("\xBA\x8C\xD0\xB0yz")); // 3-byte sequence. + assert (!utf8 ("\x8Cxyz")); // 3-byte sequence. + assert (!utf8 ("\x90\x8C\x82xyz")); // 4-byte sequence. + assert (!utf8 ("\x8C\x82xyz")); // 4-byte sequence. + assert (!utf8 ("\x82xyz")); // 4-byte sequence. + + assert (utf8_error ("\xB0") == "invalid UTF-8 sequence first byte (0xB0)"); + + // Above the valid codepoint range (0x10ffff + 1). + // + assert (!utf8 ("\xF4\x90\x80\x80")); + + assert (utf8_error ("\xF4\x90\x80\x80") == + "invalid UTF-8 sequence second byte (0x90)"); // Whitelisting. // @@ -145,6 +186,9 @@ main () assert (!utf8 ("\xF3\xA1\x80\x80", codepoint_types::graphic)); // Reserved. assert (!utf8 ("\xF3\xA0\x81\xBF", codepoint_types::graphic)); // Format. + assert (utf8_error ("\xF3\xA0\x81\xBF", codepoint_types::graphic) == + "invalid Unicode codepoint (format)"); + assert (!utf8 ("\xC2\xAC", codepoint_types::format)); // Graphic. // Private-use & Graphic. @@ -153,4 +197,145 @@ main () codepoint_types::format)); assert (!utf8 ("a", codepoint_types::none)); // None. + + assert (utf8_error ("a", codepoint_types::none) == + "invalid Unicode codepoint (graphic)"); + + // UTF-8 string length. + // + auto invalid_utf8 = [] (string s, codepoint_types ts = codepoint_types::any) + { + try + { + utf8_length (s, ts); + return false; + } + catch (const invalid_argument&) + { + return true; + } + }; + + assert (utf8_length ("") == 0); + assert (utf8_length ("x\xD0\xB0\xE4\xBA\x8C\xF0\x90\x8C\x82y") == 5); + + assert (invalid_utf8 ("\xFE")); // Invalid byte. + assert (invalid_utf8 ("\xD0")); // Incomplete. + assert (invalid_utf8 ("\n", codepoint_types::graphic)); // Invalid codepoint. + + // to_utf8() tests. + // + auto roundtrip = [] (const char* s) + { + string r (s); + to_utf8 (r, '?'); + return r == s; + }; + + auto sanitize = [] (string s, codepoint_types ts = codepoint_types::any) + { + to_utf8 (s, '?', ts); + return s; + }; + + // Empty. + // + assert (roundtrip ("")); + + // 1 code point. + // + assert (roundtrip ("a")); // 1 byte. + assert (roundtrip ("\xD0\xB0")); // 2 bytes. + assert (roundtrip ("\xE4\xBA\x8C")); // 3 bytes. + assert (roundtrip ("\xF0\x90\x8C\x82")); // 4 bytes. + + // Multiple code points. + // + assert (roundtrip ("a\xD0\xB0\xE4\xBA\x8C\xF0\x90\x8C\x82")); + + // Ill-formed sequences. + // + // Long sequence. + // + assert (sanitize ("\xF8") == "?"); // 5-byte sequence. + + // Invalid first byte followed by a second byte which ... + // + assert (sanitize ("\xC1\x80") == "??"); // is a trailing byte. + assert (sanitize ("\xC1y") == "?y"); // starts 1-byte sequence. + assert (sanitize ("\xC1\xD0\xB0") == "?\xD0\xB0"); // starts 2-byte sequence. + assert (sanitize ("\xC1\xFE") == "??"); // is not UTF-8. + + // Invalid second byte which ... + // + assert (sanitize ("\xD0y") == "?y"); // starts 1-byte sequence. + assert (sanitize ("\xD0\xD0\xB0") == "?\xD0\xB0"); // starts 2-byte sequence. + assert (sanitize ("\xD0\xFE") == "??"); // is not UTF-8. + + // Incomplete sequences. + // + assert (sanitize ("\xD0") == "?"); // 2-byte sequence. + assert (sanitize ("y\xD0") == "y?"); // 2-byte sequence. + assert (sanitize ("\xE4\xBA") == "??"); // 3-byte sequence. + assert (sanitize ("\xD0\xD0") == "??"); // 2-byte sequence. + + // Incomplete recovery. + // + assert (sanitize ("\xD0\xFE") == "??"); // 2-byte sequence. + assert (sanitize ("\xD0\xFE\xFE") == "???"); // 2-byte sequence. + + assert (sanitize ("\xF4\x90\x80\x80") == "????"); // Above the range. + assert (sanitize ("\xED\xA0\x80") == "???"); // Min UTF-16 surrogate. + assert (sanitize ("\xED\xBF\xBF") == "???"); // Max UTF-16 surrogate. + + // Invalid codepoints. + // + auto sanitize_g = [&sanitize] (string s) + { + return sanitize (move (s), codepoint_types::graphic); + }; + + assert (sanitize_g ("\xEF\xB7\x90") == "?"); + assert (sanitize_g ("y\xEF\xB7\x90") == "y?"); + assert (sanitize_g ("\xEF\xB7\x90y") == "?y"); + + // Invalid during recovery. + // + assert (sanitize_g ("\xD0\n") == "??"); + assert (sanitize_g ("\xD0\ny") == "??y"); + assert (sanitize_g ("\xD0\xFE\n") == "???"); + + assert (sanitize_g ("\xD0\xEF\xB7\x90") == "??"); + + // utf8_validator::codepoint() tests. + // + { + u32string r; + size_t invalid_codepoints (0); + + string s ("a" + "\xD0\xB0" + "\n" // Control. + "\xE4\xBA\x8C" + "\xEE\x80\x80" // Private-use. + "\xF0\x90\x8C\x82"); + + utf8_validator val (codepoint_types::graphic); + + for (char c: s) + { + pair<bool, bool> v (val.validate (c)); + + if (v.first) + { + if (v.second) + r.push_back (val.codepoint ()); + } + else + ++invalid_codepoints; + } + + assert (r == U"a\x430\x4E8C\x10302"); + assert (invalid_codepoints == 2); + } } |