dart::Utf8 Class Reference

#include <unicode.h>

Inheritance diagram for dart::Utf8:

Public Types
enum	Type { kLatin1 = 0 , kBMP , kSupplementary }

Static Public Member Functions
static intptr_t	CodeUnitCount (const uint8_t *utf8_array, intptr_t array_len, Type *type)
static bool	IsValid (const uint8_t *utf8_array, intptr_t array_len)
static intptr_t	Length (int32_t ch)
static intptr_t	Length (const String &str)
static intptr_t	Encode (int32_t ch, char *dst)
static intptr_t	Encode (const String &src, char *dst, intptr_t len)
static intptr_t	Decode (const uint8_t *utf8_array, intptr_t array_len, int32_t *ch)
static bool	DecodeToLatin1 (const uint8_t *utf8_array, intptr_t array_len, uint8_t *dst, intptr_t len)
static bool	DecodeToUTF16 (const uint8_t *utf8_array, intptr_t array_len, uint16_t *dst, intptr_t len)
static bool	DecodeToUTF32 (const uint8_t *utf8_array, intptr_t array_len, int32_t *dst, intptr_t len)
static intptr_t	ReportInvalidByte (const uint8_t *utf8_array, intptr_t array_len, intptr_t len)
static bool	DecodeCStringToUTF32 (const char *str, int32_t *dst, intptr_t len)

Static Public Attributes
static constexpr int32_t	kMaxOneByteChar = 0x7F
static constexpr int32_t	kMaxTwoByteChar = 0x7FF
static constexpr int32_t	kMaxThreeByteChar = 0xFFFF
static constexpr int32_t	kMaxFourByteChar = Utf::kMaxCodePoint

Detailed Description

Definition at line 41 of file unicode.h.

Member Enumeration Documentation

Type

enum dart::Utf8::Type

Enumerator
kLatin1
kBMP
kSupplementary

Definition at line 43 of file unicode.h.

            {
    kLatin1 = 0,     // Latin-1 code point [U+0000, U+00FF].
    kBMP,            // Basic Multilingual Plane code point [U+0000, U+FFFF].
    kSupplementary,  // Supplementary code point [U+010000, U+10FFFF].
  };

Member Function Documentation

CodeUnitCount()

intptr_t dart::Utf8::CodeUnitCount ( const uint8_t *  utf8_array, intptr_t  array_len, Type *  type  )

static

Definition at line 46 of file unicode.cc.

                                         {
  intptr_t len = 0;
  Type char_type = kLatin1;
  for (intptr_t i = 0; i < array_len; i++) {
    uint8_t code_unit = utf8_array[i];
    if (!IsTrailByte(code_unit)) {
      ++len;
      if (!IsLatin1SequenceStart(code_unit)) {          // > U+00FF
        if (IsSupplementarySequenceStart(code_unit)) {  // >= U+10000
          char_type = kSupplementary;
          ++len;
        } else if (char_type == kLatin1) {
          char_type = kBMP;
        }
      }
    }
  }
  *type = char_type;
  return len;
}

Decode()

intptr_t dart::Utf8::Decode ( const uint8_t *  utf8_array, intptr_t  array_len, int32_t *  ch  )

static

Definition at line 135 of file unicode.cc.

                                    {
  uint32_t ch = utf8_array[0] & 0xFF;
  intptr_t i = 1;
  if (ch >= 0x80) {
    intptr_t num_trail_bytes = kTrailBytes[ch];
    bool is_malformed = false;
    for (; i < num_trail_bytes; ++i) {
      if (i < array_len) {
        uint8_t code_unit = utf8_array[i];
        is_malformed |= !IsTrailByte(code_unit);
        ch = (ch << 6) + code_unit;
      } else {
        *dst = -1;
        return 0;
      }
    }
    ch -= kMagicBits[num_trail_bytes];
    if (!((is_malformed == false) && (i == num_trail_bytes) &&
          !Utf::IsOutOfRange(ch) && !IsNonShortestForm(ch, i))) {
      *dst = -1;
      return 0;
    }
  }
  *dst = ch;
  return i;
}

DecodeCStringToUTF32()

bool dart::Utf8::DecodeCStringToUTF32 ( const char *  str, int32_t *  dst, intptr_t  len  )

static

Definition at line 266 of file unicode.cc.

                                                                           {
  ASSERT(str != nullptr);
  intptr_t array_len = strlen(str);
  const uint8_t* utf8_array = reinterpret_cast<const uint8_t*>(str);
  return Utf8::DecodeToUTF32(utf8_array, array_len, dst, len);
}

DecodeToLatin1()

bool dart::Utf8::DecodeToLatin1 ( const uint8_t *  utf8_array, intptr_t  array_len, uint8_t *  dst, intptr_t  len  )

static

Definition at line 194 of file unicode.cc.

                                        {
  intptr_t i = 0;
  intptr_t j = 0;
  intptr_t num_bytes;
  for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
    int32_t ch;
    ASSERT(IsLatin1SequenceStart(utf8_array[i]));
    num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
    if (ch == -1) {
      return false;  // Invalid input.
    }
    ASSERT(Utf::IsLatin1(ch));
    dst[j] = ch;
  }
  if ((i < array_len) && (j == len)) {
    return false;  // Output overflow.
  }
  return true;  // Success.
}

DecodeToUTF16()

bool dart::Utf8::DecodeToUTF16 ( const uint8_t *  utf8_array, intptr_t  array_len, uint16_t *  dst, intptr_t  len  )

static

Definition at line 217 of file unicode.cc.

                                       {
  intptr_t i = 0;
  intptr_t j = 0;
  intptr_t num_bytes;
  for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
    int32_t ch;
    bool is_supplementary = IsSupplementarySequenceStart(utf8_array[i]);
    num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
    if (ch == -1) {
      return false;  // Invalid input.
    }
    if (is_supplementary) {
      if (j == (len - 1)) return false;  // Output overflow.
      Utf16::Encode(ch, &dst[j]);
      j = j + 1;
    } else {
      dst[j] = ch;
    }
  }
  if ((i < array_len) && (j == len)) {
    return false;  // Output overflow.
  }
  return true;  // Success.
}

DecodeToUTF32()

bool dart::Utf8::DecodeToUTF32 ( const uint8_t *  utf8_array, intptr_t  array_len, int32_t *  dst, intptr_t  len  )

static

Definition at line 245 of file unicode.cc.

                                       {
  intptr_t i = 0;
  intptr_t j = 0;
  intptr_t num_bytes;
  for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
    int32_t ch;
    num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
    if (ch == -1) {
      return false;  // Invalid input.
    }
    dst[j] = ch;
  }
  if ((i < array_len) && (j == len)) {
    return false;  // Output overflow.
  }
  return true;  // Success.
}

Encode() [1/2]

intptr_t dart::Utf8::Encode ( const String &  src, char *  dst, intptr_t  len  )

static

Definition at line 65 of file unicode.cc.

                                                                {
  uintptr_t array_len = len;
  intptr_t pos = 0;
  ASSERT(static_cast<intptr_t>(array_len) >= Length(src));
  if (src.IsOneByteString()) {
    // For 1-byte strings, all code points < 0x80 have single-byte UTF-8
    // encodings and all >= 0x80 have two-byte encodings.
    NoSafepointScope scope;
    const uintptr_t* data =
        reinterpret_cast<const uintptr_t*>(OneByteString::DataStart(src));
    uintptr_t char_length = src.Length();
    uintptr_t pos = 0;
    ASSERT(kMaxOneByteChar + 1 == 0x80);
    for (uintptr_t i = 0; i < char_length; i += sizeof(uintptr_t)) {
      // Read the input one word at a time and just write it verbatim if it is
      // plain ASCII, as determined by the mask.
      if (i + sizeof(uintptr_t) <= char_length &&
          (*data & kAsciiWordMask) == 0 &&
          pos + sizeof(uintptr_t) <= array_len) {
        StoreUnaligned(reinterpret_cast<uintptr_t*>(dst + pos), *data);
        pos += sizeof(uintptr_t);
      } else {
        // Process up to one word of input that contains non-ASCII Latin1
        // characters.
        const uint8_t* p = reinterpret_cast<const uint8_t*>(data);
        const uint8_t* limit =
            Utils::Minimum(p + sizeof(uintptr_t), p + (char_length - i));
        for (; p < limit; p++) {
          uint8_t c = *p;
          // These calls to Length and Encode get inlined and the cases for 3
          // and 4 byte sequences are removed.
          intptr_t bytes = Length(c);
          if (pos + bytes > array_len) {
            return pos;
          }
          Encode(c, reinterpret_cast<char*>(dst) + pos);
          pos += bytes;
        }
      }
      data++;
    }
  } else {
    // For two-byte strings, which can contain 3 and 4-byte UTF-8 encodings,
    // which can result in surrogate pairs, use the more general code.
    String::CodePointIterator it(src);
    while (it.Next()) {
      int32_t ch = it.Current();
      ASSERT(!Utf::IsOutOfRange(ch));
      if (Utf16::IsSurrogate(ch)) {
        // Encode unpaired surrogates as replacement characters to ensure the
        // output is valid UTF-8. Encoded size is the same (3), so the computed
        // length is still valid.
        ch = Utf::kReplacementChar;
      }
      intptr_t num_bytes = Utf8::Length(ch);
      if (pos + num_bytes > len) {
        break;
      }
      Utf8::Encode(ch, &dst[pos]);
      pos += num_bytes;
    }
  }
  return pos;
}

Encode() [2/2]

intptr_t dart::Utf8::Encode ( int32_t  ch, char *  dst  )

static

Definition at line 110 of file unicode.cc.

                                           {
  constexpr int kMask = ~(1 << 6);
  if (ch <= kMaxOneByteChar) {
    dst[0] = ch;
    return 1;
  }
  if (ch <= kMaxTwoByteChar) {
    dst[0] = 0xC0 | (ch >> 6);
    dst[1] = 0x80 | (ch & kMask);
    return 2;
  }
  if (ch <= kMaxThreeByteChar) {
    dst[0] = 0xE0 | (ch >> 12);
    dst[1] = 0x80 | ((ch >> 6) & kMask);
    dst[2] = 0x80 | (ch & kMask);
    return 3;
  }
  ASSERT(ch <= kMaxFourByteChar);
  dst[0] = 0xF0 | (ch >> 18);
  dst[1] = 0x80 | ((ch >> 12) & kMask);
  dst[2] = 0x80 | ((ch >> 6) & kMask);
  dst[3] = 0x80 | (ch & kMask);
  return 4;
}

IsValid()

bool dart::Utf8::IsValid ( const uint8_t *  utf8_array, intptr_t  array_len  )

static

Definition at line 70 of file unicode.cc.

                                                                {
  intptr_t i = 0;
  while (i < array_len) {
    uint32_t ch = utf8_array[i] & 0xFF;
    intptr_t j = 1;
    if (ch >= 0x80) {
      int8_t num_trail_bytes = kTrailBytes[ch];
      bool is_malformed = false;
      for (; j < num_trail_bytes; ++j) {
        if ((i + j) < array_len) {
          uint8_t code_unit = utf8_array[i + j];
          is_malformed |= !IsTrailByte(code_unit);
          ch = (ch << 6) + code_unit;
        } else {
          return false;
        }
      }
      ch -= kMagicBits[num_trail_bytes];
      if (!((is_malformed == false) && (j == num_trail_bytes) &&
            !Utf::IsOutOfRange(ch) && !IsNonShortestForm(ch, j))) {
        return false;
      }
    }
    i += j;
  }
  return true;
}

Length() [1/2]

intptr_t dart::Utf8::Length ( const String &  str )

static

Definition at line 21 of file unicode.cc.

                                       {
  if (str.IsOneByteString()) {
    // For 1-byte strings, all code points < 0x80 have single-byte UTF-8
    // encodings and all >= 0x80 have two-byte encodings.  To get the length,
    // start with the number of code points and add the number of high bits in
    // the bytes.
    uintptr_t char_length = str.Length();
    uintptr_t length = char_length;
    NoSafepointScope no_safepoint;
    const uintptr_t* data =
        reinterpret_cast<const uintptr_t*>(OneByteString::DataStart(str));
    uintptr_t i;
    for (i = sizeof(uintptr_t); i <= char_length; i += sizeof(uintptr_t)) {
      uintptr_t chunk = *data++;
      chunk &= kAsciiWordMask;
      if (chunk != 0) {
// Shuffle the bits until we have a count of bits in the low nibble.
#if defined(ARCH_IS_64_BIT)
        chunk += chunk >> 32;
#endif
        chunk += chunk >> 16;
        chunk += chunk >> 8;
        length += (chunk >> 7) & 0xf;
      }
    }
    // Take care of the tail of the string, the last length % wordsize chars.
    i -= sizeof(uintptr_t);
    for (; i < char_length; i++) {
      if (str.CharAt(i) > kMaxOneByteChar) length++;
    }
    return length;
  }
 
  // Slow case for 2-byte strings that handles surrogate pairs and longer UTF-8
  // encodings.
  intptr_t length = 0;
  String::CodePointIterator it(str);
  while (it.Next()) {
    int32_t ch = it.Current();
    length += Utf8::Length(ch);
  }
  return length;
}

Length() [2/2]

intptr_t dart::Utf8::Length ( int32_t  ch )

static

Definition at line 98 of file unicode.cc.

                                {
  if (ch <= kMaxOneByteChar) {
    return 1;
  } else if (ch <= kMaxTwoByteChar) {
    return 2;
  } else if (ch <= kMaxThreeByteChar) {
    return 3;
  }
  ASSERT(ch <= kMaxFourByteChar);
  return 4;
}

ReportInvalidByte()

intptr_t dart::Utf8::ReportInvalidByte ( const uint8_t *  utf8_array, intptr_t  array_len, intptr_t  len  )

static

Definition at line 163 of file unicode.cc.

                                               {
  intptr_t i = 0;
  intptr_t j = 0;
  intptr_t num_bytes;
  for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
    int32_t ch;
    bool is_supplementary = IsSupplementarySequenceStart(utf8_array[i]);
    num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
    if (ch == -1) {
      break;  // Invalid input.
    }
    if (is_supplementary) {
      j = j + 1;
    }
  }
#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
  // Remain silent while libFuzzer is active, since
  // the output only slows down the in-process fuzzing.
#else
  Syslog::PrintErr("Invalid UTF8 sequence encountered, ");
  for (intptr_t idx = 0; idx < 10 && (i + idx) < array_len; idx++) {
    Syslog::PrintErr("(Error Code: %X + idx: %" Pd " )", utf8_array[idx + i],
                     (idx + i));
  }
  Syslog::PrintErr("\n");
#endif
  return i;
}

Member Data Documentation

kMaxFourByteChar

constexpr int32_t dart::Utf8::kMaxFourByteChar = Utf::kMaxCodePoint

staticconstexpr

Definition at line 89 of file unicode.h.

kMaxOneByteChar

constexpr int32_t dart::Utf8::kMaxOneByteChar = 0x7F

staticconstexpr

Definition at line 86 of file unicode.h.

kMaxThreeByteChar

constexpr int32_t dart::Utf8::kMaxThreeByteChar = 0xFFFF

staticconstexpr

Definition at line 88 of file unicode.h.

kMaxTwoByteChar

constexpr int32_t dart::Utf8::kMaxTwoByteChar = 0x7FF

staticconstexpr

Definition at line 87 of file unicode.h.

---

The documentation for this class was generated from the following files:

• third_party/dart-lang/sdk/runtime/platform/unicode.h
• third_party/dart-lang/sdk/runtime/platform/unicode.cc
• third_party/dart-lang/sdk/runtime/vm/unicode.cc