src/dged/utf8.c


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

#include "utf8.h"

#include <stdio.h>

bool utf8_byte_is_unicode_start(uint8_t byte) { return (byte & 0xc0) == 0xc0; }
bool utf8_byte_is_unicode_continuation(uint8_t byte) {
  return utf8_byte_is_unicode(byte) && !utf8_byte_is_unicode_start(byte);
}
bool utf8_byte_is_unicode(uint8_t byte) { return (byte & 0x80) != 0x0; }
bool utf8_byte_is_ascii(uint8_t byte) { return !utf8_byte_is_unicode(byte); }

uint32_t utf8_nbytes_in_char(uint8_t byte) {
  // length of char is the number of leading ones
  // flip it and count number of leading zeros
  uint8_t invb = ~byte;
  return __builtin_clz((uint32_t)invb) - 24;
}

// TODO: grapheme clusters, this returns the number of unicode code points
uint32_t utf8_nchars(uint8_t *bytes, uint32_t nbytes) {
  uint32_t nchars = 0;
  uint32_t expected = 0;
  for (uint32_t bi = 0; bi < nbytes; ++bi) {
    uint8_t byte = bytes[bi];
    if (utf8_byte_is_unicode(byte)) {
      if (utf8_byte_is_unicode_start(byte)) {
        expected = utf8_nbytes_in_char(byte) - 1;
      } else { // continuation byte
        --expected;
        if (expected == 0) {
          ++nchars;
        }
      }
    } else { // ascii
      ++nchars;
    }
  }
  return nchars;
}

// TODO: grapheme clusters, this uses the number of unicode code points
uint32_t utf8_nbytes(uint8_t *bytes, uint32_t nbytes, uint32_t nchars) {

  uint32_t bi = 0;
  uint32_t chars = 0;
  uint32_t expected = 0;

  while (chars < nchars && bi < nbytes) {
    uint8_t byte = bytes[bi];
    if (utf8_byte_is_unicode(byte)) {
      if (utf8_byte_is_unicode_start(byte)) {
        expected = utf8_nbytes_in_char(byte) - 1;
      } else { // continuation char
        --expected;
        if (expected == 0) {
          ++chars;
        }
      }
    } else { // ascii
      ++chars;
    }

    ++bi;
  }

  return bi;
}