From: Theodore Ts'o Date: Mon, 29 Apr 2019 13:15:48 +0000 (-0400) Subject: libext2fs: remove utf8_* namespace contamination X-Git-Tag: v1.45.1-rc1~11 X-Git-Url: https://git.whamcloud.com/gitweb?a=commitdiff_plain;h=3f85a4c9a811c261b42764a2ef6dbfa10bc1dfdb;p=tools%2Fe2fsprogs.git libext2fs: remove utf8_* namespace contamination Merge nls_utf8-norm.c and nls_utf8.c. This also allows us to comment out functions which we don't actually need for e2fsprogs. Also fix some gcc -Wall complaints, including one which would have caused utf8_casefold() to misbehave (this was fixed in the kernel, but not carried back to e2fsprogs). Signed-off-by: Theodore Ts'o --- diff --git a/lib/ext2fs/Makefile.in b/lib/ext2fs/Makefile.in index 622b734..875e179 100644 --- a/lib/ext2fs/Makefile.in +++ b/lib/ext2fs/Makefile.in @@ -20,9 +20,6 @@ COMPILE_ET= _ET_DIR_OVERRIDE=$(srcdir)/../et ../et/compile_et @TEST_IO_CMT@TEST_IO_LIB_OBJS = test_io.o @IMAGER_CMT@E2IMAGE_LIB_OBJS = imager.o -NLS_OBJS=nls_utf8-norm.o nls_utf8.o -NLS_SRCS=$(srcdir)/nls_utf8-norm.c $(srcdir)/nls_utf8.c - DEBUG_OBJS= debug_cmds.o extent_cmds.o tst_cmds.o debugfs.o util.o \ ncheck.o icheck.o ls.o lsdel.o dump.o set_fields.o logdump.o \ htree.o unused.o e2freefrag.o filefrag.o extent_inode.o zap.o \ @@ -115,6 +112,7 @@ OBJS= $(DEBUGFS_LIB_OBJS) $(RESIZE_LIB_OBJS) $(E2IMAGE_LIB_OBJS) \ namei.o \ native.o \ newdir.o \ + nls_utf8.o \ openfs.o \ progress.o \ punch.o \ @@ -133,8 +131,7 @@ OBJS= $(DEBUGFS_LIB_OBJS) $(RESIZE_LIB_OBJS) $(E2IMAGE_LIB_OBJS) \ unlink.o \ valid_blk.o \ version.o \ - rbtree.o \ - $(NLS_OBJS) + rbtree.o SRCS= ext2_err.c \ $(srcdir)/alloc.c \ @@ -198,6 +195,7 @@ SRCS= ext2_err.c \ $(srcdir)/namei.c \ $(srcdir)/native.c \ $(srcdir)/newdir.c \ + $(srcdir)/nls_utf8.c \ $(srcdir)/openfs.c \ $(srcdir)/progress.c \ $(srcdir)/punch.c \ @@ -226,8 +224,7 @@ SRCS= ext2_err.c \ $(srcdir)/write_bb_file.c \ $(srcdir)/rbtree.c \ $(srcdir)/tst_libext2fs.c \ - $(DEBUG_SRCS) \ - $(NLS_SRCS) + $(DEBUG_SRCS) HFILES= bitops.h ext2fs.h ext2_io.h ext2_fs.h ext2_ext_attr.h ext3_extents.h \ tdb.h qcow2.h hashmap.h diff --git a/lib/ext2fs/nls_utf8-norm.c b/lib/ext2fs/nls_utf8-norm.c deleted file mode 100644 index a5ac7c3..0000000 --- a/lib/ext2fs/nls_utf8-norm.c +++ /dev/null @@ -1,795 +0,0 @@ -/* - * Copyright (c) 2014 SGI. - * All rights reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it would be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - */ - -/* - * This code is adapted from the Linux Kernel. We have a - * userspace version here such that the hashes will match that - * implementation. - */ - -#include "utf8n.h" - -struct utf8data { - unsigned int maxage; - unsigned int offset; -}; - -#define __INCLUDED_FROM_UTF8NORM_C__ -#include "utf8data.h" -#undef __INCLUDED_FROM_UTF8NORM_C__ - -#define ARRAY_SIZE(array) \ - (sizeof(array) / sizeof(array[0])) - -int utf8version_is_supported(uint8_t maj, uint8_t min, uint8_t rev) -{ - int i = ARRAY_SIZE(utf8agetab) - 1; - unsigned int sb_utf8version = UNICODE_AGE(maj, min, rev); - - while (i >= 0 && utf8agetab[i] != 0) { - if (sb_utf8version == utf8agetab[i]) - return 1; - i--; - } - return 0; -} - -int utf8version_latest(void) -{ - return utf8vers; -} - -/* - * UTF-8 valid ranges. - * - * The UTF-8 encoding spreads the bits of a 32bit word over several - * bytes. This table gives the ranges that can be held and how they'd - * be represented. - * - * 0x00000000 0x0000007F: 0xxxxxxx - * 0x00000000 0x000007FF: 110xxxxx 10xxxxxx - * 0x00000000 0x0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx - * 0x00000000 0x001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - * 0x00000000 0x03FFFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx - * 0x00000000 0x7FFFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx - * - * There is an additional requirement on UTF-8, in that only the - * shortest representation of a 32bit value is to be used. A decoder - * must not decode sequences that do not satisfy this requirement. - * Thus the allowed ranges have a lower bound. - * - * 0x00000000 0x0000007F: 0xxxxxxx - * 0x00000080 0x000007FF: 110xxxxx 10xxxxxx - * 0x00000800 0x0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx - * 0x00010000 0x001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - * 0x00200000 0x03FFFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx - * 0x04000000 0x7FFFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx - * - * Actual unicode characters are limited to the range 0x0 - 0x10FFFF, - * 17 planes of 65536 values. This limits the sequences actually seen - * even more, to just the following. - * - * 0 - 0x7F: 0 - 0x7F - * 0x80 - 0x7FF: 0xC2 0x80 - 0xDF 0xBF - * 0x800 - 0xFFFF: 0xE0 0xA0 0x80 - 0xEF 0xBF 0xBF - * 0x10000 - 0x10FFFF: 0xF0 0x90 0x80 0x80 - 0xF4 0x8F 0xBF 0xBF - * - * Within those ranges the surrogates 0xD800 - 0xDFFF are not allowed. - * - * Note that the longest sequence seen with valid usage is 4 bytes, - * the same a single UTF-32 character. This makes the UTF-8 - * representation of Unicode strictly smaller than UTF-32. - * - * The shortest sequence requirement was introduced by: - * Corrigendum #1: UTF-8 Shortest Form - * It can be found here: - * http://www.unicode.org/versions/corrigendum1.html - * - */ - -/* - * Return the number of bytes used by the current UTF-8 sequence. - * Assumes the input points to the first byte of a valid UTF-8 - * sequence. - */ -static inline int utf8clen(const char *s) -{ - unsigned char c = *s; - - return 1 + (c >= 0xC0) + (c >= 0xE0) + (c >= 0xF0); -} - -/* - * Decode a 3-byte UTF-8 sequence. - */ -static unsigned int -utf8decode3(const char *str) -{ - unsigned int uc; - - uc = *str++ & 0x0F; - uc <<= 6; - uc |= *str++ & 0x3F; - uc <<= 6; - uc |= *str++ & 0x3F; - - return uc; -} - -/* - * Encode a 3-byte UTF-8 sequence. - */ -static int -utf8encode3(char *str, unsigned int val) -{ - str[2] = (val & 0x3F) | 0x80; - val >>= 6; - str[1] = (val & 0x3F) | 0x80; - val >>= 6; - str[0] = val | 0xE0; - - return 3; -} - -/* - * utf8trie_t - * - * A compact binary tree, used to decode UTF-8 characters. - * - * Internal nodes are one byte for the node itself, and up to three - * bytes for an offset into the tree. The first byte contains the - * following information: - * NEXTBYTE - flag - advance to next byte if set - * BITNUM - 3 bit field - the bit number to tested - * OFFLEN - 2 bit field - number of bytes in the offset - * if offlen == 0 (non-branching node) - * RIGHTPATH - 1 bit field - set if the following node is for the - * right-hand path (tested bit is set) - * TRIENODE - 1 bit field - set if the following node is an internal - * node, otherwise it is a leaf node - * if offlen != 0 (branching node) - * LEFTNODE - 1 bit field - set if the left-hand node is internal - * RIGHTNODE - 1 bit field - set if the right-hand node is internal - * - * Due to the way utf8 works, there cannot be branching nodes with - * NEXTBYTE set, and moreover those nodes always have a righthand - * descendant. - */ -typedef const unsigned char utf8trie_t; -#define BITNUM 0x07 -#define NEXTBYTE 0x08 -#define OFFLEN 0x30 -#define OFFLEN_SHIFT 4 -#define RIGHTPATH 0x40 -#define TRIENODE 0x80 -#define RIGHTNODE 0x40 -#define LEFTNODE 0x80 - -/* - * utf8leaf_t - * - * The leaves of the trie are embedded in the trie, and so the same - * underlying datatype: unsigned char. - * - * leaf[0]: The unicode version, stored as a generation number that is - * an index into utf8agetab[]. With this we can filter code - * points based on the unicode version in which they were - * defined. The CCC of a non-defined code point is 0. - * leaf[1]: Canonical Combining Class. During normalization, we need - * to do a stable sort into ascending order of all characters - * with a non-zero CCC that occur between two characters with - * a CCC of 0, or at the begin or end of a string. - * The unicode standard guarantees that all CCC values are - * between 0 and 254 inclusive, which leaves 255 available as - * a special value. - * Code points with CCC 0 are known as stoppers. - * leaf[2]: Decomposition. If leaf[1] == 255, then leaf[2] is the - * start of a NUL-terminated string that is the decomposition - * of the character. - * The CCC of a decomposable character is the same as the CCC - * of the first character of its decomposition. - * Some characters decompose as the empty string: these are - * characters with the Default_Ignorable_Code_Point property. - * These do affect normalization, as they all have CCC 0. - * - * The decompositions in the trie have been fully expanded, with the - * exception of Hangul syllables, which are decomposed algorithmically. - * - * Casefolding, if applicable, is also done using decompositions. - * - * The trie is constructed in such a way that leaves exist for all - * UTF-8 sequences that match the criteria from the "UTF-8 valid - * ranges" comment above, and only for those sequences. Therefore a - * lookup in the trie can be used to validate the UTF-8 input. - */ -typedef const unsigned char utf8leaf_t; - -#define LEAF_GEN(LEAF) ((LEAF)[0]) -#define LEAF_CCC(LEAF) ((LEAF)[1]) -#define LEAF_STR(LEAF) ((const char *)((LEAF) + 2)) - -#define MINCCC (0) -#define MAXCCC (254) -#define STOPPER (0) -#define DECOMPOSE (255) - -/* Marker for hangul syllable decomposition. */ -#define HANGUL ((char)(255)) -/* Size of the synthesized leaf used for Hangul syllable decomposition. */ -#define UTF8HANGULLEAF (12) - -/* - * Hangul decomposition (algorithm from Section 3.12 of Unicode 6.3.0) - * - * AC00;;Lo;0;L;;;;;N;;;;; - * D7A3;;Lo;0;L;;;;;N;;;;; - * - * SBase = 0xAC00 - * LBase = 0x1100 - * VBase = 0x1161 - * TBase = 0x11A7 - * LCount = 19 - * VCount = 21 - * TCount = 28 - * NCount = 588 (VCount * TCount) - * SCount = 11172 (LCount * NCount) - * - * Decomposition: - * SIndex = s - SBase - * - * LV (Canonical/Full) - * LIndex = SIndex / NCount - * VIndex = (Sindex % NCount) / TCount - * LPart = LBase + LIndex - * VPart = VBase + VIndex - * - * LVT (Canonical) - * LVIndex = (SIndex / TCount) * TCount - * TIndex = (Sindex % TCount) - * LVPart = SBase + LVIndex - * TPart = TBase + TIndex - * - * LVT (Full) - * LIndex = SIndex / NCount - * VIndex = (Sindex % NCount) / TCount - * TIndex = (Sindex % TCount) - * LPart = LBase + LIndex - * VPart = VBase + VIndex - * if (TIndex == 0) { - * d = - * } else { - * TPart = TBase + TIndex - * d = - * } - */ - -/* Constants */ -#define SB (0xAC00) -#define LB (0x1100) -#define VB (0x1161) -#define TB (0x11A7) -#define LC (19) -#define VC (21) -#define TC (28) -#define NC (VC * TC) -#define SC (LC * NC) - -/* Algorithmic decomposition of hangul syllable. */ -static utf8leaf_t * -utf8hangul(const char *str, unsigned char *hangul) -{ - unsigned int si; - unsigned int li; - unsigned int vi; - unsigned int ti; - unsigned char *h; - - /* Calculate the SI, LI, VI, and TI values. */ - si = utf8decode3(str) - SB; - li = si / NC; - vi = (si % NC) / TC; - ti = si % TC; - - /* Fill in base of leaf. */ - h = hangul; - LEAF_GEN(h) = 2; - LEAF_CCC(h) = DECOMPOSE; - h += 2; - - /* Add LPart, a 3-byte UTF-8 sequence. */ - h += utf8encode3((char *)h, li + LB); - - /* Add VPart, a 3-byte UTF-8 sequence. */ - h += utf8encode3((char *)h, vi + VB); - - /* Add TPart if required, also a 3-byte UTF-8 sequence. */ - if (ti) - h += utf8encode3((char *)h, ti + TB); - - /* Terminate string. */ - h[0] = '\0'; - - return hangul; -} - -/* - * Use trie to scan s, touching at most len bytes. - * Returns the leaf if one exists, NULL otherwise. - * - * A non-NULL return guarantees that the UTF-8 sequence starting at s - * is well-formed and corresponds to a known unicode code point. The - * shorthand for this will be "is valid UTF-8 unicode". - */ -static utf8leaf_t *utf8nlookup(const struct utf8data *data, - unsigned char *hangul, const char *s, size_t len) -{ - utf8trie_t *trie; - int offlen; - int offset; - int mask; - int node; - - if (!data) - return NULL; - if (len == 0) - return NULL; - - trie = utf8data + data->offset; - node = 1; - while (node) { - offlen = (*trie & OFFLEN) >> OFFLEN_SHIFT; - if (*trie & NEXTBYTE) { - if (--len == 0) - return NULL; - s++; - } - mask = 1 << (*trie & BITNUM); - if (*s & mask) { - /* Right leg */ - if (offlen) { - /* Right node at offset of trie */ - node = (*trie & RIGHTNODE); - offset = trie[offlen]; - while (--offlen) { - offset <<= 8; - offset |= trie[offlen]; - } - trie += offset; - } else if (*trie & RIGHTPATH) { - /* Right node after this node */ - node = (*trie & TRIENODE); - trie++; - } else { - /* No right node. */ - return NULL; - } - } else { - /* Left leg */ - if (offlen) { - /* Left node after this node. */ - node = (*trie & LEFTNODE); - trie += offlen + 1; - } else if (*trie & RIGHTPATH) { - /* No left node. */ - return NULL; - } else { - /* Left node after this node */ - node = (*trie & TRIENODE); - trie++; - } - } - } - /* - * Hangul decomposition is done algorithmically. These are the - * codepoints >= 0xAC00 and <= 0xD7A3. Their UTF-8 encoding is - * always 3 bytes long, so s has been advanced twice, and the - * start of the sequence is at s-2. - */ - if (LEAF_CCC(trie) == DECOMPOSE && LEAF_STR(trie)[0] == HANGUL) - trie = utf8hangul(s - 2, hangul); - return trie; -} - -/* - * Use trie to scan s. - * Returns the leaf if one exists, NULL otherwise. - * - * Forwards to utf8nlookup(). - */ -static utf8leaf_t *utf8lookup(const struct utf8data *data, - unsigned char *hangul, const char *s) -{ - return utf8nlookup(data, hangul, s, (size_t)-1); -} - -/* - * Maximum age of any character in s. - * Return -1 if s is not valid UTF-8 unicode. - * Return 0 if only non-assigned code points are used. - */ -int utf8agemax(const struct utf8data *data, const char *s) -{ - utf8leaf_t *leaf; - int age = 0; - int leaf_age; - unsigned char hangul[UTF8HANGULLEAF]; - - if (!data) - return -1; - - while (*s) { - leaf = utf8lookup(data, hangul, s); - if (!leaf) - return -1; - - leaf_age = utf8agetab[LEAF_GEN(leaf)]; - if (leaf_age <= data->maxage && leaf_age > age) - age = leaf_age; - s += utf8clen(s); - } - return age; -} - -/* - * Minimum age of any character in s. - * Return -1 if s is not valid UTF-8 unicode. - * Return 0 if non-assigned code points are used. - */ -int utf8agemin(const struct utf8data *data, const char *s) -{ - utf8leaf_t *leaf; - int age; - int leaf_age; - unsigned char hangul[UTF8HANGULLEAF]; - - if (!data) - return -1; - age = data->maxage; - while (*s) { - leaf = utf8lookup(data, hangul, s); - if (!leaf) - return -1; - leaf_age = utf8agetab[LEAF_GEN(leaf)]; - if (leaf_age <= data->maxage && leaf_age < age) - age = leaf_age; - s += utf8clen(s); - } - return age; -} - -/* - * Maximum age of any character in s, touch at most len bytes. - * Return -1 if s is not valid UTF-8 unicode. - */ -int utf8nagemax(const struct utf8data *data, const char *s, size_t len) -{ - utf8leaf_t *leaf; - int age = 0; - int leaf_age; - unsigned char hangul[UTF8HANGULLEAF]; - - if (!data) - return -1; - - while (len && *s) { - leaf = utf8nlookup(data, hangul, s, len); - if (!leaf) - return -1; - leaf_age = utf8agetab[LEAF_GEN(leaf)]; - if (leaf_age <= data->maxage && leaf_age > age) - age = leaf_age; - len -= utf8clen(s); - s += utf8clen(s); - } - return age; -} - -/* - * Maximum age of any character in s, touch at most len bytes. - * Return -1 if s is not valid UTF-8 unicode. - */ -int utf8nagemin(const struct utf8data *data, const char *s, size_t len) -{ - utf8leaf_t *leaf; - int leaf_age; - int age; - unsigned char hangul[UTF8HANGULLEAF]; - - if (!data) - return -1; - age = data->maxage; - while (len && *s) { - leaf = utf8nlookup(data, hangul, s, len); - if (!leaf) - return -1; - leaf_age = utf8agetab[LEAF_GEN(leaf)]; - if (leaf_age <= data->maxage && leaf_age < age) - age = leaf_age; - len -= utf8clen(s); - s += utf8clen(s); - } - return age; -} - -/* - * Length of the normalization of s. - * Return -1 if s is not valid UTF-8 unicode. - * - * A string of Default_Ignorable_Code_Point has length 0. - */ -ssize_t utf8len(const struct utf8data *data, const char *s) -{ - utf8leaf_t *leaf; - size_t ret = 0; - unsigned char hangul[UTF8HANGULLEAF]; - - if (!data) - return -1; - while (*s) { - leaf = utf8lookup(data, hangul, s); - if (!leaf) - return -1; - if (utf8agetab[LEAF_GEN(leaf)] > data->maxage) - ret += utf8clen(s); - else if (LEAF_CCC(leaf) == DECOMPOSE) - ret += strlen(LEAF_STR(leaf)); - else - ret += utf8clen(s); - s += utf8clen(s); - } - return ret; -} - -/* - * Length of the normalization of s, touch at most len bytes. - * Return -1 if s is not valid UTF-8 unicode. - */ -ssize_t utf8nlen(const struct utf8data *data, const char *s, size_t len) -{ - utf8leaf_t *leaf; - size_t ret = 0; - unsigned char hangul[UTF8HANGULLEAF]; - - if (!data) - return -1; - while (len && *s) { - leaf = utf8nlookup(data, hangul, s, len); - if (!leaf) - return -1; - if (utf8agetab[LEAF_GEN(leaf)] > data->maxage) - ret += utf8clen(s); - else if (LEAF_CCC(leaf) == DECOMPOSE) - ret += strlen(LEAF_STR(leaf)); - else - ret += utf8clen(s); - len -= utf8clen(s); - s += utf8clen(s); - } - return ret; -} - -/* - * Set up an utf8cursor for use by utf8byte(). - * - * u8c : pointer to cursor. - * data : const struct utf8data to use for normalization. - * s : string. - * len : length of s. - * - * Returns -1 on error, 0 on success. - */ -int utf8ncursor(struct utf8cursor *u8c, const struct utf8data *data, - const char *s, size_t len) -{ - if (!data) - return -1; - if (!s) - return -1; - u8c->data = data; - u8c->s = s; - u8c->p = NULL; - u8c->ss = NULL; - u8c->sp = NULL; - u8c->len = len; - u8c->slen = 0; - u8c->ccc = STOPPER; - u8c->nccc = STOPPER; - /* Check we didn't clobber the maximum length. */ - if (u8c->len != len) - return -1; - /* The first byte of s may not be an utf8 continuation. */ - if (len > 0 && (*s & 0xC0) == 0x80) - return -1; - return 0; -} - -/* - * Set up an utf8cursor for use by utf8byte(). - * - * u8c : pointer to cursor. - * data : const struct utf8data to use for normalization. - * s : NUL-terminated string. - * - * Returns -1 on error, 0 on success. - */ -int utf8cursor(struct utf8cursor *u8c, const struct utf8data *data, - const char *s) -{ - return utf8ncursor(u8c, data, s, (unsigned int)-1); -} - -/* - * Get one byte from the normalized form of the string described by u8c. - * - * Returns the byte cast to an unsigned char on succes, and -1 on failure. - * - * The cursor keeps track of the location in the string in u8c->s. - * When a character is decomposed, the current location is stored in - * u8c->p, and u8c->s is set to the start of the decomposition. Note - * that bytes from a decomposition do not count against u8c->len. - * - * Characters are emitted if they match the current CCC in u8c->ccc. - * Hitting end-of-string while u8c->ccc == STOPPER means we're done, - * and the function returns 0 in that case. - * - * Sorting by CCC is done by repeatedly scanning the string. The - * values of u8c->s and u8c->p are stored in u8c->ss and u8c->sp at - * the start of the scan. The first pass finds the lowest CCC to be - * emitted and stores it in u8c->nccc, the second pass emits the - * characters with this CCC and finds the next lowest CCC. This limits - * the number of passes to 1 + the number of different CCCs in the - * sequence being scanned. - * - * Therefore: - * u8c->p != NULL -> a decomposition is being scanned. - * u8c->ss != NULL -> this is a repeating scan. - * u8c->ccc == -1 -> this is the first scan of a repeating scan. - */ -int utf8byte(struct utf8cursor *u8c) -{ - utf8leaf_t *leaf; - int ccc; - - for (;;) { - /* Check for the end of a decomposed character. */ - if (u8c->p && *u8c->s == '\0') { - u8c->s = u8c->p; - u8c->p = NULL; - } - - /* Check for end-of-string. */ - if (!u8c->p && (u8c->len == 0 || *u8c->s == '\0')) { - /* There is no next byte. */ - if (u8c->ccc == STOPPER) - return 0; - /* End-of-string during a scan counts as a stopper. */ - ccc = STOPPER; - goto ccc_mismatch; - } else if ((*u8c->s & 0xC0) == 0x80) { - /* This is a continuation of the current character. */ - if (!u8c->p) - u8c->len--; - return (unsigned char)*u8c->s++; - } - - /* Look up the data for the current character. */ - if (u8c->p) { - leaf = utf8lookup(u8c->data, u8c->hangul, u8c->s); - } else { - leaf = utf8nlookup(u8c->data, u8c->hangul, - u8c->s, u8c->len); - } - - /* No leaf found implies that the input is a binary blob. */ - if (!leaf) - return -1; - - ccc = LEAF_CCC(leaf); - /* Characters that are too new have CCC 0. */ - if (utf8agetab[LEAF_GEN(leaf)] > u8c->data->maxage) { - ccc = STOPPER; - } else if (ccc == DECOMPOSE) { - u8c->len -= utf8clen(u8c->s); - u8c->p = u8c->s + utf8clen(u8c->s); - u8c->s = LEAF_STR(leaf); - /* Empty decomposition implies CCC 0. */ - if (*u8c->s == '\0') { - if (u8c->ccc == STOPPER) - continue; - ccc = STOPPER; - goto ccc_mismatch; - } - - leaf = utf8lookup(u8c->data, u8c->hangul, u8c->s); - ccc = LEAF_CCC(leaf); - } - - /* - * If this is not a stopper, then see if it updates - * the next canonical class to be emitted. - */ - if (ccc != STOPPER && u8c->ccc < ccc && ccc < u8c->nccc) - u8c->nccc = ccc; - - /* - * Return the current byte if this is the current - * combining class. - */ - if (ccc == u8c->ccc) { - if (!u8c->p) - u8c->len--; - return (unsigned char)*u8c->s++; - } - - /* Current combining class mismatch. */ -ccc_mismatch: - if (u8c->nccc == STOPPER) { - /* - * Scan forward for the first canonical class - * to be emitted. Save the position from - * which to restart. - */ - u8c->ccc = MINCCC - 1; - u8c->nccc = ccc; - u8c->sp = u8c->p; - u8c->ss = u8c->s; - u8c->slen = u8c->len; - if (!u8c->p) - u8c->len -= utf8clen(u8c->s); - u8c->s += utf8clen(u8c->s); - } else if (ccc != STOPPER) { - /* Not a stopper, and not the ccc we're emitting. */ - if (!u8c->p) - u8c->len -= utf8clen(u8c->s); - u8c->s += utf8clen(u8c->s); - } else if (u8c->nccc != MAXCCC + 1) { - /* At a stopper, restart for next ccc. */ - u8c->ccc = u8c->nccc; - u8c->nccc = MAXCCC + 1; - u8c->s = u8c->ss; - u8c->p = u8c->sp; - u8c->len = u8c->slen; - } else { - /* All done, proceed from here. */ - u8c->ccc = STOPPER; - u8c->nccc = STOPPER; - u8c->sp = NULL; - u8c->ss = NULL; - u8c->slen = 0; - } - } -} - -const struct utf8data *utf8nfdi(unsigned int maxage) -{ - int i = ARRAY_SIZE(utf8nfdidata) - 1; - - while (maxage < utf8nfdidata[i].maxage) - i--; - if (maxage > utf8nfdidata[i].maxage) - return NULL; - return &utf8nfdidata[i]; -} - -const struct utf8data *utf8nfdicf(unsigned int maxage) -{ - int i = ARRAY_SIZE(utf8nfdicfdata) - 1; - - while (maxage < utf8nfdicfdata[i].maxage) - i--; - if (maxage > utf8nfdicfdata[i].maxage) - return NULL; - return &utf8nfdicfdata[i]; -} diff --git a/lib/ext2fs/nls_utf8.c b/lib/ext2fs/nls_utf8.c index 014d7e5..9cb6a78 100644 --- a/lib/ext2fs/nls_utf8.c +++ b/lib/ext2fs/nls_utf8.c @@ -1,4 +1,5 @@ /* + * Copyright (c) 2014 SGI. * Copyright (c) 2018 Collabora Ltd. * All rights reserved. * @@ -20,13 +21,870 @@ */ #include "config.h" +#include +#include +#include +#include +#include + #include "ext2_fs.h" #include "ext2fs.h" -#include "utf8n.h" +/* Encoding a unicode version number as a single unsigned int. */ +#define UNICODE_MAJ_SHIFT (16) +#define UNICODE_MIN_SHIFT (8) -#include -#include +#define UNICODE_AGE(MAJ, MIN, REV) \ + (((unsigned int)(MAJ) << UNICODE_MAJ_SHIFT) | \ + ((unsigned int)(MIN) << UNICODE_MIN_SHIFT) | \ + ((unsigned int)(REV))) + +/* Needed in struct utf8cursor below. */ +#define UTF8HANGULLEAF (12) + +/* + * Cursor structure used by the normalizer. + */ +struct utf8cursor { + const struct utf8data *data; + const char *s; + const char *p; + const char *ss; + const char *sp; + unsigned int len; + unsigned int slen; + short int ccc; + short int nccc; + unsigned char hangul[UTF8HANGULLEAF]; +}; + +/* + * Initialize a utf8cursor to normalize a string. + * Returns 0 on success. + * Returns -1 on failure. + */ +// extern int utf8cursor(struct utf8cursor *u8c, const struct utf8data *data, +// const char *s); +// extern int utf8ncursor(struct utf8cursor *u8c, const struct utf8data *data, +// const char *s, size_t len); + +/* + * Get the next byte in the normalization. + * Returns a value > 0 && < 256 on success. + * Returns 0 when the end of the normalization is reached. + * Returns -1 if the string being normalized is not valid UTF-8. + */ +// extern int utf8byte(struct utf8cursor *u8c); + + +struct utf8data { + unsigned int maxage; + unsigned int offset; +}; + +#define __INCLUDED_FROM_UTF8NORM_C__ +#include "utf8data.h" +#undef __INCLUDED_FROM_UTF8NORM_C__ + +#define ARRAY_SIZE(array) \ + (sizeof(array) / sizeof(array[0])) + +#if 0 +/* Highest unicode version supported by the data tables. */ +static int utf8version_is_supported(uint8_t maj, uint8_t min, uint8_t rev) +{ + int i = ARRAY_SIZE(utf8agetab) - 1; + unsigned int sb_utf8version = UNICODE_AGE(maj, min, rev); + + while (i >= 0 && utf8agetab[i] != 0) { + if (sb_utf8version == utf8agetab[i]) + return 1; + i--; + } + return 0; +} +#endif + +#if 0 +static int utf8version_latest(void) +{ + return utf8vers; +} +#endif + +/* + * UTF-8 valid ranges. + * + * The UTF-8 encoding spreads the bits of a 32bit word over several + * bytes. This table gives the ranges that can be held and how they'd + * be represented. + * + * 0x00000000 0x0000007F: 0xxxxxxx + * 0x00000000 0x000007FF: 110xxxxx 10xxxxxx + * 0x00000000 0x0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx + * 0x00000000 0x001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + * 0x00000000 0x03FFFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx + * 0x00000000 0x7FFFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx + * + * There is an additional requirement on UTF-8, in that only the + * shortest representation of a 32bit value is to be used. A decoder + * must not decode sequences that do not satisfy this requirement. + * Thus the allowed ranges have a lower bound. + * + * 0x00000000 0x0000007F: 0xxxxxxx + * 0x00000080 0x000007FF: 110xxxxx 10xxxxxx + * 0x00000800 0x0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx + * 0x00010000 0x001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + * 0x00200000 0x03FFFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx + * 0x04000000 0x7FFFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx + * + * Actual unicode characters are limited to the range 0x0 - 0x10FFFF, + * 17 planes of 65536 values. This limits the sequences actually seen + * even more, to just the following. + * + * 0 - 0x7F: 0 - 0x7F + * 0x80 - 0x7FF: 0xC2 0x80 - 0xDF 0xBF + * 0x800 - 0xFFFF: 0xE0 0xA0 0x80 - 0xEF 0xBF 0xBF + * 0x10000 - 0x10FFFF: 0xF0 0x90 0x80 0x80 - 0xF4 0x8F 0xBF 0xBF + * + * Within those ranges the surrogates 0xD800 - 0xDFFF are not allowed. + * + * Note that the longest sequence seen with valid usage is 4 bytes, + * the same a single UTF-32 character. This makes the UTF-8 + * representation of Unicode strictly smaller than UTF-32. + * + * The shortest sequence requirement was introduced by: + * Corrigendum #1: UTF-8 Shortest Form + * It can be found here: + * http://www.unicode.org/versions/corrigendum1.html + * + */ + +/* + * Return the number of bytes used by the current UTF-8 sequence. + * Assumes the input points to the first byte of a valid UTF-8 + * sequence. + */ +static inline int utf8clen(const char *s) +{ + unsigned char c = *s; + + return 1 + (c >= 0xC0) + (c >= 0xE0) + (c >= 0xF0); +} + +/* + * Decode a 3-byte UTF-8 sequence. + */ +static unsigned int +utf8decode3(const char *str) +{ + unsigned int uc; + + uc = *str++ & 0x0F; + uc <<= 6; + uc |= *str++ & 0x3F; + uc <<= 6; + uc |= *str++ & 0x3F; + + return uc; +} + +/* + * Encode a 3-byte UTF-8 sequence. + */ +static int +utf8encode3(char *str, unsigned int val) +{ + str[2] = (val & 0x3F) | 0x80; + val >>= 6; + str[1] = (val & 0x3F) | 0x80; + val >>= 6; + str[0] = val | 0xE0; + + return 3; +} + +/* + * utf8trie_t + * + * A compact binary tree, used to decode UTF-8 characters. + * + * Internal nodes are one byte for the node itself, and up to three + * bytes for an offset into the tree. The first byte contains the + * following information: + * NEXTBYTE - flag - advance to next byte if set + * BITNUM - 3 bit field - the bit number to tested + * OFFLEN - 2 bit field - number of bytes in the offset + * if offlen == 0 (non-branching node) + * RIGHTPATH - 1 bit field - set if the following node is for the + * right-hand path (tested bit is set) + * TRIENODE - 1 bit field - set if the following node is an internal + * node, otherwise it is a leaf node + * if offlen != 0 (branching node) + * LEFTNODE - 1 bit field - set if the left-hand node is internal + * RIGHTNODE - 1 bit field - set if the right-hand node is internal + * + * Due to the way utf8 works, there cannot be branching nodes with + * NEXTBYTE set, and moreover those nodes always have a righthand + * descendant. + */ +typedef const unsigned char utf8trie_t; +#define BITNUM 0x07 +#define NEXTBYTE 0x08 +#define OFFLEN 0x30 +#define OFFLEN_SHIFT 4 +#define RIGHTPATH 0x40 +#define TRIENODE 0x80 +#define RIGHTNODE 0x40 +#define LEFTNODE 0x80 + +/* + * utf8leaf_t + * + * The leaves of the trie are embedded in the trie, and so the same + * underlying datatype: unsigned char. + * + * leaf[0]: The unicode version, stored as a generation number that is + * an index into utf8agetab[]. With this we can filter code + * points based on the unicode version in which they were + * defined. The CCC of a non-defined code point is 0. + * leaf[1]: Canonical Combining Class. During normalization, we need + * to do a stable sort into ascending order of all characters + * with a non-zero CCC that occur between two characters with + * a CCC of 0, or at the begin or end of a string. + * The unicode standard guarantees that all CCC values are + * between 0 and 254 inclusive, which leaves 255 available as + * a special value. + * Code points with CCC 0 are known as stoppers. + * leaf[2]: Decomposition. If leaf[1] == 255, then leaf[2] is the + * start of a NUL-terminated string that is the decomposition + * of the character. + * The CCC of a decomposable character is the same as the CCC + * of the first character of its decomposition. + * Some characters decompose as the empty string: these are + * characters with the Default_Ignorable_Code_Point property. + * These do affect normalization, as they all have CCC 0. + * + * The decompositions in the trie have been fully expanded, with the + * exception of Hangul syllables, which are decomposed algorithmically. + * + * Casefolding, if applicable, is also done using decompositions. + * + * The trie is constructed in such a way that leaves exist for all + * UTF-8 sequences that match the criteria from the "UTF-8 valid + * ranges" comment above, and only for those sequences. Therefore a + * lookup in the trie can be used to validate the UTF-8 input. + */ +typedef const unsigned char utf8leaf_t; + +#define LEAF_GEN(LEAF) ((LEAF)[0]) +#define LEAF_CCC(LEAF) ((LEAF)[1]) +#define LEAF_STR(LEAF) ((const char *)((LEAF) + 2)) + +#define MINCCC (0) +#define MAXCCC (254) +#define STOPPER (0) +#define DECOMPOSE (255) + +/* Marker for hangul syllable decomposition. */ +#define HANGUL ((char)(255)) +/* Size of the synthesized leaf used for Hangul syllable decomposition. */ +#define UTF8HANGULLEAF (12) + +/* + * Hangul decomposition (algorithm from Section 3.12 of Unicode 6.3.0) + * + * AC00;;Lo;0;L;;;;;N;;;;; + * D7A3;;Lo;0;L;;;;;N;;;;; + * + * SBase = 0xAC00 + * LBase = 0x1100 + * VBase = 0x1161 + * TBase = 0x11A7 + * LCount = 19 + * VCount = 21 + * TCount = 28 + * NCount = 588 (VCount * TCount) + * SCount = 11172 (LCount * NCount) + * + * Decomposition: + * SIndex = s - SBase + * + * LV (Canonical/Full) + * LIndex = SIndex / NCount + * VIndex = (Sindex % NCount) / TCount + * LPart = LBase + LIndex + * VPart = VBase + VIndex + * + * LVT (Canonical) + * LVIndex = (SIndex / TCount) * TCount + * TIndex = (Sindex % TCount) + * LVPart = SBase + LVIndex + * TPart = TBase + TIndex + * + * LVT (Full) + * LIndex = SIndex / NCount + * VIndex = (Sindex % NCount) / TCount + * TIndex = (Sindex % TCount) + * LPart = LBase + LIndex + * VPart = VBase + VIndex + * if (TIndex == 0) { + * d = + * } else { + * TPart = TBase + TIndex + * d = + * } + */ + +/* Constants */ +#define SB (0xAC00) +#define LB (0x1100) +#define VB (0x1161) +#define TB (0x11A7) +#define LC (19) +#define VC (21) +#define TC (28) +#define NC (VC * TC) +#define SC (LC * NC) + +/* Algorithmic decomposition of hangul syllable. */ +static utf8leaf_t * +utf8hangul(const char *str, unsigned char *hangul) +{ + unsigned int si; + unsigned int li; + unsigned int vi; + unsigned int ti; + unsigned char *h; + + /* Calculate the SI, LI, VI, and TI values. */ + si = utf8decode3(str) - SB; + li = si / NC; + vi = (si % NC) / TC; + ti = si % TC; + + /* Fill in base of leaf. */ + h = hangul; + LEAF_GEN(h) = 2; + LEAF_CCC(h) = DECOMPOSE; + h += 2; + + /* Add LPart, a 3-byte UTF-8 sequence. */ + h += utf8encode3((char *)h, li + LB); + + /* Add VPart, a 3-byte UTF-8 sequence. */ + h += utf8encode3((char *)h, vi + VB); + + /* Add TPart if required, also a 3-byte UTF-8 sequence. */ + if (ti) + h += utf8encode3((char *)h, ti + TB); + + /* Terminate string. */ + h[0] = '\0'; + + return hangul; +} + +/* + * Use trie to scan s, touching at most len bytes. + * Returns the leaf if one exists, NULL otherwise. + * + * A non-NULL return guarantees that the UTF-8 sequence starting at s + * is well-formed and corresponds to a known unicode code point. The + * shorthand for this will be "is valid UTF-8 unicode". + */ +static utf8leaf_t *utf8nlookup(const struct utf8data *data, + unsigned char *hangul, const char *s, size_t len) +{ + utf8trie_t *trie; + int offlen; + int offset; + int mask; + int node; + + if (!data) + return NULL; + if (len == 0) + return NULL; + + trie = utf8data + data->offset; + node = 1; + while (node) { + offlen = (*trie & OFFLEN) >> OFFLEN_SHIFT; + if (*trie & NEXTBYTE) { + if (--len == 0) + return NULL; + s++; + } + mask = 1 << (*trie & BITNUM); + if (*s & mask) { + /* Right leg */ + if (offlen) { + /* Right node at offset of trie */ + node = (*trie & RIGHTNODE); + offset = trie[offlen]; + while (--offlen) { + offset <<= 8; + offset |= trie[offlen]; + } + trie += offset; + } else if (*trie & RIGHTPATH) { + /* Right node after this node */ + node = (*trie & TRIENODE); + trie++; + } else { + /* No right node. */ + return NULL; + } + } else { + /* Left leg */ + if (offlen) { + /* Left node after this node. */ + node = (*trie & LEFTNODE); + trie += offlen + 1; + } else if (*trie & RIGHTPATH) { + /* No left node. */ + return NULL; + } else { + /* Left node after this node */ + node = (*trie & TRIENODE); + trie++; + } + } + } + /* + * Hangul decomposition is done algorithmically. These are the + * codepoints >= 0xAC00 and <= 0xD7A3. Their UTF-8 encoding is + * always 3 bytes long, so s has been advanced twice, and the + * start of the sequence is at s-2. + */ + if (LEAF_CCC(trie) == DECOMPOSE && LEAF_STR(trie)[0] == HANGUL) + trie = utf8hangul(s - 2, hangul); + return trie; +} + +/* + * Use trie to scan s. + * Returns the leaf if one exists, NULL otherwise. + * + * Forwards to utf8nlookup(). + */ +static utf8leaf_t *utf8lookup(const struct utf8data *data, + unsigned char *hangul, const char *s) +{ + return utf8nlookup(data, hangul, s, (size_t)-1); +} + +#if 0 +/* + * Maximum age of any character in s. + * Return -1 if s is not valid UTF-8 unicode. + * Return 0 if only non-assigned code points are used. + */ +static int utf8agemax(const struct utf8data *data, const char *s) +{ + utf8leaf_t *leaf; + int age = 0; + int leaf_age; + unsigned char hangul[UTF8HANGULLEAF]; + + if (!data) + return -1; + + while (*s) { + leaf = utf8lookup(data, hangul, s); + if (!leaf) + return -1; + + leaf_age = utf8agetab[LEAF_GEN(leaf)]; + if (leaf_age <= data->maxage && leaf_age > age) + age = leaf_age; + s += utf8clen(s); + } + return age; +} +#endif + +#if 0 +/* + * Minimum age of any character in s. + * Return -1 if s is not valid UTF-8 unicode. + * Return 0 if non-assigned code points are used. + */ +static int utf8agemin(const struct utf8data *data, const char *s) +{ + utf8leaf_t *leaf; + int age; + int leaf_age; + unsigned char hangul[UTF8HANGULLEAF]; + + if (!data) + return -1; + age = data->maxage; + while (*s) { + leaf = utf8lookup(data, hangul, s); + if (!leaf) + return -1; + leaf_age = utf8agetab[LEAF_GEN(leaf)]; + if (leaf_age <= data->maxage && leaf_age < age) + age = leaf_age; + s += utf8clen(s); + } + return age; +} +#endif + +#if 0 +/* + * Maximum age of any character in s, touch at most len bytes. + * Return -1 if s is not valid UTF-8 unicode. + */ +static int utf8nagemax(const struct utf8data *data, const char *s, size_t len) +{ + utf8leaf_t *leaf; + int age = 0; + int leaf_age; + unsigned char hangul[UTF8HANGULLEAF]; + + if (!data) + return -1; + + while (len && *s) { + leaf = utf8nlookup(data, hangul, s, len); + if (!leaf) + return -1; + leaf_age = utf8agetab[LEAF_GEN(leaf)]; + if (leaf_age <= data->maxage && leaf_age > age) + age = leaf_age; + len -= utf8clen(s); + s += utf8clen(s); + } + return age; +} +#endif + +#if 0 +/* + * Maximum age of any character in s, touch at most len bytes. + * Return -1 if s is not valid UTF-8 unicode. + */ +static int utf8nagemin(const struct utf8data *data, const char *s, size_t len) +{ + utf8leaf_t *leaf; + int leaf_age; + int age; + unsigned char hangul[UTF8HANGULLEAF]; + + if (!data) + return -1; + age = data->maxage; + while (len && *s) { + leaf = utf8nlookup(data, hangul, s, len); + if (!leaf) + return -1; + leaf_age = utf8agetab[LEAF_GEN(leaf)]; + if (leaf_age <= data->maxage && leaf_age < age) + age = leaf_age; + len -= utf8clen(s); + s += utf8clen(s); + } + return age; +} +#endif + +#if 0 +/* + * Length of the normalization of s. + * Return -1 if s is not valid UTF-8 unicode. + * + * A string of Default_Ignorable_Code_Point has length 0. + */ +static ssize_t utf8len(const struct utf8data *data, const char *s) +{ + utf8leaf_t *leaf; + size_t ret = 0; + unsigned char hangul[UTF8HANGULLEAF]; + + if (!data) + return -1; + while (*s) { + leaf = utf8lookup(data, hangul, s); + if (!leaf) + return -1; + if (utf8agetab[LEAF_GEN(leaf)] > data->maxage) + ret += utf8clen(s); + else if (LEAF_CCC(leaf) == DECOMPOSE) + ret += strlen(LEAF_STR(leaf)); + else + ret += utf8clen(s); + s += utf8clen(s); + } + return ret; +} +#endif + +#if 0 +/* + * Length of the normalization of s, touch at most len bytes. + * Return -1 if s is not valid UTF-8 unicode. + */ +static ssize_t utf8nlen(const struct utf8data *data, const char *s, size_t len) +{ + utf8leaf_t *leaf; + size_t ret = 0; + unsigned char hangul[UTF8HANGULLEAF]; + + if (!data) + return -1; + while (len && *s) { + leaf = utf8nlookup(data, hangul, s, len); + if (!leaf) + return -1; + if (utf8agetab[LEAF_GEN(leaf)] > data->maxage) + ret += utf8clen(s); + else if (LEAF_CCC(leaf) == DECOMPOSE) + ret += strlen(LEAF_STR(leaf)); + else + ret += utf8clen(s); + len -= utf8clen(s); + s += utf8clen(s); + } + return ret; +} +#endif + +/* + * Set up an utf8cursor for use by utf8byte(). + * + * u8c : pointer to cursor. + * data : const struct utf8data to use for normalization. + * s : string. + * len : length of s. + * + * Returns -1 on error, 0 on success. + */ +static int utf8ncursor(struct utf8cursor *u8c, const struct utf8data *data, + const char *s, size_t len) +{ + if (!data) + return -1; + if (!s) + return -1; + u8c->data = data; + u8c->s = s; + u8c->p = NULL; + u8c->ss = NULL; + u8c->sp = NULL; + u8c->len = len; + u8c->slen = 0; + u8c->ccc = STOPPER; + u8c->nccc = STOPPER; + /* Check we didn't clobber the maximum length. */ + if (u8c->len != len) + return -1; + /* The first byte of s may not be an utf8 continuation. */ + if (len > 0 && (*s & 0xC0) == 0x80) + return -1; + return 0; +} + +#if 0 +/* + * Set up an utf8cursor for use by utf8byte(). + * + * u8c : pointer to cursor. + * data : const struct utf8data to use for normalization. + * s : NUL-terminated string. + * + * Returns -1 on error, 0 on success. + */ +static int utf8cursor(struct utf8cursor *u8c, const struct utf8data *data, + const char *s) +{ + return utf8ncursor(u8c, data, s, (unsigned int)-1); +} +#endif + +/* + * Get one byte from the normalized form of the string described by u8c. + * + * Returns the byte cast to an unsigned char on succes, and -1 on failure. + * + * The cursor keeps track of the location in the string in u8c->s. + * When a character is decomposed, the current location is stored in + * u8c->p, and u8c->s is set to the start of the decomposition. Note + * that bytes from a decomposition do not count against u8c->len. + * + * Characters are emitted if they match the current CCC in u8c->ccc. + * Hitting end-of-string while u8c->ccc == STOPPER means we're done, + * and the function returns 0 in that case. + * + * Sorting by CCC is done by repeatedly scanning the string. The + * values of u8c->s and u8c->p are stored in u8c->ss and u8c->sp at + * the start of the scan. The first pass finds the lowest CCC to be + * emitted and stores it in u8c->nccc, the second pass emits the + * characters with this CCC and finds the next lowest CCC. This limits + * the number of passes to 1 + the number of different CCCs in the + * sequence being scanned. + * + * Therefore: + * u8c->p != NULL -> a decomposition is being scanned. + * u8c->ss != NULL -> this is a repeating scan. + * u8c->ccc == -1 -> this is the first scan of a repeating scan. + */ +static int utf8byte(struct utf8cursor *u8c) +{ + utf8leaf_t *leaf; + int ccc; + + for (;;) { + /* Check for the end of a decomposed character. */ + if (u8c->p && *u8c->s == '\0') { + u8c->s = u8c->p; + u8c->p = NULL; + } + + /* Check for end-of-string. */ + if (!u8c->p && (u8c->len == 0 || *u8c->s == '\0')) { + /* There is no next byte. */ + if (u8c->ccc == STOPPER) + return 0; + /* End-of-string during a scan counts as a stopper. */ + ccc = STOPPER; + goto ccc_mismatch; + } else if ((*u8c->s & 0xC0) == 0x80) { + /* This is a continuation of the current character. */ + if (!u8c->p) + u8c->len--; + return (unsigned char)*u8c->s++; + } + + /* Look up the data for the current character. */ + if (u8c->p) { + leaf = utf8lookup(u8c->data, u8c->hangul, u8c->s); + } else { + leaf = utf8nlookup(u8c->data, u8c->hangul, + u8c->s, u8c->len); + } + + /* No leaf found implies that the input is a binary blob. */ + if (!leaf) + return -1; + + ccc = LEAF_CCC(leaf); + /* Characters that are too new have CCC 0. */ + if (utf8agetab[LEAF_GEN(leaf)] > u8c->data->maxage) { + ccc = STOPPER; + } else if (ccc == DECOMPOSE) { + u8c->len -= utf8clen(u8c->s); + u8c->p = u8c->s + utf8clen(u8c->s); + u8c->s = LEAF_STR(leaf); + /* Empty decomposition implies CCC 0. */ + if (*u8c->s == '\0') { + if (u8c->ccc == STOPPER) + continue; + ccc = STOPPER; + goto ccc_mismatch; + } + + leaf = utf8lookup(u8c->data, u8c->hangul, u8c->s); + ccc = LEAF_CCC(leaf); + } + + /* + * If this is not a stopper, then see if it updates + * the next canonical class to be emitted. + */ + if (ccc != STOPPER && u8c->ccc < ccc && ccc < u8c->nccc) + u8c->nccc = ccc; + + /* + * Return the current byte if this is the current + * combining class. + */ + if (ccc == u8c->ccc) { + if (!u8c->p) + u8c->len--; + return (unsigned char)*u8c->s++; + } + + /* Current combining class mismatch. */ +ccc_mismatch: + if (u8c->nccc == STOPPER) { + /* + * Scan forward for the first canonical class + * to be emitted. Save the position from + * which to restart. + */ + u8c->ccc = MINCCC - 1; + u8c->nccc = ccc; + u8c->sp = u8c->p; + u8c->ss = u8c->s; + u8c->slen = u8c->len; + if (!u8c->p) + u8c->len -= utf8clen(u8c->s); + u8c->s += utf8clen(u8c->s); + } else if (ccc != STOPPER) { + /* Not a stopper, and not the ccc we're emitting. */ + if (!u8c->p) + u8c->len -= utf8clen(u8c->s); + u8c->s += utf8clen(u8c->s); + } else if (u8c->nccc != MAXCCC + 1) { + /* At a stopper, restart for next ccc. */ + u8c->ccc = u8c->nccc; + u8c->nccc = MAXCCC + 1; + u8c->s = u8c->ss; + u8c->p = u8c->sp; + u8c->len = u8c->slen; + } else { + /* All done, proceed from here. */ + u8c->ccc = STOPPER; + u8c->nccc = STOPPER; + u8c->sp = NULL; + u8c->ss = NULL; + u8c->slen = 0; + } + } +} + +#if 0 +/* + * Look for the correct const struct utf8data for a unicode version. + * Returns NULL if the version requested is too new. + * + * Two normalization forms are supported: nfdi and nfdicf. + * + * nfdi: + * - Apply unicode normalization form NFD. + * - Remove any Default_Ignorable_Code_Point. + * + * nfdicf: + * - Apply unicode normalization form NFD. + * - Remove any Default_Ignorable_Code_Point. + * - Apply a full casefold (C + F). + */ +static const struct utf8data *utf8nfdi(unsigned int maxage) +{ + int i = ARRAY_SIZE(utf8nfdidata) - 1; + + while (maxage < utf8nfdidata[i].maxage) + i--; + if (maxage > utf8nfdidata[i].maxage) + return NULL; + return &utf8nfdidata[i]; +} +#endif + +static const struct utf8data *utf8nfdicf(unsigned int maxage) +{ + int i = ARRAY_SIZE(utf8nfdicfdata) - 1; + + while (maxage < utf8nfdicfdata[i].maxage) + i--; + if (maxage > utf8nfdicfdata[i].maxage) + return NULL; + return &utf8nfdicfdata[i]; +} static int utf8_casefold(const struct ext2fs_nls_table *table, const unsigned char *str, size_t len, @@ -36,14 +894,16 @@ static int utf8_casefold(const struct ext2fs_nls_table *table, struct utf8cursor cur; size_t nlen = 0; - if (utf8ncursor(&cur, data, str, len) < 0) + if (utf8ncursor(&cur, data, (const char *) str, len) < 0) goto invalid_seq; for (nlen = 0; nlen < dlen; nlen++) { - dest[nlen] = utf8byte(&cur); - if (!dest[nlen]) + int c = utf8byte(&cur); + + dest[nlen] = c; + if (!c) return nlen; - if (dest[nlen] == -1) + if (c == -1) break; } @@ -57,7 +917,7 @@ invalid_seq: return -EINVAL; } -const static struct ext2fs_nls_ops utf8_ops = { +static const struct ext2fs_nls_ops utf8_ops = { .casefold = utf8_casefold, }; @@ -68,8 +928,6 @@ static const struct ext2fs_nls_table nls_utf8 = { const struct ext2fs_nls_table *ext2fs_load_nls_table(int encoding) { - int i; - if (encoding == EXT4_ENC_UTF8_12_1) return &nls_utf8;