From 1b5f839db92051a35a04b265c1fcdedd25edfb64 Mon Sep 17 00:00:00 2001 From: Robert Sipka Date: Wed, 15 Feb 2017 15:57:55 +0100 Subject: [PATCH] Improve `toLowerCase` and `toUpperCase` functions. (#1575) Language-sensitive mappings are not processed now. Fixes #323 JerryScript-DCO-1.0-Signed-off-by: Robert Sipka rsipka.uszeged@partner.samsung.com --- jerry-core/lit/lit-char-helpers.c | 257 ++++++- jerry-core/lit/lit-unicode-conversions.inc.h | 162 +++++ jerry-core/profiles/minimal.profile | 1 + .../string-upper-lower-case-conversion.js | 67 +- tools/unicode_case_conversion.py | 681 ++++++++++++++++++ 5 files changed, 1163 insertions(+), 5 deletions(-) create mode 100644 jerry-core/lit/lit-unicode-conversions.inc.h create mode 100755 tools/unicode_case_conversion.py diff --git a/jerry-core/lit/lit-char-helpers.c b/jerry-core/lit/lit-char-helpers.c index 70dbd249ee..1cc15fa520 100644 --- a/jerry-core/lit/lit-char-helpers.c +++ b/jerry-core/lit/lit-char-helpers.c @@ -17,6 +17,10 @@ #include "lit/lit-unicode-ranges.inc.h" #include "lit-strings.h" +#ifndef CONFIG_DISABLE_UNICODE_CASE_CONVERSION +#include "lit-unicode-conversions.inc.h" +#endif /* !CONFIG_DISABLE_UNICODE_CASE_CONVERSION */ + #define NUM_OF_ELEMENTS(array) (sizeof (array) / sizeof ((array)[0])) /** @@ -458,6 +462,184 @@ lit_char_is_word_char (ecma_char_t c) /**< code unit */ || c == LIT_CHAR_UNDERSCORE); } /* lit_char_is_word_char */ +#ifndef CONFIG_DISABLE_UNICODE_CASE_CONVERSION + +/** + * Check if the specified character is in one of those tables which contain bidirectional conversions. + * + * @return the mapped character sequence of an ecma character, if it's in the table. + * 0 - otherwise. + */ +static ecma_length_t +search_in_bidirectional_conversion_tables (ecma_char_t character, /**< code unit */ + ecma_char_t *output_buffer_p, /**< [out] buffer for the result characters */ + bool is_lowercase) /**< is lowercase conversion */ +{ + /* 1, Check if the specified character is part of the jerry_character_case_ranges table. */ + int number_of_case_ranges = NUM_OF_ELEMENTS (jerry_character_case_ranges); + int conv_counter = 0; + + for (int i = 0; i < number_of_case_ranges; i++) + { + if (i % 2 == 0 && i > 0) + { + conv_counter++; + } + + int range_length = jerry_character_case_range_lengths[conv_counter]; + ecma_char_t start_point = jerry_character_case_ranges[i]; + + if (start_point > character || character >= start_point + range_length) + { + continue; + } + + int char_dist = character - start_point; + + if (i % 2 == 0) + { + output_buffer_p[0] = is_lowercase ? (ecma_char_t) (jerry_character_case_ranges[i + 1] + char_dist) : character; + } + else + { + output_buffer_p[0] = is_lowercase ? character : (ecma_char_t) (jerry_character_case_ranges[i - 1] + char_dist); + } + + return 1; + } + + /* 2, Check if the specified character is part of the character_pair_ranges table. */ + int bottom = 0; + int top = NUM_OF_ELEMENTS (jerry_character_pair_ranges) - 1; + + while (bottom <= top) + { + int middle = (bottom + top) / 2; + ecma_char_t current_sp = jerry_character_pair_ranges[middle]; + + if (current_sp <= character && character < current_sp + jerry_character_pair_range_lengths[middle]) + { + int char_dist = character - current_sp; + + if ((character - current_sp) % 2 == 0) + { + output_buffer_p[0] = is_lowercase ? (ecma_char_t) (current_sp + char_dist + 1) : character; + } + else + { + output_buffer_p[0] = is_lowercase ? character : (ecma_char_t) (current_sp + char_dist - 1); + } + + return 1; + } + + if (character > current_sp) + { + bottom = middle + 1; + } + else + { + top = middle - 1; + } + } + + /* 3, Check if the specified character is part of the character_pairs table. */ + int number_of_character_pairs = NUM_OF_ELEMENTS (jerry_character_pairs); + + for (int i = 0; i < number_of_character_pairs; i++) + { + if (character != jerry_character_pairs[i]) + { + continue; + } + + if (i % 2 == 0) + { + output_buffer_p[0] = is_lowercase ? jerry_character_pairs[i + 1] : character; + } + else + { + output_buffer_p[0] = is_lowercase ? character : jerry_character_pairs[i - 1]; + } + + return 1; + } + + return 0; +} /* search_in_bidirectional_conversion_tables */ + +/** + * Check if the specified character is in the given conversion table. + * + * @return the mapped character sequence of an ecma character, if it's in the table. + * 0 - otherwise. + */ +static ecma_length_t +search_in_conversion_table (ecma_char_t character, /**< code unit */ + ecma_char_t *output_buffer_p, /**< [out] buffer for the result characters */ + const ecma_char_t *array, /**< array */ + const uint8_t *counters) /**< case_values counter */ +{ + int end_point = 0; + + for (int i = 0; i < 3; i++) + { + int start_point = end_point; + int size_of_case_value = i + 1; + end_point += counters[i] * (size_of_case_value + 1); + + int bottom = start_point; + int top = end_point - size_of_case_value; + + while (bottom <= top) + { + int middle = (bottom + top) / 2; + + middle -= ((middle - bottom) % (size_of_case_value + 1)); + + ecma_char_t current = array[middle]; + + if (current == character) + { + ecma_length_t char_sequence = 1; + + switch (size_of_case_value) + { + case 3: + { + output_buffer_p[2] = array[middle + 3]; + char_sequence++; + /* FALLTHRU */ + } + case 2: + { + output_buffer_p[1] = array[middle + 2]; + char_sequence++; + /* FALLTHRU */ + } + default: + { + output_buffer_p[0] = array[middle + 1]; + return char_sequence; + } + } + } + + if (character < current) + { + top = middle - (size_of_case_value + 1); + } + else + { + bottom = middle + (size_of_case_value + 1); + } + } + } + + return 0; +} /* search_in_conversion_table */ +#endif /* !CONFIG_DISABLE_UNICODE_CASE_CONVERSION */ + /** * Returns the lowercase character sequence of an ecma character. * @@ -471,8 +653,6 @@ lit_char_to_lower_case (ecma_char_t character, /**< input character value */ ecma_char_t *output_buffer_p, /**< [out] buffer for the result characters */ ecma_length_t buffer_size) /**< buffer size */ { - /* TODO: Needs a proper lower case implementation. See issue #323. */ - JERRY_ASSERT (buffer_size >= LIT_MAXIMUM_OTHER_CASE_LENGTH); if (character >= LIT_CHAR_UPPERCASE_A && character <= LIT_CHAR_UPPERCASE_Z) @@ -481,6 +661,41 @@ lit_char_to_lower_case (ecma_char_t character, /**< input character value */ return 1; } +#ifndef CONFIG_DISABLE_UNICODE_CASE_CONVERSION + + ecma_length_t lowercase_sequence = search_in_bidirectional_conversion_tables (character, output_buffer_p, true); + + if (lowercase_sequence != 0) + { + return lowercase_sequence; + } + + int num_of_lowercase_ranges = NUM_OF_ELEMENTS (jerry_lower_case_ranges); + + for (int i = 0, j = 0; i < num_of_lowercase_ranges; i += 2, j++) + { + int range_length = jerry_lower_case_range_lengths[j] - 1; + ecma_char_t start_point = jerry_lower_case_ranges[i]; + + if (start_point <= character && character <= start_point + range_length) + { + output_buffer_p[0] = (ecma_char_t) (jerry_lower_case_ranges[i + 1] + (character - start_point)); + return 1; + } + } + + lowercase_sequence = search_in_conversion_table (character, + output_buffer_p, + jerry_lower_case_conversions, + jerry_lower_case_conversion_counters); + + if (lowercase_sequence != 0) + { + return lowercase_sequence; + } + +#endif /* !CONFIG_DISABLE_UNICODE_CASE_CONVERSION */ + output_buffer_p[0] = character; return 1; } /* lit_char_to_lower_case */ @@ -498,8 +713,6 @@ lit_char_to_upper_case (ecma_char_t character, /**< input character value */ ecma_char_t *output_buffer_p, /**< buffer for the result characters */ ecma_length_t buffer_size) /**< buffer size */ { - /* TODO: Needs a proper upper case implementation. See issue #323. */ - JERRY_ASSERT (buffer_size >= LIT_MAXIMUM_OTHER_CASE_LENGTH); if (character >= LIT_CHAR_LOWERCASE_A && character <= LIT_CHAR_LOWERCASE_Z) @@ -508,6 +721,42 @@ lit_char_to_upper_case (ecma_char_t character, /**< input character value */ return 1; } +#ifndef CONFIG_DISABLE_UNICODE_CASE_CONVERSION + + ecma_length_t uppercase_sequence = search_in_bidirectional_conversion_tables (character, output_buffer_p, false); + + if (uppercase_sequence != 0) + { + return uppercase_sequence; + } + + int num_of_upper_case_special_ranges = NUM_OF_ELEMENTS (jerry_upper_case_special_ranges); + + for (int i = 0, j = 0; i < num_of_upper_case_special_ranges; i += 3, j++) + { + int range_length = jerry_upper_case_special_range_lengths[j]; + ecma_char_t start_point = jerry_upper_case_special_ranges[i]; + + if (start_point <= character && character <= start_point + range_length) + { + output_buffer_p[0] = (ecma_char_t) (jerry_upper_case_special_ranges[i + 1] + (character - start_point)); + output_buffer_p[1] = (ecma_char_t) (jerry_upper_case_special_ranges[i + 2]); + return 2; + } + } + + uppercase_sequence = search_in_conversion_table (character, + output_buffer_p, + jerry_upper_case_conversions, + jerry_upper_case_conversion_counters); + + if (uppercase_sequence != 0) + { + return uppercase_sequence; + } + +#endif /* !CONFIG_DISABLE_UNICODE_CASE_CONVERSION */ + output_buffer_p[0] = character; return 1; } /* lit_char_to_upper_case */ diff --git a/jerry-core/lit/lit-unicode-conversions.inc.h b/jerry-core/lit/lit-unicode-conversions.inc.h new file mode 100644 index 0000000000..0e7725733a --- /dev/null +++ b/jerry-core/lit/lit-unicode-conversions.inc.h @@ -0,0 +1,162 @@ +/* Copyright JS Foundation and other contributors, http://js.foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * This file is automatically generated by the unicode_case_conversion.py script. Do not edit! + */ + +/* Contains start points of character case ranges (these are bidirectional conversions). */ +static const uint16_t jerry_character_case_ranges[] JERRY_CONST_DATA = +{ + 0x00c0, 0x00e0, 0x00d8, 0x00f8, 0x0189, 0x0256, 0x01b1, 0x028a, 0x0388, 0x03ad, + 0x038e, 0x03cd, 0x0391, 0x03b1, 0x03a3, 0x03c3, 0x03fd, 0x037b, 0x0400, 0x0450, + 0x0410, 0x0430, 0x0531, 0x0561, 0x10a0, 0x2d00, 0x13a0, 0xab70, 0x13f0, 0x13f8, + 0x1f08, 0x1f00, 0x1f18, 0x1f10, 0x1f28, 0x1f20, 0x1f38, 0x1f30, 0x1f48, 0x1f40, + 0x1f68, 0x1f60, 0x1fb8, 0x1fb0, 0x1fba, 0x1f70, 0x1fc8, 0x1f72, 0x1fd8, 0x1fd0, + 0x1fda, 0x1f76, 0x1fe8, 0x1fe0, 0x1fea, 0x1f7a, 0x1ff8, 0x1f78, 0x1ffa, 0x1f7c, + 0x2160, 0x2170, 0x24b6, 0x24d0, 0x2c00, 0x2c30, 0x2c7e, 0x023f, 0xff21, 0xff41 +}; + +/* Interval lengths of start points in `character_case_ranges` table. */ +static const uint8_t jerry_character_case_range_lengths[] JERRY_CONST_DATA = +{ + 0x0017, 0x0007, 0x0002, 0x0002, 0x0003, 0x0002, 0x0011, 0x0009, 0x0003, 0x0010, + 0x0020, 0x0026, 0x0026, 0x0050, 0x0006, 0x0008, 0x0006, 0x0008, 0x0008, 0x0006, + 0x0008, 0x0002, 0x0002, 0x0004, 0x0002, 0x0002, 0x0002, 0x0002, 0x0002, 0x0002, + 0x0010, 0x001a, 0x002f, 0x0002, 0x001a +}; + +/* Contains the start points of bidirectional conversion ranges. */ +static const uint16_t jerry_character_pair_ranges[] JERRY_CONST_DATA = +{ + 0x0100, 0x0132, 0x0139, 0x014a, 0x0179, 0x0182, 0x0187, 0x018b, 0x0191, 0x0198, + 0x01a0, 0x01a7, 0x01ac, 0x01af, 0x01b3, 0x01b8, 0x01bc, 0x01cd, 0x01de, 0x01f4, + 0x01f8, 0x0222, 0x023b, 0x0241, 0x0246, 0x0370, 0x0376, 0x03d8, 0x03f7, 0x03fa, + 0x0460, 0x048a, 0x04c1, 0x04d0, 0x1e00, 0x1ea0, 0x2183, 0x2c60, 0x2c67, 0x2c72, + 0x2c75, 0x2c80, 0x2ceb, 0x2cf2, 0xa640, 0xa680, 0xa722, 0xa732, 0xa779, 0xa77e, + 0xa78b, 0xa790, 0xa796, 0xa7b4 +}; + +/* Interval lengths of start points in `character_pair_ranges` table. */ +static const uint8_t jerry_character_pair_range_lengths[] JERRY_CONST_DATA = +{ + 0x0030, 0x0006, 0x0010, 0x002e, 0x0006, 0x0004, 0x0002, 0x0002, 0x0002, 0x0002, + 0x0006, 0x0002, 0x0002, 0x0002, 0x0004, 0x0002, 0x0002, 0x0010, 0x0012, 0x0002, + 0x0028, 0x0012, 0x0002, 0x0002, 0x000a, 0x0004, 0x0002, 0x0018, 0x0002, 0x0002, + 0x0022, 0x0036, 0x000e, 0x0060, 0x0096, 0x0060, 0x0002, 0x0002, 0x0006, 0x0002, + 0x0002, 0x0064, 0x0004, 0x0002, 0x002e, 0x001c, 0x000e, 0x003e, 0x0004, 0x000a, + 0x0002, 0x0004, 0x0014, 0x0004 +}; + +/* Contains lower/upper case bidirectional conversion pairs. */ +static const uint16_t jerry_character_pairs[] JERRY_CONST_DATA = +{ + 0x0178, 0x00ff, 0x0181, 0x0253, 0x0186, 0x0254, 0x018e, 0x01dd, 0x018f, 0x0259, + 0x0190, 0x025b, 0x0193, 0x0260, 0x0194, 0x0263, 0x0196, 0x0269, 0x0197, 0x0268, + 0x019c, 0x026f, 0x019d, 0x0272, 0x019f, 0x0275, 0x01a6, 0x0280, 0x01a9, 0x0283, + 0x01ae, 0x0288, 0x01b7, 0x0292, 0x01c4, 0x01c6, 0x01c7, 0x01c9, 0x01ca, 0x01cc, + 0x01f1, 0x01f3, 0x01f6, 0x0195, 0x01f7, 0x01bf, 0x0220, 0x019e, 0x023a, 0x2c65, + 0x023d, 0x019a, 0x023e, 0x2c66, 0x0243, 0x0180, 0x0244, 0x0289, 0x0245, 0x028c, + 0x037f, 0x03f3, 0x0386, 0x03ac, 0x038c, 0x03cc, 0x03cf, 0x03d7, 0x03f9, 0x03f2, + 0x04c0, 0x04cf, 0x10c7, 0x2d27, 0x10cd, 0x2d2d, 0x1f59, 0x1f51, 0x1f5b, 0x1f53, + 0x1f5d, 0x1f55, 0x1f5f, 0x1f57, 0x1fec, 0x1fe5, 0x2132, 0x214e, 0x2c62, 0x026b, + 0x2c63, 0x1d7d, 0x2c64, 0x027d, 0x2c6d, 0x0251, 0x2c6e, 0x0271, 0x2c6f, 0x0250, + 0x2c70, 0x0252, 0xa77d, 0x1d79, 0xa78d, 0x0265, 0xa7aa, 0x0266, 0xa7ab, 0x025c, + 0xa7ac, 0x0261, 0xa7ad, 0x026c, 0xa7ae, 0x026a, 0xa7b0, 0x029e, 0xa7b1, 0x0287, + 0xa7b2, 0x029d, 0xa7b3, 0xab53 +}; + +/* Contains start points of one-to-two uppercase ranges where the second character + * is always the same. + */ +static const uint16_t jerry_upper_case_special_ranges[] JERRY_CONST_DATA = +{ + 0x1f80, 0x1f08, 0x0399, 0x1f88, 0x1f08, 0x0399, 0x1f90, 0x1f28, 0x0399, 0x1f98, + 0x1f28, 0x0399, 0x1fa0, 0x1f68, 0x0399, 0x1fa8, 0x1f68, 0x0399 +}; + +/* Interval lengths for start points in `upper_case_special_ranges` table. */ +static const uint8_t jerry_upper_case_special_range_lengths[] JERRY_CONST_DATA = +{ + 0x0007, 0x0007, 0x0007, 0x0007, 0x0007, 0x0007 +}; + +/* Contains start points of lowercase ranges. */ +static const uint16_t jerry_lower_case_ranges[] JERRY_CONST_DATA = +{ + 0x1e96, 0x1e96, 0x1f80, 0x1f80, 0x1f88, 0x1f80, 0x1f90, 0x1f90, 0x1f98, 0x1f90, + 0x1fa0, 0x1fa0, 0x1fa8, 0x1fa0, 0x1fb2, 0x1fb2, 0x1fb6, 0x1fb6, 0x1fc2, 0x1fc2, + 0x1fc6, 0x1fc6, 0x1fd2, 0x1fd2, 0x1fd6, 0x1fd6, 0x1fe2, 0x1fe2, 0x1fe6, 0x1fe6, + 0x1ff2, 0x1ff2, 0x1ff6, 0x1ff6, 0xfb00, 0xfb00, 0xfb13, 0xfb13 +}; + +/* Interval lengths for start points in `lower_case_ranges` table. */ +static const uint8_t jerry_lower_case_range_lengths[] JERRY_CONST_DATA = +{ + 0x0005, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0003, 0x0002, 0x0003, + 0x0002, 0x0002, 0x0002, 0x0003, 0x0002, 0x0003, 0x0002, 0x0007, 0x0005 +}; + +/* The remaining lowercase conversions. The lowercase variant can be one-to-three character long. */ +static const uint16_t jerry_lower_case_conversions[] JERRY_CONST_DATA = +{ + 0x00df, 0x00df, 0x0149, 0x0149, 0x01c5, 0x01c6, 0x01c8, 0x01c9, 0x01cb, 0x01cc, + 0x01f0, 0x01f0, 0x01f2, 0x01f3, 0x0390, 0x0390, 0x03b0, 0x03b0, 0x03f4, 0x03b8, + 0x0587, 0x0587, 0x1e9e, 0x00df, 0x1f50, 0x1f50, 0x1f52, 0x1f52, 0x1f54, 0x1f54, + 0x1f56, 0x1f56, 0x1fbc, 0x1fb3, 0x1fcc, 0x1fc3, 0x1ffc, 0x1ff3, 0x2126, 0x03c9, + 0x212a, 0x006b, 0x212b, 0x00e5, 0x0130, 0x0069, 0x0307 +}; + +/* Number of one-to-one, one-to-two, and one-to-three lowercase conversions. */ +static const uint8_t jerry_lower_case_conversion_counters[] JERRY_CONST_DATA = +{ + 0x0016, 0x0001, 0x0000 +}; + +/* The remaining uppercase conversions. The uppercase variant can be one-to-three character long. */ +static const uint16_t jerry_upper_case_conversions[] JERRY_CONST_DATA = +{ + 0x00b5, 0x039c, 0x0130, 0x0130, 0x0131, 0x0049, 0x017f, 0x0053, 0x01c5, 0x01c4, + 0x01c8, 0x01c7, 0x01cb, 0x01ca, 0x01f2, 0x01f1, 0x0345, 0x0399, 0x03c2, 0x03a3, + 0x03d0, 0x0392, 0x03d1, 0x0398, 0x03d5, 0x03a6, 0x03d6, 0x03a0, 0x03f0, 0x039a, + 0x03f1, 0x03a1, 0x03f5, 0x0395, 0x1c80, 0x0412, 0x1c81, 0x0414, 0x1c82, 0x041e, + 0x1c83, 0x0421, 0x1c84, 0x0422, 0x1c85, 0x0422, 0x1c86, 0x042a, 0x1c87, 0x0462, + 0x1c88, 0xa64a, 0x1e9b, 0x1e60, 0x1fbe, 0x0399, 0x00df, 0x0053, 0x0053, 0x0149, + 0x02bc, 0x004e, 0x01f0, 0x004a, 0x030c, 0x0587, 0x0535, 0x0552, 0x1e96, 0x0048, + 0x0331, 0x1e97, 0x0054, 0x0308, 0x1e98, 0x0057, 0x030a, 0x1e99, 0x0059, 0x030a, + 0x1e9a, 0x0041, 0x02be, 0x1f50, 0x03a5, 0x0313, 0x1f87, 0x1f0f, 0x0399, 0x1f8f, + 0x1f0f, 0x0399, 0x1f97, 0x1f2f, 0x0399, 0x1f9f, 0x1f2f, 0x0399, 0x1fa7, 0x1f6f, + 0x0399, 0x1faf, 0x1f6f, 0x0399, 0x1fb2, 0x1fba, 0x0399, 0x1fb3, 0x0391, 0x0399, + 0x1fb4, 0x0386, 0x0399, 0x1fb6, 0x0391, 0x0342, 0x1fbc, 0x0391, 0x0399, 0x1fc2, + 0x1fca, 0x0399, 0x1fc3, 0x0397, 0x0399, 0x1fc4, 0x0389, 0x0399, 0x1fc6, 0x0397, + 0x0342, 0x1fcc, 0x0397, 0x0399, 0x1fd6, 0x0399, 0x0342, 0x1fe4, 0x03a1, 0x0313, + 0x1fe6, 0x03a5, 0x0342, 0x1ff2, 0x1ffa, 0x0399, 0x1ff3, 0x03a9, 0x0399, 0x1ff4, + 0x038f, 0x0399, 0x1ff6, 0x03a9, 0x0342, 0x1ffc, 0x03a9, 0x0399, 0xfb00, 0x0046, + 0x0046, 0xfb01, 0x0046, 0x0049, 0xfb02, 0x0046, 0x004c, 0xfb05, 0x0053, 0x0054, + 0xfb06, 0x0053, 0x0054, 0xfb13, 0x0544, 0x0546, 0xfb14, 0x0544, 0x0535, 0xfb15, + 0x0544, 0x053b, 0xfb16, 0x054e, 0x0546, 0xfb17, 0x0544, 0x053d, 0x0390, 0x0399, + 0x0308, 0x0301, 0x03b0, 0x03a5, 0x0308, 0x0301, 0x1f52, 0x03a5, 0x0313, 0x0300, + 0x1f54, 0x03a5, 0x0313, 0x0301, 0x1f56, 0x03a5, 0x0313, 0x0342, 0x1fb7, 0x0391, + 0x0342, 0x0399, 0x1fc7, 0x0397, 0x0342, 0x0399, 0x1fd2, 0x0399, 0x0308, 0x0300, + 0x1fd3, 0x0399, 0x0308, 0x0301, 0x1fd7, 0x0399, 0x0308, 0x0342, 0x1fe2, 0x03a5, + 0x0308, 0x0300, 0x1fe3, 0x03a5, 0x0308, 0x0301, 0x1fe7, 0x03a5, 0x0308, 0x0342, + 0x1ff7, 0x03a9, 0x0342, 0x0399, 0xfb03, 0x0046, 0x0046, 0x0049, 0xfb04, 0x0046, + 0x0046, 0x004c +}; + +/* Number of one-to-one, one-to-two, and one-to-three lowercase conversions. */ +static const uint8_t jerry_upper_case_conversion_counters[] JERRY_CONST_DATA = +{ + 0x001c, 0x002c, 0x0010 +}; + diff --git a/jerry-core/profiles/minimal.profile b/jerry-core/profiles/minimal.profile index b9a877944b..d7504439b3 100644 --- a/jerry-core/profiles/minimal.profile +++ b/jerry-core/profiles/minimal.profile @@ -10,4 +10,5 @@ CONFIG_DISABLE_NUMBER_BUILTIN CONFIG_DISABLE_REGEXP_BUILTIN CONFIG_DISABLE_STRING_BUILTIN CONFIG_DISABLE_TYPEDARRAY_BUILTIN +CONFIG_DISABLE_UNICODE_CASE_CONVERSION diff --git a/tests/jerry/string-upper-lower-case-conversion.js b/tests/jerry/string-upper-lower-case-conversion.js index 3ac373f64b..75a6dea926 100644 --- a/tests/jerry/string-upper-lower-case-conversion.js +++ b/tests/jerry/string-upper-lower-case-conversion.js @@ -12,7 +12,72 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Conversion +// LATIN SMALL LIGATURES +// LATIN SMALL LIGATURE FF +assert ("\ufb00".toLowerCase() == "\ufb00"); +assert ("\ufb00".toUpperCase() == "\u0046\u0046"); +// LATIN SMALL LIGATURE FI +assert ("\ufb01".toLowerCase() == "\ufb01"); +assert ("\ufb01".toUpperCase() == "\u0046\u0049"); +// LATIN SMALL LIGATURE FL +assert ("\ufb02".toLowerCase() == "\ufb02"); +assert ("\ufb02".toUpperCase() == "\u0046\u004c"); +// LATIN SMALL LIGATURE FFI +assert ("\ufb03".toLowerCase() == "\ufb03"); +assert ("\ufb03".toUpperCase() == "\u0046\u0046\u0049"); +// LATIN SMALL LIGATURE FFL +assert ("\ufb04".toLowerCase() == "\ufb04"); +assert ("\ufb04".toUpperCase() == "\u0046\u0046\u004c"); +// LATIN SMALL LIGATURE LONG S T +assert ("\ufb05".toLowerCase() == "\ufb05"); +assert ("\ufb05".toUpperCase() == "\u0053\u0054"); +// LATIN SMALL LIGATURE ST +assert ("\ufb06".toLowerCase() == "\ufb06"); +assert ("\ufb06".toUpperCase() == "\u0053\u0054"); + +// LATIN CAPITAL LETTER I WITH DOT ABOVE +assert ("\u0130".toLowerCase() == "\u0069\u0307"); +assert ("\u0130".toUpperCase() == "\u0130"); + +// LATIN SMALL LETTER SHARP S +assert ("\u00df".toLowerCase() == "\u00df"); +assert ("\u00df".toUpperCase() == "\u0053\u0053"); + +// LATIN CAPITAL LETTER I WITH BREVE +assert ("\u012c".toLowerCase() == "\u012d"); +assert ("\u012c".toUpperCase() == "\u012c"); +// LATIN SMALL LETTER I WITH BREVE +assert ("\u012d".toLowerCase() == "\u012d") +assert ("\u012d".toUpperCase() == "\u012c"); + +// Check randomly selected characters from conversion tables + +// lower-case conversions +assert ("\u01c5\u01c8\u01cb\u212b".toLowerCase() == "\u01c6\u01c9\u01cc\u00e5"); +assert ("\u0130".toLowerCase() == "\u0069\u0307"); + +// upper-case conversions +assert ("\u00b5\u017f".toUpperCase() == "\u039c\u0053"); +assert ("\ufb17\u00df\u1fbc".toUpperCase() == "\u0544\u053D\u0053\u0053\u0391\u0399"); +assert ("\ufb03\ufb04".toUpperCase() == "\u0046\u0046\u0049\u0046\u0046\u004c"); + +// character case ranges +assert ("\u0100\u0101\u0139\u03fa\ua7b4".toLowerCase() == "\u0101\u0101\u013a\u03fb\ua7b5"); +assert ("\u0101\u0100\u013a\u03fb\ua7b5".toUpperCase() == "\u0100\u0100\u0139\u03fa\ua7b4"); + +// character pairs +assert ("\u0178\ua7b1\u0287\ua7b3".toLowerCase() == "\u00ff\u0287\u0287\uab53"); +assert ("\u00ff\u0287\ua7b1\uab53".toUpperCase() == "\u0178\ua7b1\ua7b1\ua7b3"); + +// character case ranges +assert ("\u00e0\u00c0\u00c1\u00c2\uff21".toLowerCase() == "\u00e0\u00e0\u00e1\u00e2\uff41"); +assert ("\u00e0\u00c0\u00e1\u00e2\uff41".toUpperCase() == "\u00c0\u00c0\u00c1\u00c2\uff21"); + +// lower-case ranges +assert ("\u1f88\u1f98\u1fa8\u1f8b\u1faf".toLowerCase() == "\u1f80\u1f90\u1fa0\u1f83\u1fa7"); + +// upper-case special ranges +assert ("\u1f80\u1f81\u1fa7".toUpperCase() == "\u1f08\u0399\u1f09\u0399\u1f6f\u0399"); assert ("0123456789abcdefghijklmnopqrstuvwxzyABCDEFGHIJKLMNOPQRSTUVWXYZ".toLowerCase() == "0123456789abcdefghijklmnopqrstuvwxzyabcdefghijklmnopqrstuvwxyz"); diff --git a/tools/unicode_case_conversion.py b/tools/unicode_case_conversion.py new file mode 100755 index 0000000000..473953c588 --- /dev/null +++ b/tools/unicode_case_conversion.py @@ -0,0 +1,681 @@ +#!/usr/bin/env python + +# Copyright JS Foundation and other contributors, http://js.foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import csv +import itertools +import os +import sys +import warnings + +try: + unichr +except NameError: + unichr = chr + +TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) +PROJECT_DIR = os.path.normpath(os.path.join(TOOLS_DIR, '..')) +C_SOURCE_FILE = os.path.join(PROJECT_DIR, 'jerry-core/lit/lit-unicode-conversions.inc.h') + + +def parse_unicode_sequence(raw_data): + """ + Parse unicode sequence from raw data. + + :param raw_data: Contains the unicode sequence which needs to parse. + :return: The parsed unicode sequence. + """ + + result = '' + + for unicode_char in raw_data.split(' '): + if unicode_char == '': + continue + + # Convert it to unicode code point (from hex value without 0x prefix) + result += unichr(int(unicode_char, 16)) + return result + + +def read_case_mappings(unicode_data_file, special_casing_file): + """ + Read the corresponding unicode values of lower and upper case letters and store these in tables. + + :param unicode_data_file: Contains the default case mappings (one-to-one mappings). + :param special_casing_file: Contains additional informative case mappings that are either not one-to-one + or which are context-sensitive. + :return: Upper and lower case mappings. + """ + + lower_case_mapping = CaseMapping() + upper_case_mapping = CaseMapping() + + # Add one-to-one mappings + with open(unicode_data_file) as unicode_data: + unicode_data_reader = csv.reader(unicode_data, delimiter=';') + + for line in unicode_data_reader: + letter_id = int(line[0], 16) + + # Skip supplementary planes and ascii chars + if letter_id >= 0x10000 or letter_id < 128: + continue + + capital_letter = line[12] + small_letter = line[13] + + if capital_letter: + upper_case_mapping.add(letter_id, parse_unicode_sequence(capital_letter)) + + if small_letter: + lower_case_mapping.add(letter_id, parse_unicode_sequence(small_letter)) + + # Update the conversion tables with the special cases + with open(special_casing_file) as special_casing: + special_casing_reader = csv.reader(special_casing, delimiter=';') + + for line in special_casing_reader: + # Skip comment sections and empty lines + if not line or line[0].startswith('#'): + continue + + # Replace '#' character with empty string + for idx, i in enumerate(line): + if i.find('#') >= 0: + line[idx] = '' + + letter_id = int(line[0], 16) + condition_list = line[4] + + # Skip supplementary planes, ascii chars, and condition_list + if letter_id >= 0x10000 or letter_id < 128 or condition_list: + continue + + small_letter = parse_unicode_sequence(line[1]) + capital_letter = parse_unicode_sequence(line[3]) + + lower_case_mapping.add(letter_id, small_letter) + upper_case_mapping.add(letter_id, capital_letter) + + return lower_case_mapping, upper_case_mapping + + +class CaseMapping(dict): + """Class defines an informative, default mapping.""" + + def __init__(self): + """Initialize the case mapping table.""" + self._conversion_table = {} + + def add(self, letter_id, mapped_value): + """ + Add mapped value of the unicode letter. + + :param letter_id: An integer, representing the unicode code point of the character. + :param mapped_value: Corresponding character of the case type. + """ + self._conversion_table[letter_id] = mapped_value + + def remove(self, letter_id): + """ + Remove mapping from the conversion table. + + :param letter_id: An integer, representing the unicode code point of the character. + """ + del self._conversion_table[letter_id] + + def get_value(self, letter_id): + """ + Get the mapped value of the given unicode character. + + :param letter_id: An integer, representing the unicode code point of the character. + :return: The mapped value of the character. + """ + + if self.contains(letter_id): + return self._conversion_table[letter_id] + + return None + + def get_conversion_distance(self, letter_id): + """ + Calculate the distance between the unicode character and its mapped value + (only needs and works with one-to-one mappings). + + :param letter_id: An integer, representing the unicode code point of the character. + :return: The conversion distance. + """ + + mapped_value = self.get_value(letter_id) + + if mapped_value and len(mapped_value) == 1: + return ord(mapped_value) - letter_id + + return None + + def is_bidirectional_conversion(self, letter_id, other_case_mapping): + """ + Check that two unicode value are also a mapping value of each other. + + :param letter_id: An integer, representing the unicode code point of the character. + :param other_case_mapping: Comparable case mapping table which possible contains + the return direction of the conversion. + :return: True, if it's a reverible conversion, false otherwise. + """ + + if not self.contains(letter_id): + return False + + # Check one-to-one mapping + mapped_value = self.get_value(letter_id) + if len(mapped_value) > 1: + return False + + # Check two way conversions + mapped_value_id = ord(mapped_value) + if other_case_mapping.get_value(mapped_value_id) != unichr(letter_id): + return False + + return True + + def contains(self, letter_id): + """ + Check that a unicode character is in the conversion table. + + :param letter_id: An integer, representing the unicode code point of the character. + :return: True, if it contains the character, false otherwise. + """ + if letter_id in self._conversion_table: + return True + + return False + + def get_table(self): + return self._conversion_table + + def extract_ranges(self, other_case_mapping=None): + """ + Extract ranges from case mappings + (the second param is optional, if it's not empty, a range will contains bidirectional conversions only). + + :param letter_id: An integer, representing the unicode code point of the character. + :param other_case_mapping: Comparable case mapping table which contains the return direction of the conversion. + :return: A table with the start points and their mapped value, and another table with the lengths of the ranges. + """ + + in_range = False + range_position = -1 + ranges = [] + range_lengths = [] + + for letter_id in sorted(self._conversion_table.keys()): + prev_letter_id = letter_id - 1 + + # One-way conversions + if other_case_mapping is None: + if len(self.get_value(letter_id)) > 1: + in_range = False + continue + + if not self.contains(prev_letter_id) or len(self.get_value(prev_letter_id)) > 1: + in_range = False + continue + + # Two way conversions + else: + if not self.is_bidirectional_conversion(letter_id, other_case_mapping): + in_range = False + continue + + if not self.is_bidirectional_conversion(prev_letter_id, other_case_mapping): + in_range = False + continue + + conv_distance = self.get_conversion_distance(letter_id) + prev_conv_distance = self.get_conversion_distance(prev_letter_id) + + if (conv_distance != prev_conv_distance): + in_range = False + continue + + if in_range: + range_lengths[range_position] += 1 + else: + in_range = True + range_position += 1 + + # Add the start point of the range and its mapped value + ranges.extend([prev_letter_id, ord(self.get_value(prev_letter_id))]) + range_lengths.append(2) + + # Remove all ranges from the case mapping table. + index = 0 + while index != len(ranges): + range_length = range_lengths[index // 2] + + for incr in range(range_length): + self.remove(ranges[index] + incr) + if other_case_mapping is not None: + other_case_mapping.remove(ranges[index + 1] + incr) + + index += 2 + + return ranges, range_lengths + + def extract_character_pair_ranges(self, other_case_mapping): + """ + Extract two or more character pairs from the case mapping tables. + + :param other_case_mapping: Comparable case mapping table which contains the return direction of the conversion. + :return: A table with the start points, and another table with the lengths of the ranges. + """ + + start_points = [] + lengths = [] + in_range = False + element_counter = -1 + + for letter_id in sorted(self._conversion_table.keys()): + # Only extract character pairs + if not self.is_bidirectional_conversion(letter_id, other_case_mapping): + in_range = False + continue + + if self.get_value(letter_id) == unichr(letter_id + 1): + prev_letter_id = letter_id - 2 + + if not self.is_bidirectional_conversion(prev_letter_id, other_case_mapping): + in_range = False + + if in_range: + lengths[element_counter] += 2 + else: + element_counter += 1 + start_points.append(letter_id) + lengths.append(2) + in_range = True + + else: + in_range = False + + # Remove all founded case mapping from the conversion tables after the scanning method + idx = 0 + while idx != len(start_points): + letter_id = start_points[idx] + conv_length = lengths[idx] + + for incr in range(0, conv_length, 2): + self.remove(letter_id + incr) + other_case_mapping.remove(letter_id + 1 + incr) + + idx += 1 + + return start_points, lengths + + def extract_character_pairs(self, other_case_mapping): + """ + Extract character pairs. Check that two unicode value are also a mapping value of each other. + + :param other_case_mapping: Comparable case mapping table which contains the return direction of the conversion. + :return: A table with character pairs. + """ + + character_pairs = [] + + for letter_id in sorted(self._conversion_table.keys()): + if self.is_bidirectional_conversion(letter_id, other_case_mapping): + mapped_value = self.get_value(letter_id) + character_pairs.extend([letter_id, ord(mapped_value)]) + + # Remove character pairs from case mapping tables + self.remove(letter_id) + other_case_mapping.remove(ord(mapped_value)) + + return character_pairs + + def extract_special_ranges(self): + """ + Extract special ranges. It contains that ranges of one-to-two mappings where the second character + of the mapped values are equals and the other characters are following each other. + eg.: \u1f80 and \u1f81 will be in one range becase their upper-case values are \u1f08\u0399 and \u1f09\u0399 + + :return: A table with the start points and their mapped values, and a table with the lengths of the ranges. + """ + + special_ranges = [] + special_range_lengths = [] + + range_position = -1 + + for letter_id in sorted(self._conversion_table.keys()): + mapped_value = self.get_value(letter_id) + + if len(mapped_value) != 2: + continue + + prev_letter_id = letter_id - 1 + + if not self.contains(prev_letter_id): + in_range = False + continue + + prev_mapped_value = self.get_value(prev_letter_id) + + if len(prev_mapped_value) != 2: + continue + + if prev_mapped_value[1] != mapped_value[1]: + continue + + if (ord(prev_mapped_value[0]) - prev_letter_id) != (ord(mapped_value[0]) - letter_id): + in_range = False + continue + + if in_range: + special_range_lengths[range_position] += 1 + else: + range_position += 1 + in_range = True + + special_ranges.extend([prev_letter_id, ord(prev_mapped_value[0]), ord(prev_mapped_value[1])]) + special_range_lengths.append(1) + + # Remove special ranges from the conversion table + idx = 0 + while idx != len(special_ranges): + range_length = special_range_lengths[idx // 3] + letter_id = special_ranges[idx] + + for incr in range(range_length): + self.remove(special_ranges[idx] + incr) + + idx += 3 + + return special_ranges, special_range_lengths + + def extract_conversions(self): + """ + Extract conversions. It provide the full (or remained) case mappings from the table. + The counter table contains the information of how much one-to-one, one-to-two or one-to-three mappings + exists successively in the conversion table. + + :return: A table with conversions, and a table with counters. + """ + + unicodes = [[], [], []] + unicode_lengths = [0, 0, 0] + + # 1 to 1 byte + for letter_id in sorted(self._conversion_table.keys()): + mapped_value = self.get_value(letter_id) + + if len(mapped_value) != 1: + continue + + unicodes[0].extend([letter_id, ord(mapped_value)]) + self.remove(letter_id) + + # 1 to 2 bytes + for letter_id in sorted(self._conversion_table.keys()): + mapped_value = self.get_value(letter_id) + + if len(mapped_value) != 2: + continue + + unicodes[1].extend([letter_id, ord(mapped_value[0]), ord(mapped_value[1])]) + self.remove(letter_id) + + # 1 to 3 bytes + for letter_id in sorted(self._conversion_table.keys()): + mapped_value = self.get_value(letter_id) + + if len(mapped_value) != 3: + continue + + unicodes[2].extend([letter_id, ord(mapped_value[0]), ord(mapped_value[1]), ord(mapped_value[2])]) + self.remove(letter_id) + + unicode_lengths = [int(len(unicodes[0]) / 2), int(len(unicodes[1]) / 3), int(len(unicodes[2]) / 4)] + + return list(itertools.chain.from_iterable(unicodes)), unicode_lengths + + +def regroup(l, n): + return [l[i:i+n] for i in range(0, len(l), n)] + + +def hex_format(ch): + if isinstance(ch, str): + ch = ord(ch) + + return "0x{:04x}".format(ch) + + +def format_code(code, indent): + lines = [] + # convert all characters to hex format + converted_code = map(hex_format, code) + # 10 hex number per line + for line in regroup(", ".join(converted_code), 10 * 8): + lines.append((' ' * indent) + line.strip()) + return "\n".join(lines) + + +def create_c_format_table(type_name, array_name, table, description=""): + return """{DESC} +static const {TYPE} jerry_{NAME}[] JERRY_CONST_DATA = +{{ +{TABLE} +}}; + +""".format(DESC=description, TYPE=type_name, NAME=array_name, TABLE=format_code(table, 1)) + + +def copy_tables_to_c_source(gen_tables, c_source): + data = [] + + header = """/* Copyright JS Foundation and other contributors, http://js.foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * This file is automatically generated by the {SCRIPT} script. Do not edit! + */ + +""".format(SCRIPT=os.path.basename(__file__)) + + data.append(header) + + character_case_ranges = gen_tables.get_character_case_ranges() + character_pair_ranges = gen_tables.get_character_pair_ranges() + character_pairs = gen_tables.get_character_pairs() + upper_case_special_ranges = gen_tables.get_upper_case_special_ranges() + lower_case_ranges = gen_tables.get_lower_case_ranges() + lower_case_conversions = gen_tables.get_lower_case_conversions() + upper_case_conversions = gen_tables.get_upper_case_conversions() + + description = "/* Contains start points of character case ranges (these are bidirectional conversions). */" + data.append(create_c_format_table('uint16_t', 'character_case_ranges', + character_case_ranges[0], + description)) + + description = "/* Interval lengths of start points in `character_case_ranges` table. */" + data.append(create_c_format_table('uint8_t', + 'character_case_range_lengths', + character_case_ranges[1], + description)) + + description = "/* Contains the start points of bidirectional conversion ranges. */" + data.append(create_c_format_table('uint16_t', + 'character_pair_ranges', + character_pair_ranges[0], + description)) + + description = "/* Interval lengths of start points in `character_pair_ranges` table. */" + data.append(create_c_format_table('uint8_t', + 'character_pair_range_lengths', + character_pair_ranges[1], + description)) + + description = "/* Contains lower/upper case bidirectional conversion pairs. */" + data.append(create_c_format_table('uint16_t', + 'character_pairs', + character_pairs, + description)) + + description = """/* Contains start points of one-to-two uppercase ranges where the second character + * is always the same. + */""" + data.append(create_c_format_table('uint16_t', + 'upper_case_special_ranges', + upper_case_special_ranges[0], + description)) + + description = "/* Interval lengths for start points in `upper_case_special_ranges` table. */" + data.append(create_c_format_table('uint8_t', + 'upper_case_special_range_lengths', + upper_case_special_ranges[1], + description)) + + description = "/* Contains start points of lowercase ranges. */" + data.append(create_c_format_table('uint16_t', 'lower_case_ranges', lower_case_ranges[0], description)) + + description = "/* Interval lengths for start points in `lower_case_ranges` table. */" + data.append(create_c_format_table('uint8_t', 'lower_case_range_lengths', lower_case_ranges[1], description)) + + description = "/* The remaining lowercase conversions. The lowercase variant can be one-to-three character long. */" + data.append(create_c_format_table('uint16_t', + 'lower_case_conversions', + lower_case_conversions[0], + description)) + + description = "/* Number of one-to-one, one-to-two, and one-to-three lowercase conversions. */" + + data.append(create_c_format_table('uint8_t', + 'lower_case_conversion_counters', + lower_case_conversions[1], + description)) + + description = "/* The remaining uppercase conversions. The uppercase variant can be one-to-three character long. */" + data.append(create_c_format_table('uint16_t', + 'upper_case_conversions', + upper_case_conversions[0], + description)) + + description = "/* Number of one-to-one, one-to-two, and one-to-three lowercase conversions. */" + data.append(create_c_format_table('uint8_t', + 'upper_case_conversion_counters', + upper_case_conversions[1], + description)) + + with open(c_source, 'w') as genereted_source: + genereted_source.write(''.join(data)) + + +class GenTables(object): + """Class defines an informative, default generated tables.""" + + def __init__(self, lower_case_table, upper_case_table): + """ + Generate the extracted tables from the given case mapping tables. + + :param lower_case_table: Lower-case mappings. + :param upper_case_table: Upper-case mappings. + """ + + self._character_case_ranges = lower_case_table.extract_ranges(upper_case_table) + self._character_pair_ranges = lower_case_table.extract_character_pair_ranges(upper_case_table) + self._character_pairs = lower_case_table.extract_character_pairs(upper_case_table) + self._upper_case_special_ranges = upper_case_table.extract_special_ranges() + self._lower_case_ranges = lower_case_table.extract_ranges() + self._lower_case_conversions = lower_case_table.extract_conversions() + self._upper_case_conversions = upper_case_table.extract_conversions() + + if lower_case_table.get_table(): + warnings.warn('Not all elements extracted from the lowercase conversion table!') + if upper_case_table.get_table(): + warnings.warn('Not all elements extracted from the uppercase conversion table!') + + def get_character_case_ranges(self): + return self._character_case_ranges + + def get_character_pair_ranges(self): + return self._character_pair_ranges + + def get_character_pairs(self): + return self._character_pairs + + def get_upper_case_special_ranges(self): + return self._upper_case_special_ranges + + def get_lower_case_ranges(self): + return self._lower_case_ranges + + def get_lower_case_conversions(self): + return self._lower_case_conversions + + def get_upper_case_conversions(self): + return self._upper_case_conversions + + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument('--unicode-data', + metavar='FILE', + action='store', + required=True, + help='specify the unicode data file') + + parser.add_argument('--special-casing', + metavar='FILE', + action='store', + required=True, + help='specify the special casing file') + + parser.add_argument('--c-source', + metavar='FILE', + action='store', + default=C_SOURCE_FILE, + help='specify the output c source (default: %(default)s)') + + script_args = parser.parse_args() + + if not os.path.isfile(script_args.unicode_data) or not os.access(script_args.unicode_data, os.R_OK): + print('The %s file is missing or not readable!' % script_args.unicode_data) + sys.exit(1) + + if not os.path.isfile(script_args.special_casing) or not os.access(script_args.special_casing, os.R_OK): + print('The %s file is missing or not readable!' % script_args.special_casing) + sys.exit(1) + + lower_case_table, upper_case_table = read_case_mappings(script_args.unicode_data, script_args.special_casing) + + gen_tables = GenTables(lower_case_table, upper_case_table) + + copy_tables_to_c_source(gen_tables, script_args.c_source) + +if __name__ == "__main__": + main()