--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/make/jdk/src/classes/build/tools/generatecharacter/GenerateCharacter.java Tue Sep 12 19:03:39 2017 +0200
@@ -0,0 +1,1993 @@
+/*
+ * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package build.tools.generatecharacter;
+
+import java.io.IOException;
+import java.io.FileNotFoundException;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.PrintWriter;
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.File;
+import java.util.List;
+
+import build.tools.generatecharacter.CharacterName;
+
+/**
+ * This program generates the source code for the class java.lang.Character.
+ * It also generates native C code that can perform the same operations.
+ * It requires two external input data files:
+ * <ul>
+ * <li> Unicode specification file
+ * <li> Character class template file
+ * </ul>
+ * The Unicode specification file is available from the Unicode consortium.
+ * It has character specification lines that look like this:
+ * <listing>
+ * 0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061;
+ * </listing>
+ * The Character class template file is filled in with additional
+ * information to produce the file Character.java, which can then be
+ * compiled by a Java compiler. The template file contains certain
+ * markers consisting of an alphabetic name string preceded by "$$".
+ * Such markers are replaced with generated program text. As a special
+ * case, the marker "Lookup(xxx)" is recognized, where "xxx" consists of
+ * alphabetic characters constituting a variable name. The character "_"
+ * is considered alphabetic for these purposes.
+ *
+ * @author Guy Steele
+ * @author Alan Liu
+ * @author John O'Conner
+ */
+
+public class GenerateCharacter {
+
+ final static boolean DEBUG = false;
+
+ final static String commandMarker = "$$";
+ static String ROOT = "";
+ static String DefaultUnicodeSpecFileName = ROOT + "UnicodeData.txt";
+ static String DefaultSpecialCasingFileName = ROOT + "SpecialCasing.txt";
+ static String DefaultPropListFileName = ROOT + "PropList.txt";
+ static String DefaultJavaTemplateFileName = ROOT + "Character.java.template";
+ static String DefaultJavaOutputFileName = ROOT + "Character.java";
+ static String DefaultCTemplateFileName = ROOT + "Character.c.template";
+ static String DefaultCOutputFileName = ROOT + "Character.c";
+
+ static int plane = 0;
+
+ /* The overall idea is that, in the generated Character class source code,
+ most character property data is stored in a special multi-level table whose
+ structure is defined by a sequence of nonnegative integers [k1, k2, ..., kn].
+ The integers must sum to 16 (the number of bits in a character).
+ The first table is indexed by the k1 high-order bits of the character code.
+ The result is concatenated to the next k2 bits of the character code to index
+ the second table, and so on. Eventually the kn low-order bits of the character
+ code are concatenated and used to index one of two tables A and B; A contains
+ 32-bit integer entries and B contains 16-bit short entries. The 48 bits that
+ can be thus obtained encode the properties for the character.
+
+ The default specification is [9, 4, 3, 0]. This particular table format was
+ designed by conducting an exhaustive search of table formats to minimize the
+ space consumed by the tables: the first and third tables need have only byte
+ values (the second table must have short values). Another good choice is
+ [10, 6, 0], which produces a larger table but allows particularly fast table
+ lookup code.
+
+ In each case, where the word "concatenated" is used, this may imply
+ first a << and then a | operation, or perhaps just a | operation if
+ the values in the table can be preshifted (generally possible if the table
+ entries are short rather than byte).
+ */
+
+ /* The character properties are currently encoded into A (32 bits)and B (16 bits)
+ two parts.
+
+ A: the low 32 bits are defined in the following manner:
+
+ 1 bit Mirrored property.
+ 4 bits Bidirectional category (see below) (unused if -nobidi switch specified)
+ 9 bits A signed offset used for converting case .
+ 1 bit If 1, adding the signed offset converts the character to lowercase.
+ 1 bit If 1, subtracting the signed offset converts the character to uppercase.
+ Note: for a titlecase character, both of the preceding bits will be 1
+ and the signed offset will be 1.
+ 1 bit If 1, this character has a titlecase equivalent (possibly itself);
+ in this case, the two bits before this bit can be used to decide
+ whether this character is in fact uppercase, lowercase, or titlecase.
+ 3 bits This field provides a quick way to lex identifiers.
+ The eight possible values for this field are as follows:
+ 0 May not be part of an identifier
+ 1 Ignorable control; may continue a Unicode identifier or Java identifier
+ 2 May continue a Java identifier but not a Unicode identifier (unused)
+ 3 May continue a Unicode identifier or Java identifier
+ 4 Is a Java whitespace character
+ 5 May start or continue a Java identifier;
+ may continue but not start a Unicode identifier
+ (this value is used for connector punctuation such as _)
+ 6 May start or continue a Java identifier;
+ may not occur in a Unicode identifier
+ (this value is used for currency symbols such as $)
+ 7 May start or continue a Unicode identifier or Java identifier
+ Thus:
+ 5, 6, 7 may start a Java identifier
+ 1, 2, 3, 5, 6, 7 may continue a Java identifier
+ 7 may start a Unicode identifier
+ 1, 3, 5, 7 may continue a Unicode identifier
+ 1 is ignorable within an identifier
+ 4 is Java whitespace
+ 2 bits This field indicates whether the character has a numeric property.
+ The four possible values for this field are as follows:
+ 0 This character has no numeric property.
+ 1 Adding the digit offset to the character code and then
+ masking with 0x1F will produce the desired numeric value.
+ 2 This character has a "strange" numeric value.
+ 3 A Java supradecimal digit: adding the digit offset to the
+ character code, then masking with 0x1F, then adding 10
+ will produce the desired numeric value.
+ 5 bits The digit offset (see description of previous field)
+ 5 bits Character type (see below)
+
+ B: the high 16 bits are defined as:
+ 1 bit Other_Lowercase property
+ 1 bit Other_Uppercase property
+ 1 bit Other_Alphabetic property
+ 1 bit Other_Math property
+ 1 bit Ideographic property
+ 1 bit Noncharacter codepoint property
+ */
+
+
+ // bit masks identify each component of a 32-bit property field described
+ // above.
+ // shift* indicates how many shifts right must happen to get the
+ // indicated property value in the lowest bits of the 32-bit space.
+ private static final int
+ shiftType = 0, maskType = 0x001F,
+ shiftDigitOffset = 5, maskDigitOffset = 0x03E0,
+ shiftNumericType = 10, maskNumericType = 0x0C00,
+ shiftIdentifierInfo = 12, maskIdentifierInfo = 0x7000,
+ maskUnicodePart = 0x1000,
+ shiftCaseInfo = 15, maskCaseInfo = 0x38000,
+ maskLowerCase = 0x20000,
+ maskUpperCase = 0x10000,
+ maskTitleCase = 0x08000,
+ shiftCaseOffset = 18, maskCaseOffset = 0x07FC0000,
+ shiftCaseOffsetSign = 5,
+ // used only when calculating and
+ // storing digit offsets from char values
+ maskDigit = 0x001F,
+ // case offset are 9 bits
+ maskCase = 0x01FF,
+ shiftBidi = 27, maskBidi = 0x78000000,
+ shiftMirrored = 31, //maskMirrored = 0x80000000,
+ shiftPlane = 16, maskPlane = 0xFF0000;
+
+ // maskMirrored needs to be long, if up 16-bit
+ private static final long maskMirrored = 0x80000000L;
+
+ // bit masks identify the 16-bit priperty field described above, in B
+ // table
+ private static final long
+ maskOtherLowercase = 0x100000000L,
+ maskOtherUppercase = 0x200000000L,
+ maskOtherAlphabetic = 0x400000000L,
+ maskOtherMath = 0x800000000L,
+ maskIdeographic = 0x1000000000L,
+ maskNoncharacterCP = 0x2000000000L;
+
+ // Can compare masked values with these to determine
+ // numeric or lexical types.
+ public static int
+ valueNotNumeric = 0x0000,
+ valueDigit = 0x0400,
+ valueStrangeNumeric = 0x0800,
+ valueJavaSupradecimal = 0x0C00,
+ valueIgnorable = 0x1000,
+ valueJavaOnlyPart = 0x2000,
+ valueJavaUnicodePart = 0x3000,
+ valueJavaWhitespace = 0x4000,
+ valueJavaStartUnicodePart = 0x5000,
+ valueJavaOnlyStart = 0x6000,
+ valueJavaUnicodeStart = 0x7000,
+ lowJavaStart = 0x5000,
+ nonzeroJavaPart = 0x3000,
+ valueUnicodeStart = 0x7000;
+
+ // these values are used when only identifier properties are generated
+ // for use in verifier code. Shortens the property down to a single byte.
+ private static final int
+ bitJavaStart = 0x02,
+ bitJavaPart = 0x01,
+ maskIsJavaIdentifierPart = bitJavaPart,
+ maskIsJavaIdentifierStart = bitJavaStart;
+
+ static int maxOffset = maskCase/2 ;
+ static int minOffset = -maxOffset;
+
+ /* The following routines provide simple, concise formatting of long integer values.
+ The number in the name of the method indicates the desired number of characters
+ to be produced. If the number of digits required to represent the integer value
+ is less than that number, then the output is padded on the left with zeros
+ (for hex) or with spaces (for decimal). If the number of digits required to
+ represent the integer value is greater than the desired number, then all the digits
+ that are required are actually produced.
+ */
+
+ static String hex(long n) { return Long.toHexString(n).toUpperCase(); }
+
+ static String hex2(long n) {
+ String q = Long.toHexString(n & 0xFF).toUpperCase();
+ return "00".substring(Math.min(2, q.length())) + q;
+ }
+
+ static String hex4(long n) {
+ String q = Long.toHexString(n & 0xFFFF).toUpperCase();
+ return "0000".substring(Math.min(4, q.length())) + q;
+ }
+
+ static String hex8(long n) {
+ String q = Long.toHexString(n & 0xFFFFFFFFL).toUpperCase();
+ return "00000000".substring(Math.min(8, q.length())) + q;
+ }
+
+ static String hex16(long n) {
+ String q = Long.toHexString(n).toUpperCase();
+ return "0000000000000000".substring(Math.min(16, q.length())) + q;
+ }
+
+ static String dec3(long n) {
+ String q = Long.toString(n);
+ return " ".substring(Math.min(3, q.length())) + q;
+ }
+
+ static String dec5(long n) {
+ String q = Long.toString(n);
+ return " ".substring(Math.min(5, q.length())) + q;
+ }
+
+ /* This routine is called when some failure occurs. */
+
+ static void FAIL(String s) {
+ System.out.println("** " + s);
+ }
+
+ /**
+ * Given the data from the Unicode specification file, this routine builds a map.
+ *
+ * The specification file is assumed to contain its data in sorted order by
+ * character code; as a result, the array passed as an argument to this method
+ * has its components in the same sorted order, with one entry for each defined
+ * Unicode character or character range. (A range is indicated by two consecutive
+ * entries, such that the name of the first entry begins with "<" and ends with
+ * "First>" and the second entry begins with "<" and ends with "Last>".) This is
+ * therefore a sparse representation of the character property data.
+ *
+ * The resulting map is dense representation of the character data. It contains
+ * 2^16 = 65536 entries, each of which is a long integer. (Right now only 32 bits
+ * of this long value are used, but type long is used rather than int to facilitate
+ * future extensions of this source code generator that might require more than
+ * 32 bits to encode relevant character properties.) Entry k holds the encoded
+ * properties for character k.
+ *
+ * Method buildMap manages the transformation from the sparse representation to
+ * the dense representation. It calls method buildOne to handle the encoding
+ * of character property data from a single UnicodeSpec object into 32 bits.
+ * For undefined characters, method buildOne is not called and the map entry for
+ * that character is set to UnicodeSpec.UNASSIGNED.
+ *
+ * @param data character property data from the Unicode specification file
+ * @return an array of length 65536 with one entry for every possible char value
+ *
+ * @see GenerateCharacter#buildOne
+ */
+
+ static long[] buildMap(UnicodeSpec[] data, SpecialCaseMap[] specialMaps, PropList propList)
+ {
+ long[] result;
+ if (bLatin1 == true) {
+ result = new long[256];
+ } else {
+ result = new long[1<<16];
+ }
+ int k=0;
+ int codePoint = plane<<16;
+ UnicodeSpec nonCharSpec = new UnicodeSpec();
+ for (int j = 0; j < data.length && k < result.length; j++) {
+ if (data[j].codePoint == codePoint) {
+ result[k] = buildOne(codePoint, data[j], specialMaps);
+ ++k;
+ ++codePoint;
+ }
+ else if(data[j].codePoint > codePoint) {
+ if (data[j].name.endsWith("Last>")) {
+ // build map data for all chars except last in range
+ while (codePoint < data[j].codePoint && k < result.length) {
+ result[k] = buildOne(codePoint, data[j], specialMaps);
+ ++k;
+ ++codePoint;
+ }
+ }
+ else {
+ // we have a few unassigned chars before data[j].codePoint
+ while (codePoint < data[j].codePoint && k < result.length) {
+ result[k] = buildOne(codePoint, nonCharSpec, specialMaps);
+ ++k;
+ ++codePoint;
+ }
+ }
+ k = data[j].codePoint & 0xFFFF;
+ codePoint = data[j].codePoint;
+ result[k] = buildOne(codePoint, data[j], specialMaps);
+ ++k;
+ ++codePoint;
+ }
+ else {
+ System.out.println("An error has occured during spec mapping.");
+ System.exit(0);
+ }
+ }
+ // if there are still unprocessed chars, process them
+ // as unassigned/undefined.
+ codePoint = (plane<<16) | k;
+ while (k < result.length) {
+ result[k] = buildOne(codePoint, nonCharSpec, specialMaps);
+ ++k;
+ ++codePoint;
+ }
+ // now add all extra supported properties from PropList, to the
+ // upper 16-bit
+ addExProp(result, propList, "Other_Lowercase", maskOtherLowercase);
+ addExProp(result, propList, "Other_Uppercase", maskOtherUppercase);
+ addExProp(result, propList, "Other_Alphabetic", maskOtherAlphabetic);
+ addExProp(result, propList, "Ideographic", maskIdeographic);
+ //addExProp(result, propList, "Other_Math", maskOtherMath);
+ //addExProp(result, propList, "Noncharacter_CodePoint", maskNoncharacterCP);
+
+ return result;
+ }
+
+ // The maximum and minimum offsets found while scanning the database
+ static int maxOffsetSeen = 0;
+ static int minOffsetSeen = 0;
+
+ /**
+ * Some Unicode separator characters are not considered Java whitespace.
+ * @param c character to test
+ * @return true if c in an invalid Java whitespace character, false otherwise.
+ */
+ static boolean isInvalidJavaWhiteSpace(int c) {
+ int[] exceptions = {0x00A0, 0x2007, 0x202F, 0xFEFF};
+ boolean retValue = false;
+ for(int x=0;x<exceptions.length;x++) {
+ if(c == exceptions[x]) {
+ retValue = true;
+ break;
+ }
+ }
+ return retValue;
+
+ }
+
+ /**
+ * Given the character property data for one Unicode character, encode the data
+ * of interest into a single long integer value. (Right now only 32 bits
+ * of this long value are used, but type long is used rather than int to facilitate
+ * future extensions of this source code generator that might require more than
+ * 32 bits to encode relevant character properties.)
+ *
+ * @param c the character code for which to encode property data
+ * @param us property data record from the Unicode specification file
+ * (its character code might not be equal to c if it specifies data
+ * for a range of characters)
+ * @return an encoded long value that contains the properties for a single char
+ *
+ * @see GenerateCharacter#buildMap
+ */
+
+ static long buildOne(int c, UnicodeSpec us, SpecialCaseMap[] specialMaps) {
+ long resultA = 0;
+ // record the general category
+ resultA |= us.generalCategory;
+
+ // record the numeric properties
+ NUMERIC: {
+ STRANGE: {
+ int val = 0;
+ // c is A-Z
+ if ((c >= 0x0041) && (c <= 0x005A)) {
+ val = c - 0x0041;
+ resultA |= valueJavaSupradecimal;
+ // c is a-z
+ } else if ((c >= 0x0061) && (c <= 0x007A)) {
+ val = c - 0x0061;
+ resultA |= valueJavaSupradecimal;
+ // c is a full-width A-Z
+ } else if ((c >= 0xFF21) && (c <= 0xFF3A)) {
+ val = c - 0xFF21;
+ resultA |= valueJavaSupradecimal;
+ // c is a full-width a-z
+ } else if ((c >= 0xFF41) && (c <= 0xFF5A)) {
+ val = c - 0xFF41;
+ resultA |= valueJavaSupradecimal;
+ } else if (us.isDecimalValue()) {
+ val = us.decimalValue;
+ resultA |= valueDigit;
+ } else if (us.isDigitValue()) {
+ val = us.digitValue;
+ resultA |= valueDigit;
+ } else {
+ if (us.numericValue.length() == 0) {
+ break NUMERIC; // no numeric value at all
+ } else {
+ try {
+ val = Integer.parseInt(us.numericValue);
+ if (val >= 32 || val < 0) break STRANGE;
+ if (c == 0x215F) break STRANGE;
+ } catch(NumberFormatException e) {
+ break STRANGE;
+ }
+ resultA |= valueDigit;
+ }
+ }
+ if (val >= 32 || val < 0) break STRANGE;
+ resultA |= ((val - c & maskDigit) << shiftDigitOffset);
+ break NUMERIC;
+ } // end STRANGE
+ resultA |= valueStrangeNumeric;
+ } // end NUMERIC
+
+ // record case mapping
+ int offset = 0;
+ // might have a 1:M mapping
+ int specialMap = SpecialCaseMap.find(c, specialCaseMaps);
+ boolean bHasUpper = (us.hasUpperMap()) || (specialMap != -1);
+ if (bHasUpper) {
+ resultA |= maskUpperCase;
+ }
+ if (specialMap != -1) {
+ // has mapping, but cannot record the
+ // proper offset; can only flag it and provide special case
+ // code in Character.java
+ offset = -1;
+ }
+ else if (us.hasUpperMap()) {
+ offset = c - us.upperMap;
+ }
+
+ if (us.hasLowerMap()) {
+ resultA |= maskLowerCase;
+ if (offset == 0)
+ offset = us.lowerMap - c;
+ else if (offset != (us.lowerMap - c)) {
+ if (DEBUG) {
+ FAIL("Character " + hex(c) +
+ " has incompatible lowercase and uppercase mappings");
+ }
+ }
+ }
+ if ((us.hasTitleMap() && us.titleMap != us.upperMap) ||
+ (bHasUpper && us.hasLowerMap())) {
+ resultA |= maskTitleCase;
+ }
+ if (bHasUpper && !us.hasLowerMap() && !us.hasTitleMap() && verbose) {
+ System.out.println("Warning: Character " + hex4(c) + " has upper but " +
+ "no title case; Java won't know this");
+ }
+ if (offset < minOffsetSeen) minOffsetSeen = offset;
+ if (offset > maxOffsetSeen) maxOffsetSeen = offset;
+ if (offset > maxOffset || offset < minOffset) {
+ if (DEBUG) {
+ FAIL("Case offset " + offset + " for character " + hex4(c) + " must be handled as a special case");
+ }
+ offset = maskCase;
+ }
+ resultA |= ((offset & maskCase) << shiftCaseOffset);
+
+ // record lexical info about this character
+ if (us.generalCategory == UnicodeSpec.LOWERCASE_LETTER
+ || us.generalCategory == UnicodeSpec.UPPERCASE_LETTER
+ || us.generalCategory == UnicodeSpec.TITLECASE_LETTER
+ || us.generalCategory == UnicodeSpec.MODIFIER_LETTER
+ || us.generalCategory == UnicodeSpec.OTHER_LETTER
+ || us.generalCategory == UnicodeSpec.LETTER_NUMBER) {
+ resultA |= valueJavaUnicodeStart;
+ }
+ else if (us.generalCategory == UnicodeSpec.COMBINING_SPACING_MARK
+ || us.generalCategory == UnicodeSpec.NON_SPACING_MARK
+ || us.generalCategory == UnicodeSpec.DECIMAL_DIGIT_NUMBER) {
+ resultA |= valueJavaUnicodePart;
+ }
+ else if (us.generalCategory == UnicodeSpec.CONNECTOR_PUNCTUATION) {
+ resultA |= valueJavaStartUnicodePart;
+ }
+ else if (us.generalCategory == UnicodeSpec.CURRENCY_SYMBOL) {
+ resultA |= valueJavaOnlyStart;
+ }
+ else if (((c >= 0x0000) && (c <= 0x0008))
+ || ((c >= 0x000E) && (c <= 0x001B))
+ || ((c >= 0x007F) && (c <= 0x009F))
+ || us.generalCategory == UnicodeSpec.FORMAT) {
+ resultA |= valueIgnorable;
+ }
+ else if (us.generalCategory == UnicodeSpec.SPACE_SEPARATOR
+ || us.generalCategory == UnicodeSpec.LINE_SEPARATOR
+ || us.generalCategory == UnicodeSpec.PARAGRAPH_SEPARATOR) {
+ if (!isInvalidJavaWhiteSpace(c)) resultA |= valueJavaWhitespace;
+ }
+ else if (((c >= 0x0009) && (c <= 0x000D))
+ || ((c >= 0x001C) && (c <= 0x001F))) {
+ resultA |= valueJavaWhitespace;
+ }
+
+ // record bidi category
+ if (!nobidi) {
+ int tmpBidi =
+ (us.bidiCategory > UnicodeSpec.DIRECTIONALITY_OTHER_NEUTRALS ||
+ us.bidiCategory == -1) ? maskBidi : (us.bidiCategory << shiftBidi);
+ resultA |= tmpBidi;
+ }
+
+ // record mirrored property
+ if (!nomirror) {
+ resultA |= us.mirrored ? maskMirrored : 0;
+ }
+
+ if (identifiers) {
+ long replacement = 0;
+ if ((resultA & maskIdentifierInfo) >= lowJavaStart) {
+ replacement |= bitJavaStart;
+ }
+ if ( ((resultA & nonzeroJavaPart) != 0)
+ && ((resultA & maskIdentifierInfo) != valueIgnorable)) {
+ replacement |= bitJavaPart;
+ }
+ resultA = replacement;
+ }
+ return resultA;
+ }
+
+ static void addExProp(long[] map, PropList propList, String prop, long mask) {
+ List<Integer> cps = propList.codepoints(prop);
+ if (cps != null) {
+ for (Integer cp : cps) {
+ if (cp < map.length)
+ map[cp] |= mask;
+ }
+ }
+ }
+
+ /**
+ * This is the heart of the table compression strategy. The inputs are a map
+ * and a number of bits (size). The map is simply an array of long integer values;
+ * the number of bits indicates how index values for that map are to be split.
+ * The length of the given map must be a multiple of (1 << size). The result is
+ * a new map z and a compressed table t such that for every valid index value k
+ * for the original map, t[(z[k>>size]<<size)|(k & ((1<<size)-1))] == map[k].
+ *
+ * In other words, the index k can be split into two parts, namely the "size"
+ * low-order bits and all the remaining high-order bits; the high-order bits are then
+ * remapped by map z to produce an index into table t. In effect, the data of the
+ * original map m is broken up into blocks of size (1<<size); the compression relies
+ * on the expectation that many of these blocks will be identical and therefore need
+ * be represented only once in the compressed table t.
+ *
+ * This method is intended to be used iteratively. The first map to be handed
+ * to it is the one constructed by method buildMap. After that, the first of the
+ * two arrays returned by this method is fed back into it for further compression.
+ * At the end of the iteration, one has a starter map and a sequence of tables.
+ *
+ * The algorithm used to implement this computation is straightforward and not
+ * especially clever. It uses brute-force linear search (the loop labeled MIDDLE)
+ * to locate identical blocks, so overall the time complexity of the algorithm
+ * is quadratic in the length of the input map. Fortunately, speed is not crucial
+ * to this application.
+ *
+ * @param map a map to be compressed
+ * @param size the number of index bits to be split off by the compression
+ * @return an array of length 2 containing two arrays; the first is a new map
+ * and the second is a compressed data table
+ *
+ * @see GenerateCharacter#buildMap
+ */
+
+ static long[][] buildTable(long[] map, int size) {
+ int n = map.length;
+ if (((n >> size) << size) != n) {
+ FAIL("Length " + n + " is not a multiple of " + (1 << size));
+ }
+ int m = 1 << size;
+ // We know the final length of the new map up front.
+ long[] newmap = new long[n >> size];
+ // The buffer is used temporarily to hold data for the compressed table
+ // because we don't know its final length yet.
+ long[] buffer = new long[n];
+ int ptr = 0;
+OUTER: for (int i = 0; i < n; i += m) {
+ // For every block of size m in the original map...
+ MIDDLE: for (int j = 0; j < ptr; j += m) {
+ // Find out whether there is already a block just like it in the buffer.
+ for (int k = 0; k < m; k++) {
+ if (buffer[j+k] != map[i+k])
+ continue MIDDLE;
+ }
+ // There is a block just like it at position j, so just
+ // put its index into the new map (thereby sharing it).
+ newmap[i >> size] = (j >> size);
+ continue OUTER;
+ } // end MIDDLE
+ // There is no block just like it already, so add it to
+ // the buffer and put its index into the new map.
+ for (int k = 0; k < m; k++) {
+ buffer[ptr+k] = map[i+k];
+ }
+ newmap[i >> size] = (ptr >> size);
+ ptr += m;
+ } // end OUTER
+ // Now we know how long the compressed table should be,
+ // so create a new array and copy data from the temporary buffer.
+ long[] newdata = new long[ptr];
+ for (int j = 0; j < ptr; j++) {
+ newdata[j] = buffer[j];
+ }
+ // Return the new map and the new data table.
+ long[][] result = { newmap, newdata };
+ return result;
+ }
+
+ /**
+ * Once the compressed tables have been computed, this method reads in a
+ * template file for the source code to be generated and writes out the final
+ * source code by acting as a sort of specialized macro processor.
+ *
+ * The first output line is a comment saying that the file was automatically
+ * generated; it includes a timestamp. All other output is generated by
+ * reading a line from the template file, performing macro replacements,
+ * and then writing the resulting line or lines of code to the output file.
+ *
+ * This method handles the I/O, the timestamp comment, and the locating of
+ * macro calls within each input line. The method replaceCommand is called
+ * to generate replacement text for each macro call.
+ *
+ * Macro calls to be replaced are indicated in the template file by
+ * occurrences of the commandMarker "$$". The rest of the call may consist
+ * of Java letters (including the underscore "_") and also of balanced
+ * parentheses.
+ *
+ * @param theTemplateFileName
+ * the file name for the template input file
+ * @param theOutputFileName
+ * the file name for the source code output file
+ *
+ * @see GenerateCharacter#replaceCommand
+ */
+
+ static void generateCharacterClass(String theTemplateFileName,
+ String theOutputFileName)
+ throws FileNotFoundException, IOException {
+ BufferedReader in = new BufferedReader(new FileReader(theTemplateFileName));
+ PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(theOutputFileName)));
+ out.println(commentStart +
+ " This file was generated AUTOMATICALLY from a template file " +
+ new java.util.Date() + commentEnd);
+ int marklen = commandMarker.length();
+ LOOP: while(true) {
+ try {
+ String line = in.readLine();
+ if (line == null) break LOOP;
+ int pos = 0;
+ int depth = 0;
+ while ((pos = line.indexOf(commandMarker, pos)) >= 0) {
+ int newpos = pos + marklen;
+ char ch = 'x';
+ SCAN: while (newpos < line.length() &&
+ (Character.isJavaIdentifierStart(ch = line.charAt(newpos))
+ || ch == '(' || (ch == ')' && depth > 0))) {
+ ++newpos;
+ if (ch == '(') {
+ ++depth;
+ }
+ else if (ch == ')') {
+ --depth;
+ if (depth == 0)
+ break SCAN;
+ }
+ }
+ String replacement = replaceCommand(line.substring(pos + marklen, newpos));
+ line = line.substring(0, pos) + replacement + line.substring(newpos);
+ pos += replacement.length();
+ }
+ out.println(line);
+ }
+ catch (IOException e) {
+ break LOOP;
+ }
+ }
+ in.close();
+ out.close();
+ }
+
+ /**
+ * The replaceCommand method takes a command (a macro call without the
+ * leading marker "$$") and computes replacement text for it.
+ *
+ * Most of the commands are simply names of integer constants that are defined
+ * in the source code of this GenerateCharacter class. The replacement text is
+ * simply the value of the constant as an appropriately formatted integer literal.
+ *
+ * Two cases are more complicated, however. The command "Tables" causes the
+ * final map and compressed tables to be emitted, with elaborate comments
+ * describing their contents. (This is actually handled by method genTables.)
+ * The command "Lookup(xxx)", where "xxx" is the name of a variable, generates
+ * an expression that will return the character property data for the character
+ * whose code is the value of the variable "xxx". (this is handled by method
+ * "genAccess".)
+ *
+ * @param x a command from the template file to be replaced
+ * @return the replacement text, as a String
+ *
+ * @see GenerateCharacter#genTables
+ * @see GenerateCharacter#genAccess
+ * @see GenerateCharacter#generateCharacterClass
+ */
+
+ static String replaceCommand(String x) {
+ if (x.equals("Tables")) return genTables();
+ if (x.equals("Initializers")) return genInitializers();
+ if (x.length() >= 9 && x.substring(0, 7).equals("Lookup(") &&
+ x.substring(x.length()-1).equals(")") )
+ return genAccess("A", x.substring(7, x.length()-1), (identifiers ? 2 : 32));
+ if (x.length() >= 11 && x.substring(0, 9).equals("LookupEx(") &&
+ x.substring(x.length()-1).equals(")") )
+ return genAccess("B", x.substring(9, x.length()-1), 16);
+ if (x.equals("shiftType")) return Long.toString(shiftType);
+ if (x.equals("shiftIdentifierInfo")) return Long.toString(shiftIdentifierInfo);
+ if (x.equals("maskIdentifierInfo")) return "0x" + hex8(maskIdentifierInfo);
+ if (x.equals("maskUnicodePart")) return "0x" + hex8(maskUnicodePart);
+ if (x.equals("shiftCaseOffset")) return Long.toString(shiftCaseOffset);
+ if (x.equals("shiftCaseInfo")) return Long.toString(shiftCaseInfo);
+ if (x.equals("shiftCaseOffsetSign")) return Long.toString(shiftCaseOffsetSign);
+ if (x.equals("maskCase")) return "0x" + hex8(maskCase);
+ if (x.equals("maskCaseOffset")) return "0x" + hex8(maskCaseOffset);
+ if (x.equals("maskLowerCase")) return "0x" + hex8(maskLowerCase);
+ if (x.equals("maskUpperCase")) return "0x" + hex8(maskUpperCase);
+ if (x.equals("maskTitleCase")) return "0x" + hex8(maskTitleCase);
+ if (x.equals("maskOtherLowercase")) return "0x" + hex4(maskOtherLowercase >> 32);
+ if (x.equals("maskOtherUppercase")) return "0x" + hex4(maskOtherUppercase >> 32);
+ if (x.equals("maskOtherAlphabetic")) return "0x" + hex4(maskOtherAlphabetic >> 32);
+ if (x.equals("maskIdeographic")) return "0x" + hex4(maskIdeographic >> 32);
+ if (x.equals("valueIgnorable")) return "0x" + hex8(valueIgnorable);
+ if (x.equals("valueJavaUnicodeStart")) return "0x" + hex8(valueJavaUnicodeStart);
+ if (x.equals("valueJavaOnlyStart")) return "0x" + hex8(valueJavaOnlyStart);
+ if (x.equals("valueJavaUnicodePart")) return "0x" + hex8(valueJavaUnicodePart);
+ if (x.equals("valueJavaOnlyPart")) return "0x" + hex8(valueJavaOnlyPart);
+ if (x.equals("valueJavaWhitespace")) return "0x" + hex8(valueJavaWhitespace);
+ if (x.equals("lowJavaStart")) return "0x" + hex8(lowJavaStart);
+ if (x.equals("nonzeroJavaPart")) return "0x" + hex8(nonzeroJavaPart);
+ if (x.equals("bitJavaStart")) return "0x" + hex8(bitJavaStart);
+ if (x.equals("bitJavaPart")) return Long.toString(bitJavaPart);
+ if (x.equals("valueUnicodeStart")) return "0x" + hex8(valueUnicodeStart);
+ if (x.equals("maskIsJavaIdentifierStart")) return "0x" + hex(maskIsJavaIdentifierStart);
+ if (x.equals("maskIsJavaIdentifierPart")) return "0x" + hex(maskIsJavaIdentifierPart);
+ if (x.equals("shiftDigitOffset")) return Long.toString(shiftDigitOffset);
+ if (x.equals("maskDigitOffset")) return "0x" + hex(maskDigitOffset);
+ if (x.equals("maskDigit")) return "0x" + hex(maskDigit);
+ if (x.equals("shiftNumericType")) return Long.toString(shiftNumericType);
+ if (x.equals("maskNumericType")) return "0x" + hex(maskNumericType);
+ if (x.equals("valueNotNumeric")) return "0x" + hex8(valueNotNumeric);
+ if (x.equals("valueDigit")) return "0x" + hex8(valueDigit);
+ if (x.equals("valueStrangeNumeric")) return "0x" + hex8(valueStrangeNumeric);
+ if (x.equals("valueJavaSupradecimal")) return "0x" + hex8(valueJavaSupradecimal);
+ if (x.equals("valueDigit")) return "0x" + hex8(valueDigit);
+ if (x.equals("valueStrangeNumeric")) return "0x" + hex8(valueStrangeNumeric);
+ if (x.equals("maskType")) return "0x" + hex(maskType);
+ if (x.equals("shiftBidi")) return Long.toString(shiftBidi);
+ if (x.equals("maskBidi")) return "0x" + hex(maskBidi);
+ if (x.equals("maskMirrored")) return "0x" + hex8(maskMirrored);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.UNASSIGNED][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.UNASSIGNED);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.UPPERCASE_LETTER][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.UPPERCASE_LETTER);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.LOWERCASE_LETTER][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.LOWERCASE_LETTER);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.TITLECASE_LETTER][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.TITLECASE_LETTER);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.MODIFIER_LETTER][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.MODIFIER_LETTER);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.OTHER_LETTER][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.OTHER_LETTER);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.NON_SPACING_MARK][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.NON_SPACING_MARK);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.ENCLOSING_MARK][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.ENCLOSING_MARK);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.COMBINING_SPACING_MARK][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.COMBINING_SPACING_MARK);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.DECIMAL_DIGIT_NUMBER][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DECIMAL_DIGIT_NUMBER);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.OTHER_NUMBER][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.OTHER_NUMBER);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.SPACE_SEPARATOR][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.SPACE_SEPARATOR);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.LINE_SEPARATOR][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.LINE_SEPARATOR);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.PARAGRAPH_SEPARATOR][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.PARAGRAPH_SEPARATOR);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.CONTROL][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.CONTROL);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.FORMAT][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.FORMAT);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.PRIVATE_USE][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.PRIVATE_USE);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.SURROGATE][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.SURROGATE);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.DASH_PUNCTUATION][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DASH_PUNCTUATION);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.START_PUNCTUATION][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.START_PUNCTUATION);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.END_PUNCTUATION][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.END_PUNCTUATION);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.INITIAL_QUOTE_PUNCTUATION][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.INITIAL_QUOTE_PUNCTUATION);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.FINAL_QUOTE_PUNCTUATION][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.FINAL_QUOTE_PUNCTUATION);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.CONNECTOR_PUNCTUATION][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.CONNECTOR_PUNCTUATION);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.OTHER_PUNCTUATION][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.OTHER_PUNCTUATION);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.LETTER_NUMBER][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.LETTER_NUMBER);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.MATH_SYMBOL][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.MATH_SYMBOL);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.CURRENCY_SYMBOL][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.CURRENCY_SYMBOL);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.MODIFIER_SYMBOL][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.MODIFIER_SYMBOL);
+ if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.OTHER_SYMBOL][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.OTHER_SYMBOL);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT_EMBEDDING][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT_EMBEDDING);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT_OVERRIDE][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT_OVERRIDE);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_EMBEDDING][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_EMBEDDING);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_OVERRIDE][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_OVERRIDE);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_POP_DIRECTIONAL_FORMAT][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_POP_DIRECTIONAL_FORMAT);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER_TERMINATOR][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER_TERMINATOR);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_ARABIC_NUMBER][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_ARABIC_NUMBER);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_COMMON_NUMBER_SEPARATOR][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_COMMON_NUMBER_SEPARATOR);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_NONSPACING_MARK][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_NONSPACING_MARK);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_BOUNDARY_NEUTRAL][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_BOUNDARY_NEUTRAL);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_PARAGRAPH_SEPARATOR][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_PARAGRAPH_SEPARATOR);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_SEGMENT_SEPARATOR][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_SEGMENT_SEPARATOR);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_WHITESPACE][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_WHITESPACE);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_OTHER_NEUTRALS][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_OTHER_NEUTRALS);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT_ISOLATE][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT_ISOLATE);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_ISOLATE][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_ISOLATE);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_FIRST_STRONG_ISOLATE][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_FIRST_STRONG_ISOLATE);
+ if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_POP_DIRECTIONAL_ISOLATE][UnicodeSpec.LONG]))
+ return Integer.toString(UnicodeSpec.DIRECTIONALITY_POP_DIRECTIONAL_ISOLATE);
+ FAIL("Unknown text substitution marker " + commandMarker + x);
+ return commandMarker + x;
+ }
+
+ /**
+ * The genTables method generates source code for all the lookup tables
+ * needed to represent the various Unicode character properties.
+ * It simply calls the method genTable once for each table to be generated
+ * and then generates a summary comment.
+ *
+ * @return the replacement text for the "Tables" command, as a String
+ *
+ * @see GenerateCharacter#genTable
+ * @see GenerateCharacter#replaceCommand
+ */
+ static String genTables() {
+ int n = sizes.length;
+ StringBuffer result = new StringBuffer();
+ // liu : Add a comment showing the source of this table
+ result.append(commentStart + " The following tables and code generated using:" +
+ commentEnd + "\n ");
+ result.append(commentStart + ' ' + commandLineDescription + commentEnd + "\n ");
+
+ if (plane == 0 && bLatin1 == false) {
+ genCaseMapTableDeclaration(result);
+ genCaseMapTable(initializers, specialCaseMaps);
+ }
+ int totalBytes = 0;
+ for (int k = 0; k < n - 1; k++) {
+ genTable(result, tableNames[k], tables[k], 0, bytes[k]<<3, sizes[k], preshifted[k],
+ sizes[k+1], false, false, k==0);
+ int s = bytes[k];
+ if (s == 1 && useCharForByte) {
+ s = 2;
+ }
+ totalBytes += tables[k].length * s;
+ }
+ genTable(result, "A", tables[n - 1], 0, (identifiers ? 2 : 32),
+ sizes[n - 1], false, 0, true, !(identifiers), false);
+
+ // If we ever need more than 32 bits to represent the character properties,
+ // then a table "B" may be needed as well.
+ genTable(result, "B", tables[n - 1], 32, 16, sizes[n - 1], false, 0, true, true, false);
+
+ totalBytes += ((((tables[n - 1].length * (identifiers ? 2 : 32)) + 31) >> 5) << 2);
+ result.append(commentStart);
+ result.append(" In all, the character property tables require ");
+ result.append(totalBytes).append(" bytes.").append(commentEnd);
+ if (verbose) {
+ System.out.println("The character property tables require "
+ + totalBytes + " bytes.");
+ }
+ return result.toString();
+ }
+
+ /**
+ * The genInitializers method generates the body of the
+ * ensureInitted() method, which enables lazy initialization of
+ * the case map table and other tables.
+ */
+ static String genInitializers() {
+ return initializers.toString();
+ }
+
+ /**
+ * Return the total number of bytes needed by all tables. This is a stripped-
+ * down copy of genTables().
+ */
+ static int getTotalBytes() {
+ int n = sizes.length;
+ int totalBytes = 0;
+ for (int k = 0; k < n - 1; k++) {
+ totalBytes += tables[k].length * bytes[k];
+ }
+ totalBytes += ((((tables[n - 1].length * (identifiers ? 2 : 32))
+ + 31) >> 5) << 2);
+ return totalBytes;
+ }
+
+ static void appendEscapedStringFragment(StringBuffer result,
+ char[] line,
+ int length,
+ boolean lastFragment) {
+ result.append(" \"");
+ for (int k=0; k<length; ++k) {
+ result.append("\\u");
+ result.append(hex4(line[k]));
+ }
+ result.append("\"");
+ result.append(lastFragment ? ";" : "+");
+ result.append("\n");
+ }
+
+ static String SMALL_INITIALIZER =
+ " { // THIS CODE WAS AUTOMATICALLY CREATED BY GenerateCharacter:\n"+
+ // " $$name = new $$type[$$size];\n"+
+ " int len = $$name_DATA.length();\n"+
+ " int j=0;\n"+
+ " for (int i=0; i<len; ++i) {\n"+
+ " int c = $$name_DATA.charAt(i);\n"+
+ " for (int k=0; k<$$entriesPerChar; ++k) {\n"+
+ " $$name[j++] = ($$type)c;\n"+
+ " c >>= $$bits;\n"+
+ " }\n"+
+ " }\n"+
+ " assert (j == $$size);\n"+
+ " }\n";
+
+ static String SAME_SIZE_INITIALIZER =
+ " { // THIS CODE WAS AUTOMATICALLY CREATED BY GenerateCharacter:\n"+
+ " assert ($$name_DATA.length() == $$size);\n"+
+ // " $$name = new $$type[$$size];\n"+
+ " for (int i=0; i<$$size; ++i)\n"+
+ " $$name[i] = ($$type)$$name_DATA.charAt(i);\n"+
+ " }\n";
+
+ static String BIG_INITIALIZER =
+ " { // THIS CODE WAS AUTOMATICALLY CREATED BY GenerateCharacter:\n"+
+ // " $$name = new $$type[$$size];\n"+
+ " int len = $$name_DATA.length();\n"+
+ " int j=0;\n"+
+ " int charsInEntry=0;\n"+
+ " $$type entry=0;\n"+
+ " for (int i=0; i<len; ++i) {\n"+
+ " entry |= $$name_DATA.charAt(i);\n"+
+ " if (++charsInEntry == $$charsPerEntry) {\n"+
+ " $$name[j++] = entry;\n"+
+ " entry = 0;\n"+
+ " charsInEntry = 0;\n"+
+ " }\n"+
+ " else {\n"+
+ " entry <<= 16;\n"+
+ " }\n"+
+ " }\n"+
+ " assert (j == $$size);\n"+
+ " }\n";
+
+ static String INT32_INITIALIZER =
+ " { // THIS CODE WAS AUTOMATICALLY CREATED BY GenerateCharacter:\n"+
+ " char[] data = $$name_DATA.toCharArray();\n"+
+ " assert (data.length == ($$size * 2));\n"+
+ " int i = 0, j = 0;\n"+
+ " while (i < ($$size * 2)) {\n"+
+ " int entry = data[i++] << 16;\n"+
+ " $$name[j++] = entry | data[i++];\n"+
+ " }\n"+
+ " }\n";
+
+ static void addInitializer(String name, String type, int entriesPerChar,
+ int bits, int size) {
+
+ String template = (entriesPerChar == 1) ? SAME_SIZE_INITIALIZER :
+ ((entriesPerChar > 0) ? SMALL_INITIALIZER : BIG_INITIALIZER);
+ if (entriesPerChar == -2) {
+ template = INT32_INITIALIZER;
+ }
+ int marklen = commandMarker.length();
+ int pos = 0;
+ while ((pos = template.indexOf(commandMarker, pos)) >= 0) {
+ int newpos = pos + marklen;
+ char ch = 'x';
+ while (newpos < template.length() &&
+ Character.isJavaIdentifierStart(ch = template.charAt(newpos)) &&
+ ch != '_') // Don't allow this in token names
+ ++newpos;
+ String token = template.substring(pos+marklen, newpos);
+ String replacement = "ERROR";
+
+ if (token.equals("name")) replacement = name;
+ else if (token.equals("type")) replacement = type;
+ else if (token.equals("bits")) replacement = ""+bits;
+ else if (token.equals("size")) replacement = ""+size;
+ else if (token.equals("entriesPerChar")) replacement = ""+entriesPerChar;
+ else if (token.equals("charsPerEntry")) replacement = ""+(-entriesPerChar);
+ else FAIL("Unrecognized token: " + token);
+
+ template = template.substring(0, pos) + replacement + template.substring(newpos);
+ pos += replacement.length();
+ }
+ initializers.append(template);
+ }
+
+ /**
+ * The genTable method generates source code for one lookup table.
+ * Most of the complexity stems from handling various options as to
+ * the type of the array components, the precise representation of the
+ * values, the format in which to render each value, the number of values
+ * to emit on each line of source code, and the kinds of useful comments
+ * to be generated.
+ *
+ * @param result a StringBuffer, to which the generated source code
+ * text is to be appended
+ * @param name the name of the table
+ * @param table the table data (an array of long values)
+ * @param extract a distance, in bits, by which each entry of the table
+ * is to be right-shifted before it is processed
+ * @param bits the number of bits (not bytes) to be used to represent
+ * each table entry
+ * @param size the table data is divided up into blocks of size (1<<size);
+ * in this method, this information is used only to affect
+ * how many table values are to be generated per line
+ * @param preshifted if this flag is true, then the table entries are to be
+ * emitted in a preshifted form; that is, each value should
+ * be left-shifted by the amount "shift", so that this work
+ * is built into the table and need not be performed by an
+ * explicit shift operator at run time
+ * @param shift this is the shift amount for preshifting of table entries
+ * @param hexFormat if this flag is true, table entries should be emitted as
+ * hexadecimal literals; otherwise decimal literals are used
+ * @param properties if this flag is true, the table entries are encoded
+ * character properties rather than indexes into yet other tables;
+ * therefore comments describing the encoded properties should
+ * be generated
+ * @param hexComment if this flag is true, each line of output is labelled with
+ * a hexadecimal comment indicating the character values to
+ * which that line applies; otherwise, decimal values indicating
+ * table indices are generated
+ *
+ * @see GenerateCharacter#genTables
+ * @see GenerateCharacter#replaceCommand
+ */
+
+ static void genTable(StringBuffer result, String name,
+ long[] table, int extract, int bits, int size,
+ boolean preshifted, int shift, boolean hexFormat,
+ boolean properties, boolean hexComment) {
+
+ String atype = bits == 1 ? (Csyntax ? "unsigned long" : "int") :
+ bits == 2 ? (Csyntax ? "unsigned long" : "int") :
+ bits == 4 ? (Csyntax ? "unsigned long" : "int") :
+ bits == 8 ? (Csyntax ? "unsigned char" : "byte") :
+ bits == 16 ? (Csyntax ? "unsigned short" : "char") :
+ bits == 32 ? (Csyntax ? "unsigned long" : "int") :
+ (Csyntax ? "int64" : "long");
+ long maxPosEntry = bits == 1 ? Integer.MAX_VALUE : // liu
+ bits == 2 ? Integer.MAX_VALUE :
+ bits == 4 ? Integer.MAX_VALUE :
+ bits == 8 ? Byte.MAX_VALUE :
+ bits == 16 ? Short.MAX_VALUE :
+ bits == 32 ? Integer.MAX_VALUE :
+ Long.MAX_VALUE;
+ int entriesPerChar = bits <= 16 ? (16 / bits) : -(bits / 16);
+ boolean shiftEntries = preshifted && shift != 0;
+ if (bits == 8 && tableAsString && useCharForByte) {
+ atype = "char";
+ maxPosEntry = Character.MAX_VALUE;
+ entriesPerChar = 1;
+ }
+ boolean noConversion = atype.equals("char");
+
+ result.append(commentStart);
+ result.append(" The ").append(name).append(" table has ").append(table.length);
+ result.append(" entries for a total of ");
+ int sizeOfTable = ((table.length * bits + 31) >> 5) << 2;
+ if (bits == 8 && useCharForByte) {
+ sizeOfTable *= 2;
+ }
+ result.append(sizeOfTable);
+ result.append(" bytes.").append(commentEnd).append("\n\n");
+ if (Csyntax)
+ result.append(" static ");
+ else
+ result.append(" static final ");
+ result.append(atype);
+ result.append(" ").append(name).append("[");
+ if (Csyntax)
+ result.append(table.length >> (bits == 1 ? 5 : bits == 2 ? 4 : bits == 4 ? 3 : 0));
+ if (tableAsString) {
+ if (noConversion) {
+ result.append("] = (\n");
+ } else {
+ result.append("] = new ").append(atype).append("["+table.length+"];\n ");
+ result.append("static final String ").append(name).append("_DATA =\n");
+ }
+ int CHARS_PER_LINE = 8;
+ StringBuffer theString = new StringBuffer();
+ int entriesInCharSoFar = 0;
+ char ch = '\u0000';
+ int charsPerEntry = -entriesPerChar;
+ for (int j=0; j<table.length; ++j) {
+ //long entry = table[j] >> extract;
+ long entry;
+ if ("A".equals(name))
+ entry = (table[j] & 0xffffffffL) >> extract;
+ else
+ entry = (table[j] >> extract);
+ if (shiftEntries) entry <<= shift;
+ if (entry >= (1L << bits)) {
+ FAIL("Entry too big");
+ }
+ if (entriesPerChar > 0) {
+ // Pack multiple entries into a character
+ ch = (char)(((int)ch >> bits) | (entry << (entriesPerChar-1)*bits));
+ ++entriesInCharSoFar;
+ if (entriesInCharSoFar == entriesPerChar) {
+ // Character is full
+ theString.append(ch);
+ entriesInCharSoFar = 0;
+ ch = '\u0000';
+ }
+ }
+ else {
+ // Use multiple characters per entry
+ for (int k=0; k<charsPerEntry; ++k) {
+ ch = (char)(entry >> ((charsPerEntry-1)*16));
+ entry <<= 16;
+ theString.append(ch);
+ }
+ }
+ }
+ if (entriesInCharSoFar > 0) {
+ while (entriesInCharSoFar < entriesPerChar) {
+ ch = (char)((int)ch >> bits);
+ ++entriesInCharSoFar;
+ }
+ theString.append(ch);
+ entriesInCharSoFar = 0;
+ }
+ result.append(Utility.formatForSource(theString.toString(), " "));
+ if (noConversion) {
+ result.append(").toCharArray()");
+ }
+ result.append(";\n\n ");
+
+ if (!noConversion) {
+ addInitializer(name, atype, entriesPerChar, bits, table.length);
+ }
+ }
+ else {
+ result.append("] = {");
+ boolean castEntries = shiftEntries && (bits < 32);
+ int printPerLine = hexFormat ? (bits == 1 ? 32*4 :
+ bits == 2 ? 16*4 :
+ bits == 4 ? 8*4 :
+ bits == 8 ? 8 :
+ bits == 16 ? 8 :
+ bits == 32 ? 4 : 2) :
+ (bits == 8 ? 8 :
+ bits == 16 ? 8 : 4);
+ int printMask = properties ? 0 :
+ Math.min(1 << size,
+ printPerLine >> (castEntries ? (Csyntax ? 2 : 1) : 0)) - 1;
+ int commentShift = ((1 << size) == table.length) ? 0 : size;
+ int commentMask = ((1 << size) == table.length) ? printMask : (1 << size) - 1;
+ long val = 0;
+ for (int j = 0; j < table.length; j++) {
+ if ((j & printMask) == 0) {
+ while (result.charAt(result.length() - 1) == ' ')
+ result.setLength(result.length() - 1);
+ result.append("\n ");
+ }
+ PRINT: {
+ if (castEntries)
+ result.append("(").append(atype).append(")(");
+ long entry = table[j] >> extract;
+ int packMask = ((1 << (bits == 1 ? 5 : bits == 2 ? 4 : bits == 4 ? 3 : 2)) - 1);
+ int k = j & packMask;
+ if (bits >= 8)
+ val = entry;
+ else if (k == 0) {
+ val = entry;
+ break PRINT;
+ }
+ else {
+ val |= (entry << (k*bits));
+ if (k != packMask)
+ break PRINT;
+ }
+ if (val > maxPosEntry && !Csyntax) { // liu
+ // For values that are out of range, convert them to in-range negative values.
+ // Actually, output the '-' and convert them to the negative of the corresponding
+ // in-range negative values. E.g., convert 130 == -126 (in 8 bits) -> 126.
+ result.append('-');
+ val = maxPosEntry + maxPosEntry + 2 - val;
+ }
+ if (hexFormat) {
+ result.append("0x");
+ if (bits == 8)
+ result.append(hex2((byte)val));
+ else if (bits == 16)
+ result.append(hex4((short)val));
+ else if (bits == 32 || bits < 8)
+ result.append(hex8((int)val));
+ else {
+ result.append(hex16(val));
+ if (!Csyntax)
+ result.append("L");
+ }
+ }
+ else {
+ if (bits == 8)
+ result.append(dec3(val));
+ else if (bits == 64) {
+ result.append(dec5(val));
+ if (!Csyntax)
+ result.append("L");
+ }
+ else
+ result.append(dec5(val));
+ }
+ if (shiftEntries)
+ result.append("<<").append(shift);
+ if (castEntries) result.append(")");
+ if (j < (table.length - 1))
+ result.append(", ");
+ else
+ result.append(" ");
+ if ((j & printMask) == printMask) {
+ result.append(" ").append(commentStart).append(" ");
+ if (hexComment)
+ result.append("0x").append(hex4((j & ~commentMask) << (16 - size)));
+ else
+ result.append(dec3((j & ~commentMask) >> commentShift));
+ if (properties) propertiesComments(result, val);
+ result.append(commentEnd);
+ }
+ } // end PRINT
+ }
+ result.append("\n };\n\n ");
+ }
+ }
+
+ static void genCaseMapTableDeclaration(StringBuffer result) {
+ String myTab = " ";
+ result.append(myTab + "static final char[][][] charMap;\n");
+ }
+
+ static void genCaseMapTable(StringBuffer result, SpecialCaseMap[] specialCaseMaps){
+ String myTab = " ";
+ int ch;
+ char[] map;
+ result.append(myTab + "charMap = new char[][][] {\n");
+ for (int x = 0; x < specialCaseMaps.length; x++) {
+ ch = specialCaseMaps[x].getCharSource();
+ map = specialCaseMaps[x].getUpperCaseMap();
+ result.append(myTab + myTab);
+ result.append("{ ");
+ result.append("{\'\\u"+hex4(ch)+"\'}, {");
+ for (int y = 0; y < map.length; y++) {
+ result.append("\'\\u"+hex4(map[y])+"\', ");
+ }
+ result.append("} },\n");
+ }
+ result.append(myTab + "};\n");
+
+ }
+
+ /**
+ * The propertiesComments method generates comments describing encoded
+ * character properties.
+ *
+ * @param result a StringBuffer, to which the generated source code
+ * text is to be appended
+ * @param val encoded character properties
+ *
+ * @see GenerateCharacter#genTable
+ */
+
+ static void propertiesComments(StringBuffer result, long val) {
+ result.append(" ");
+ switch ((int)(val & maskType)) {
+ case UnicodeSpec.CONTROL:
+ result.append("Cc");
+ break;
+ case UnicodeSpec.FORMAT:
+ result.append("Cf");
+ break;
+ case UnicodeSpec.PRIVATE_USE:
+ result.append("Co");
+ break;
+ case UnicodeSpec.SURROGATE:
+ result.append("Cs");
+ break;
+ case UnicodeSpec.LOWERCASE_LETTER:
+ result.append("Ll");
+ break;
+ case UnicodeSpec.MODIFIER_LETTER:
+ result.append("Lm");
+ break;
+ case UnicodeSpec.OTHER_LETTER:
+ result.append("Lo");
+ break;
+ case UnicodeSpec.TITLECASE_LETTER:
+ result.append("Lt");
+ break;
+ case UnicodeSpec.UPPERCASE_LETTER:
+ result.append("Lu");
+ break;
+ case UnicodeSpec.COMBINING_SPACING_MARK:
+ result.append("Mc");
+ break;
+ case UnicodeSpec.ENCLOSING_MARK:
+ result.append("Me");
+ break;
+ case UnicodeSpec.NON_SPACING_MARK:
+ result.append("Mn");
+ break;
+ case UnicodeSpec.DECIMAL_DIGIT_NUMBER:
+ result.append("Nd");
+ break;
+ case UnicodeSpec.LETTER_NUMBER:
+ result.append("Nl");
+ break;
+ case UnicodeSpec.OTHER_NUMBER:
+ result.append("No");
+ break;
+ case UnicodeSpec.CONNECTOR_PUNCTUATION:
+ result.append("Pc");
+ break;
+ case UnicodeSpec.DASH_PUNCTUATION:
+ result.append("Pd");
+ break;
+ case UnicodeSpec.END_PUNCTUATION:
+ result.append("Pe");
+ break;
+ case UnicodeSpec.OTHER_PUNCTUATION:
+ result.append("Po");
+ break;
+ case UnicodeSpec.START_PUNCTUATION:
+ result.append("Ps");
+ break;
+ case UnicodeSpec.CURRENCY_SYMBOL:
+ result.append("Sc");
+ break;
+ case UnicodeSpec.MODIFIER_SYMBOL:
+ result.append("Sk");
+ break;
+ case UnicodeSpec.MATH_SYMBOL:
+ result.append("Sm");
+ break;
+ case UnicodeSpec.OTHER_SYMBOL:
+ result.append("So");
+ break;
+ case UnicodeSpec.LINE_SEPARATOR:
+ result.append("Zl"); break;
+ case UnicodeSpec.PARAGRAPH_SEPARATOR:
+ result.append("Zp");
+ break;
+ case UnicodeSpec.SPACE_SEPARATOR:
+ result.append("Zs");
+ break;
+ case UnicodeSpec.UNASSIGNED:
+ result.append("unassigned");
+ break;
+ }
+
+ switch ((int)((val & maskBidi) >> shiftBidi)) {
+ case UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT:
+ result.append(", L");
+ break;
+ case UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT:
+ result.append(", R");
+ break;
+ case UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER:
+ result.append(", EN");
+ break;
+ case UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR:
+ result.append(", ES");
+ break;
+ case UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER_TERMINATOR:
+ result.append(", ET");
+ break;
+ case UnicodeSpec.DIRECTIONALITY_ARABIC_NUMBER:
+ result.append(", AN");
+ break;
+ case UnicodeSpec.DIRECTIONALITY_COMMON_NUMBER_SEPARATOR:
+ result.append(", CS");
+ break;
+ case UnicodeSpec.DIRECTIONALITY_PARAGRAPH_SEPARATOR:
+ result.append(", B");
+ break;
+ case UnicodeSpec.DIRECTIONALITY_SEGMENT_SEPARATOR:
+ result.append(", S");
+ break;
+ case UnicodeSpec.DIRECTIONALITY_WHITESPACE:
+ result.append(", WS");
+ break;
+ case UnicodeSpec.DIRECTIONALITY_OTHER_NEUTRALS:
+ result.append(", ON");
+ break;
+ }
+ if ((val & maskUpperCase) != 0) {
+ result.append(", hasUpper (subtract ");
+ result.append((val & maskCaseOffset) >> shiftCaseOffset).append(")");
+ }
+ if ((val & maskLowerCase) != 0) {
+ result.append(", hasLower (add ");
+ result.append((val & maskCaseOffset) >> shiftCaseOffset).append(")");
+ }
+ if ((val & maskTitleCase) != 0) {
+ result.append(", hasTitle");
+ }
+ if ((val & maskIdentifierInfo) == valueIgnorable) {
+ result.append(", ignorable");
+ }
+ if ((val & maskIdentifierInfo) == valueJavaUnicodePart) {
+ result.append(", identifier part");
+ }
+ if ((val & maskIdentifierInfo) == valueJavaStartUnicodePart) {
+ result.append(", underscore");
+ }
+ if ((val & maskIdentifierInfo) == valueJavaWhitespace) {
+ result.append(", whitespace");
+ }
+ if ((val & maskIdentifierInfo) == valueJavaOnlyStart) {
+ result.append(", currency");
+ }
+ if ((val & maskIdentifierInfo) == valueJavaUnicodeStart) {
+ result.append(", identifier start");
+ }
+ if ((val & maskNumericType) == valueDigit) {
+ result.append(", decimal ");
+ result.append((val & maskDigitOffset) >> shiftDigitOffset);
+ }
+ if ((val & maskNumericType) == valueStrangeNumeric) {
+ result.append(", strange");
+ }
+ if ((val & maskNumericType) == valueJavaSupradecimal) {
+ result.append(", supradecimal ");
+ result.append((val & maskDigitOffset) >> shiftDigitOffset);
+ }
+ }
+
+ static String[] tableNames = { "X", "Y", "Z", "P", "Q", "R", "S", "T", "U", "V", "W" };
+
+ static String tableName(int j) { return tableNames[j]; }
+
+ /**
+ * The genAccess method generates source code for one table access expression.
+ *
+ * Most of the complexity stems from handling various options as to
+ * table representation, such as whether it contains values so large that
+ * they are represented as negative values and whether the table values are
+ * preshifted. This method also avoids such "ugly" expressions as shifting
+ * by distance zero, masking when no masking is necessary, and so on.
+ * For clarity, it generates expressions that do not rely on operator
+ * precedence, but otherwise it avoids generating redundant parentheses.
+ *
+ * A generated expression might look like A[Y[(X[ch>>6]<<6)|(ch&0x3F)]]
+ * or A[Z[Y[(X[ch>>7]<<4)|((ch>>3)&0xF)]|(ch&0x7)]], for example.
+ *
+ * @param tbl the name of the final table to be accessed
+ * @param var the variable name that appeared in parentheses in the
+ * "Lookup" command
+ * @param bits the number of bits (not bytes) to be used to represent
+ * the final table entry
+ * @return the replacement text for the "Lookup(xxx)" command, as a String
+ *
+ * @see GenerateCharacter#replaceCommand
+ */
+
+ static String genAccess(String tbl, String var, int bits) {
+ String access = null;
+ int bitoffset = bits == 1 ? 5 : bits == 2 ? 4 : bits == 4 ? 3 : 0;
+ for (int k = 0; k < sizes.length; k++) {
+ int offset = ((k < sizes.length - 1) ? 0 : bitoffset);
+ int shift = shifts[k] + offset;
+ String shifted = (shift == 0) ? var : "(" + var + ">>" + shift + ")";
+ int mask = (1 << (sizes[k] - offset)) - 1;
+ String masked = (k == 0) ? shifted :
+ "(" + shifted + "&0x" + hex(mask) + ")";
+ String index = (k == 0) ? masked :
+ (mask == 0) ? access : "(" + access + "|" + masked + ")";
+ String indexNoParens = (index.charAt(0) != '(') ? index :
+ index.substring(1, index.length() - 1);
+ String tblname = (k == sizes.length - 1) ? tbl : tableName(k);
+ String fetched = tblname + "[" + indexNoParens + "]";
+ String zeroextended = (zeroextend[k] == 0) ? fetched :
+ "(" + fetched + "&0x" + hex(zeroextend[k]) + ")";
+ int adjustment = preshifted[k] ? 0 :
+ sizes[k+1] - ((k == sizes.length - 2) ? bitoffset : 0);
+ String adjusted = (preshifted[k] || adjustment == 0) ? zeroextended :
+ "(" + zeroextended + "<<" + adjustment + ")";
+ String bitshift = (bits == 1) ? "(" + var + "&0x1F)" :
+ (bits == 2) ? "((" + var + "&0xF)<<1)" :
+ (bits == 4) ? "((" + var + "&7)<<2)" : null;
+ String extracted = ((k < sizes.length - 1) || (bits >= 8)) ? adjusted :
+ "((" + adjusted + ">>" + bitshift + ")&" +
+ (bits == 4 ? "0xF" : "" + ((1 << bits) - 1)) + ")";
+ access = extracted;
+ }
+ return access;
+ }
+
+ /* The command line arguments are decoded and used to set the following
+ global variables.
+ */
+
+ static boolean verbose = false;
+ static boolean nobidi = false;
+ static boolean nomirror = false;
+ static boolean identifiers = false;
+ static boolean Csyntax = false;
+ static String TemplateFileName = null;
+ static String OutputFileName = null;
+ static String UnicodeSpecFileName = null; // liu
+ static String SpecialCasingFileName = null;
+ static String PropListFileName = null;
+ static boolean useCharForByte = false;
+ static int[] sizes;
+ static int bins = 0; // liu; if > 0, then perform search
+ static boolean tableAsString = false;
+ static boolean bLatin1 = false;
+
+ static String commandLineDescription;
+
+ /* Other global variables, equal in length to the "sizes" array. */
+
+ static int[] shifts;
+ static int[] zeroextend;
+ static int[] bytes;
+ static boolean[] preshifted;
+ static long[][] tables;
+
+
+ /* Other global variables */
+ static String commentStart;
+ static String commentEnd;
+
+ static StringBuffer initializers = new StringBuffer();
+
+ /* special casing rules for 1:M toUpperCase mappings */
+ static SpecialCaseMap[] specialCaseMaps;
+
+ /**
+ * Process the command line arguments.
+ *
+ * The allowed flags in command line are:
+ * <dl>
+ * <dt> -verbose <dd> Emit comments to standard output describing
+ * what's going on during the processing.
+ * <dt> -nobidi <dd> Do not include bidi categories in the
+ * encoded character properties.
+ * <dt> -nomirror <dd> Do no include mirror property in the encoded
+ * character properties.
+ * <dt> -identifiers <dd> Generate tables for scanning identifiers only.
+ * <dt> -c <dd> Output code in C syntax instead of Java syntax.
+ * <dt> -o filename <dd> Specify output file name.
+ * <dt> -template filename <dd> Specify template input file name.
+ * <dt> -spec filename <dd> Specify Unicode spec file name.
+ * <dt> -specialcasing filename <dd> Specify Unicode special casing file name.
+ * <dt> -search bins <dd> Try different partitions into the specified
+ * number of bins. E.g., for 2 bins, try
+ * 16 0, 15 1,..., 0 16.
+ * <dt> -string <dd> Create table as string. Only valid with Java
+ * syntax.
+ * <dt> -latin1 <dd> Create a latin 1 only property table.
+ * </dl>
+ * In addition, decimal literals may appear as command line arguments;
+ * each one represents the number of bits of the character to be broken
+ * off at each lookup step. If present, they must add up to 16 (the number
+ * of bits in a char value). For smaller tables, the last value should
+ * be 0; values other than the last one may not be zero. If no such
+ * numeric values are provided, default values are used.
+ *
+ * @param args the command line arguments, as an array of String
+ *
+ * @see GenerateCharacter#main
+ */
+
+ static void processArgs(String[] args) {
+ StringBuffer desc = new StringBuffer("java GenerateCharacter");
+ for (int j=0; j<args.length; ++j) {
+ desc.append(" " + args[j]);
+ }
+ for (int j = 0; j < args.length; j++) {
+ if (args[j].equals("-verbose") || args[j].equals("-v"))
+ verbose = true;
+ else if (args[j].equals("-nobidi"))
+ nobidi = true;
+ else if (args[j].equals("-nomirror"))
+ nomirror = true;
+ else if (args[j].equals("-identifiers"))
+ identifiers = true;
+ else if (args[j].equals("-c"))
+ Csyntax = true;
+ else if (args[j].equals("-string"))
+ tableAsString = true;
+ else if (args[j].equals("-o")) {
+ if (j == args.length - 1) {
+ FAIL("File name missing after -o");
+ }
+ else {
+ OutputFileName = args[++j];
+ }
+ }
+ else if (args[j].equals("-search")) {
+ if (j == args.length - 1)
+ FAIL("Bin count missing after -search");
+ else {
+ bins = Integer.parseInt(args[++j]);
+ if (bins < 1 || bins > 10)
+ FAIL("Bin count must be >= 1 and <= 10");
+ }
+ }
+ else if (args[j].equals("-template")) {
+ if (j == args.length - 1)
+ FAIL("File name missing after -template");
+ else
+ TemplateFileName = args[++j];
+ }
+ else if (args[j].equals("-spec")) { // liu
+ if (j == args.length - 1) {
+ FAIL("File name missing after -spec");
+ }
+ else {
+ UnicodeSpecFileName = args[++j];
+ }
+ }
+ else if (args[j].equals("-specialcasing")) {
+ if (j == args.length -1) {
+ FAIL("File name missing after -specialcasing");
+ }
+ else {
+ SpecialCasingFileName = args[++j];
+ }
+ }
+ else if (args[j].equals("-proplist")) {
+ if (j == args.length -1) {
+ FAIL("File name missing after -proplist");
+ }
+ else {
+ PropListFileName = args[++j];
+ }
+ }
+ else if (args[j].equals("-plane")) {
+ if (j == args.length -1) {
+ FAIL("Plane number missing after -plane");
+ }
+ else {
+ plane = Integer.parseInt(args[++j]);
+ }
+ if (plane > 0) {
+ bLatin1 = false;
+ }
+ }
+ else if ("-usecharforbyte".equals(args[j])) {
+ useCharForByte = true;
+ }
+ else if (args[j].equals("-latin1")) {
+ bLatin1 = true;
+ plane = 0;
+ }
+ else {
+ try {
+ int val = Integer.parseInt(args[j]);
+ if (val < 0 || val > 32) FAIL("Incorrect bit field width: " + args[j]);
+ if (sizes == null)
+ sizes = new int[1];
+ else {
+ int[] newsizes = new int[sizes.length + 1];
+ System.arraycopy(sizes, 0, newsizes, 0, sizes.length);
+ sizes = newsizes;
+ }
+ sizes[sizes.length - 1] = val;
+ }
+ catch(NumberFormatException e) {
+ FAIL("Unknown switch: " + args[j]);
+ }
+ }
+ }
+ if (Csyntax && tableAsString) {
+ FAIL("Can't specify table as string with C syntax");
+ }
+ if (sizes == null) {
+ desc.append(" [");
+ if (identifiers) {
+ int[] newsizes = { 8, 4, 4 }; // Good default values
+ desc.append("8 4 4]");
+ sizes = newsizes;
+ }
+ else {
+ int[] newsizes = { 10, 5, 1 }; // Guy's old defaults for 2.0.14: { 9, 4, 3, 0 }
+ desc.append("10 5 1]");
+ sizes = newsizes;
+ }
+ }
+ if (UnicodeSpecFileName == null) { // liu
+ UnicodeSpecFileName = DefaultUnicodeSpecFileName;
+ desc.append(" [-spec " + UnicodeSpecFileName + ']');
+ }
+ if (SpecialCasingFileName == null) {
+ SpecialCasingFileName = DefaultSpecialCasingFileName;
+ desc.append(" [-specialcasing " + SpecialCasingFileName + ']');
+ }
+ if (PropListFileName == null) {
+ PropListFileName = DefaultPropListFileName;
+ desc.append(" [-proplist " + PropListFileName + ']');
+ }
+ if (TemplateFileName == null) {
+ TemplateFileName = (Csyntax ? DefaultCTemplateFileName
+ : DefaultJavaTemplateFileName);
+ desc.append(" [-template " + TemplateFileName + ']');
+ }
+ if (OutputFileName == null) {
+ OutputFileName = (Csyntax ? DefaultCOutputFileName
+ : DefaultJavaOutputFileName);
+ desc.append(" [-o " + OutputFileName + ']');
+ }
+ commentStart = (Csyntax ? "/*" : "//");
+ commentEnd = (Csyntax ? " */" : "");
+ commandLineDescription = desc.toString();
+ }
+
+ private static void searchBins(long[] map, int binsOccupied) throws Exception {
+ int bitsFree = 16;
+ for (int i=0; i<binsOccupied; ++i) bitsFree -= sizes[i];
+ if (binsOccupied == (bins-1)) {
+ sizes[binsOccupied] = bitsFree;
+ generateForSizes(map);
+ }
+ else {
+ for (int i=1; i<bitsFree; ++i) { // Don't allow bins of 0 except for last one
+ sizes[binsOccupied] = i;
+ searchBins(map, binsOccupied+1);
+ }
+ }
+ }
+
+ private static void generateForSizes(long[] map) throws Exception {
+ int sum = 0;
+ shifts = new int[sizes.length];
+ for (int k = sizes.length - 1; k >= 0; k--) {
+ shifts[k] = sum;
+ sum += sizes[k];
+ }
+ if ((1 << sum) < map.length || (1 << (sum - 1)) >= map.length) {
+ FAIL("Bit field widths total to " + sum +
+ ": wrong total for map of size " + map.length);
+ }
+ // need a table for each set of lookup bits in char
+ tables = new long[sizes.length][];
+ // the last table is the map
+ tables[sizes.length - 1] = map;
+ for (int j = sizes.length - 1; j > 0; j--) {
+ if (verbose && bins==0)
+ System.err.println("Building map " + (j+1) + " of bit width " + sizes[j]);
+ long[][] temp = buildTable(tables[j], sizes[j]);
+ tables[j-1] = temp[0];
+ tables[j] = temp[1];
+ }
+ preshifted = new boolean[sizes.length];
+ zeroextend = new int[sizes.length];
+ bytes = new int[sizes.length];
+ for (int j = 0; j < sizes.length - 1; j++) {
+ int len = tables[j+1].length;
+ int size = sizes[j+1];
+ if (len > 0x100 && (len >> size) <= 0x100) {
+ len >>= size;
+ preshifted[j] = false;
+ }
+ else if (len > 0x10000 && (len >> size) <= 0x10000) {
+ len >>= size;
+ preshifted[j] = false;
+ }
+ else preshifted[j] = true;
+ if (Csyntax)
+ zeroextend[j] = 0;
+ else if (len > 0x7F && len <= 0xFF) {
+ if (!useCharForByte) {
+ zeroextend[j] = 0xFF;
+ }
+ } else if (len > 0x7FFF && len <= 0xFFFF)
+ zeroextend[j] = 0xFFFF;
+ else zeroextend[j] = 0;
+ if (len <= 0x100) bytes[j] = 1;
+ else if (len <= 0x10000) bytes[j] = 2;
+ else bytes[j] = 4;
+ }
+ preshifted[sizes.length - 1] = true;
+ zeroextend[sizes.length - 1] = 0;
+ bytes[sizes.length - 1] = 0;
+ if (bins > 0) {
+ int totalBytes = getTotalBytes();
+ String access = genAccess("A", "ch", (identifiers ? 2 : 32));
+ int accessComplexity = 0;
+ for (int j=0; j<access.length(); ++j) {
+ char ch = access.charAt(j);
+ if ("[&|><".indexOf(ch) >= 0) ++accessComplexity;
+ if (ch == '<' || ch == '>') ++j;
+ }
+ System.out.print("(");
+ for (int j=0; j<sizes.length; ++j) System.out.print(" " + sizes[j]);
+ System.out.println(" ) " + totalBytes + " " + accessComplexity + " " + access);
+ return;
+ }
+ if (verbose) {
+ System.out.println(" n\t size\tlength\tshift\tzeroext\tbytes\tpreshifted");
+ for (int j = 0; j < sizes.length; j++) {
+ System.out.println(dec5(j) + "\t" +
+ dec5(sizes[j]) + "\t" +
+ dec5(tables[j].length) + "\t" +
+ dec5(shifts[j]) + "\t" +
+ dec5(zeroextend[j]) + "\t" +
+ dec5(bytes[j]) + "\t " +
+ preshifted[j]);
+ }
+ }
+ if (verbose) {
+ System.out.println("Generating source code for class Character");
+ System.out.println("A table access looks like " +
+ genAccess("A", "ch", (identifiers ? 2 : 32)));
+ }
+ generateCharacterClass(TemplateFileName, OutputFileName);
+ }
+
+ /**
+ * The main program for generating source code for the Character class.
+ * The basic outline of its operation is:
+ * <ol>
+ * <li> Process the command line arguments. One result of this process
+ * is a list of sizes (measured in bits and summing to 16).
+ * <li> Get the Unicode character property data from the specification file.
+ * <li> From that, build a map that has, for each character code, its
+ * relevant properties encoded as a long integer value.
+ * <li> Repeatedly compress the map, producing a compressed table and a
+ * new map. This is done once for each size value in the list.
+ * When this is done, we have a set of tables.
+ * <li> Make some decisions about table representation; record these
+ * decisions in arrays named preshifted, zeroextend, and bytes.
+ * <li> Generate the source code for the class Character by performing
+ * macro processing on a template file.
+ * </ol>
+ *
+ * @param args the command line arguments, as an array of String
+ *
+ * @see GenerateCharacter#processArgs
+ * @see UnicodeSpec@readSpecFile
+ * @see GenerateCharacter#buildMap
+ * @see GenerateCharacter#buildTable
+ * @see GenerateCharacter#generateCharacterClass
+ */
+
+ public static void main(String[] args) {
+ processArgs(args);
+ try {
+
+ UnicodeSpec[] data = UnicodeSpec.readSpecFile(new File(UnicodeSpecFileName), plane);
+ specialCaseMaps = SpecialCaseMap.readSpecFile(new File(SpecialCasingFileName), plane);
+ PropList propList = PropList.readSpecFile(new File(PropListFileName), plane);
+
+ if (verbose) {
+ System.out.println(data.length + " items read from Unicode spec file " + UnicodeSpecFileName); // liu
+ }
+ long[] map = buildMap(data, specialCaseMaps, propList);
+ if (verbose) {
+ System.err.println("Completed building of initial map");
+ }
+
+ if (bins == 0) {
+ generateForSizes(map);
+ }
+ else {
+ while (bins > 0) {
+ sizes = new int[bins];
+ searchBins(map, 0);
+ --bins;
+ }
+ }
+ if (verbose && false) {
+ System.out.println("Offset range seen: -" + hex8(-minOffsetSeen) + "..+" +
+ hex8(maxOffsetSeen));
+ System.out.println(" allowed: -" + hex8(-minOffset) + "..+" +
+ hex8(maxOffset));
+ }
+ }
+ catch (FileNotFoundException e) { FAIL(e.toString()); }
+ catch (IOException e) { FAIL(e.toString()); }
+ catch (Throwable e) {
+ System.out.println("Unexpected exception:");
+ e.printStackTrace();
+ FAIL("Unexpected exception!");
+ }
+ if (verbose) { System.out.println("Done!");}
+ }
+
+} // end class