/*

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.  Oracle designates this

 * particular file as subject to the "Classpath" exception as provided

 * by Oracle in the LICENSE file that accompanied this code.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 */

package build.tools.generatecharacter;

import java.io.IOException;

import java.io.FileNotFoundException;

import java.io.BufferedReader;

import java.io.FileReader;

import java.io.PrintWriter;

import java.io.BufferedWriter;

import java.io.FileWriter;

import java.io.File;

import java.util.List;

import build.tools.generatecharacter.CharacterName;

/**

 * This program generates the source code for the class java.lang.Character.

 * It also generates native C code that can perform the same operations.

 * It requires two external input data files:

 * <ul>

 * <li> Unicode specification file

 * <li> Character class template file

 * </ul>

 * The Unicode specification file is available from the Unicode consortium.

 * It has character specification lines that look like this:

 * <listing>

 * 0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061;

 * </listing>

 * The Character class template file is filled in with additional

 * information to produce the file Character.java, which can then be

 * compiled by a Java compiler.  The template file contains certain

 * markers consisting of an alphabetic name string preceded by "$$".

 * Such markers are replaced with generated program text.  As a special

 * case, the marker "Lookup(xxx)" is recognized, where "xxx" consists of

 * alphabetic characters constituting a variable name.  The character "_"

 * is considered alphabetic for these purposes.

 *

 * @author  Guy Steele

 * @author  Alan Liu

 * @author  John O'Conner

 */

public class GenerateCharacter {

    final static boolean DEBUG = false;

    final static String commandMarker = "$$";

    static String ROOT                        = "";

    static String DefaultUnicodeSpecFileName  = ROOT + "UnicodeData.txt";

    static String DefaultSpecialCasingFileName = ROOT + "SpecialCasing.txt";

    static String DefaultPropListFileName     = ROOT + "PropList.txt";

    static String DefaultJavaTemplateFileName = ROOT + "Character.java.template";

    static String DefaultJavaOutputFileName   = ROOT + "Character.java";

    static String DefaultCTemplateFileName    = ROOT + "Character.c.template";

    static String DefaultCOutputFileName      = ROOT + "Character.c";

    static int plane = 0;

    /* The overall idea is that, in the generated Character class source code,

    most character property data is stored in a special multi-level table whose

    structure is defined by a sequence of nonnegative integers [k1, k2, ..., kn].

    The integers must sum to 16 (the number of bits in a character).

    The first table is indexed by the k1 high-order bits of the character code.

    The result is concatenated to the next k2 bits of the character code to index

    the second table, and so on.  Eventually the kn low-order bits of the character

    code are concatenated and used to index one of two tables A and B; A contains

    32-bit integer entries and B contains 16-bit short entries.  The 48 bits that

    can be thus obtained encode the properties for the character.

    The default specification is [9, 4, 3, 0].  This particular table format was

    designed by conducting an exhaustive search of table formats to minimize the

    space consumed by the tables: the first and third tables need have only byte

    values (the second table must have short values).  Another good choice is

    [10, 6, 0], which produces a larger table but allows particularly fast table

    lookup code.

    In each case, where the word "concatenated" is used, this may imply

    first a << and then a | operation, or perhaps just a | operation if

    the values in the table can be preshifted (generally possible if the table

    entries are short rather than byte).

    */

    /* The character properties are currently encoded into A (32 bits)and B (16 bits)

       two parts.

    A: the low 32 bits are defined  in the following manner:

    1 bit Mirrored property.

    4 bits      Bidirectional category (see below) (unused if -nobidi switch specified)

    9 bits      A signed offset used for converting case .

    1 bit       If 1, adding the signed offset converts the character to lowercase.

    1 bit       If 1, subtracting the signed offset converts the character to uppercase.

        Note: for a titlecase character, both of the preceding bits will be 1

        and the signed offset will be 1.

    1 bit   If 1, this character has a titlecase equivalent (possibly itself);

        in this case, the two bits before this bit can be used to decide

        whether this character is in fact uppercase, lowercase, or titlecase.

    3 bits      This field provides a quick way to lex identifiers.

        The eight possible values for this field are as follows:

        0  May not be part of an identifier

        1  Ignorable control; may continue a Unicode identifier or Java identifier

        2  May continue a Java identifier but not a Unicode identifier (unused)

        3  May continue a Unicode identifier or Java identifier

        4  Is a Java whitespace character

        5  May start or continue a Java identifier;

           may continue but not start a Unicode identifier

           (this value is used for connector punctuation such as _)

        6  May start or continue a Java identifier;

           may not occur in a Unicode identifier

           (this value is used for currency symbols such as $)

        7  May start or continue a Unicode identifier or Java identifier

        Thus:

           5, 6, 7 may start a Java identifier

           1, 2, 3, 5, 6, 7 may continue a Java identifier

           7 may start a Unicode identifier

           1, 3, 5, 7 may continue a Unicode identifier

           1 is ignorable within an identifier

           4 is Java whitespace

    2 bits      This field indicates whether the character has a numeric property.

        The four possible values for this field are as follows:

        0  This character has no numeric property.

        1  Adding the digit offset to the character code and then

           masking with 0x1F will produce the desired numeric value.

        2  This character has a "strange" numeric value.

        3  A Java supradecimal digit: adding the digit offset to the

           character code, then masking with 0x1F, then adding 10

           will produce the desired numeric value.

    5 bits  The digit offset (see description of previous field)

    5 bits      Character type (see below)

    B: the high 16 bits are defined as:

    1 bit Other_Lowercase property

    1 bit Other_Uppercase property

    1 bit Other_Alphabetic property

    1 bit Other_Math property

    1 bit Ideographic property

    1 bit Noncharacter codepoint property

    */

    // bit masks identify each component of a 32-bit property field described

    // above.

    // shift* indicates how many shifts right must happen to get the

    // indicated property value in the lowest bits of the 32-bit space.

    private static final int

        shiftType           = 0,        maskType            =       0x001F,

        shiftDigitOffset    = 5,        maskDigitOffset     =       0x03E0,

        shiftNumericType    = 10,       maskNumericType     =       0x0C00,

        shiftIdentifierInfo = 12,       maskIdentifierInfo  =       0x7000,

                                        maskUnicodePart     =       0x1000,

        shiftCaseInfo       = 15,       maskCaseInfo        =      0x38000,

                                        maskLowerCase       =      0x20000,

                                        maskUpperCase       =      0x10000,

                                        maskTitleCase       =      0x08000,

        shiftCaseOffset     = 18,       maskCaseOffset      =   0x07FC0000,

        shiftCaseOffsetSign = 5,

                                        // used only when calculating and

                                        // storing digit offsets from char values

                                        maskDigit               =   0x001F,

                                        // case offset are 9 bits

                                        maskCase                =   0x01FF,

        shiftBidi           = 27,       maskBidi              = 0x78000000,

        shiftMirrored       = 31,       //maskMirrored          = 0x80000000,

        shiftPlane          = 16,       maskPlane = 0xFF0000;

    // maskMirrored needs to be long, if up 16-bit

    private static final long maskMirrored          = 0x80000000L;

    // bit masks identify the 16-bit priperty field described above, in B

    // table

    private static final long

        maskOtherLowercase  = 0x100000000L,

        maskOtherUppercase  = 0x200000000L,

        maskOtherAlphabetic = 0x400000000L,

        maskOtherMath       = 0x800000000L,

        maskIdeographic     = 0x1000000000L,

        maskNoncharacterCP  = 0x2000000000L;

    // Can compare masked values with these to determine

    // numeric or lexical types.

    public static int

        valueNotNumeric             = 0x0000,

        valueDigit                  = 0x0400,

        valueStrangeNumeric         = 0x0800,

        valueJavaSupradecimal       = 0x0C00,

        valueIgnorable              = 0x1000,

        valueJavaOnlyPart           = 0x2000,

        valueJavaUnicodePart        = 0x3000,

        valueJavaWhitespace         = 0x4000,

        valueJavaStartUnicodePart   = 0x5000,

        valueJavaOnlyStart          = 0x6000,

        valueJavaUnicodeStart       = 0x7000,

        lowJavaStart                = 0x5000,

        nonzeroJavaPart             = 0x3000,

        valueUnicodeStart           = 0x7000;

    // these values are used when only identifier properties are generated

    // for use in verifier code. Shortens the property down to a single byte.

    private static final int

        bitJavaStart            = 0x02,

        bitJavaPart             = 0x01,

        maskIsJavaIdentifierPart = bitJavaPart,

        maskIsJavaIdentifierStart = bitJavaStart;

    static int maxOffset = maskCase/2 ;

    static int minOffset = -maxOffset;

    /* The following routines provide simple, concise formatting of long integer values.

     The number in the name of the method indicates the desired number of characters

     to be produced.  If the number of digits required to represent the integer value

     is less than that number, then the output is padded on the left  with zeros

     (for hex) or with spaces (for decimal).  If the number of digits required to

     represent the integer value is greater than the desired number, then all the digits

     that are required are actually produced.

    */

    static String hex(long n) { return Long.toHexString(n).toUpperCase(); }

    static String hex2(long n) {

        String q = Long.toHexString(n & 0xFF).toUpperCase();

        return "00".substring(Math.min(2, q.length())) + q;

    }

    static String hex4(long n) {

        String q = Long.toHexString(n & 0xFFFF).toUpperCase();

        return "0000".substring(Math.min(4, q.length())) + q;

    }

    static String hex8(long n) {

        String q = Long.toHexString(n & 0xFFFFFFFFL).toUpperCase();

        return "00000000".substring(Math.min(8, q.length())) + q;

    }

    static String hex16(long n) {

        String q = Long.toHexString(n).toUpperCase();

        return "0000000000000000".substring(Math.min(16, q.length())) + q;

    }

    static String dec3(long n) {

        String q = Long.toString(n);

        return "   ".substring(Math.min(3, q.length())) + q;

    }

    static String dec5(long n) {

        String q = Long.toString(n);

        return "     ".substring(Math.min(5, q.length())) + q;

    }

    /* This routine is called when some failure occurs. */

    static void FAIL(String s) {

        System.out.println("** " + s);

    }

    /**

    * Given the data from the Unicode specification file, this routine builds a map.

    *

    * The specification file is assumed to contain its data in sorted order by

    * character code; as a result, the array passed as an argument to this method

    * has its components in the same sorted order, with one entry for each defined

    * Unicode character or character range.  (A range is indicated by two consecutive

    * entries, such that the name of the first entry begins with "<" and ends with

    * "First>" and the second entry begins with "<" and ends with "Last>".)  This is

    * therefore a sparse representation of the character property data.

    *

    * The resulting map is dense representation of the character data.  It contains

    * 2^16 = 65536 entries, each of which is a long integer.  (Right now only 32 bits

    * of this long value are used, but type long is used rather than int to facilitate

    * future extensions of this source code generator that might require more than

    * 32 bits to encode relevant character properties.)  Entry k holds the encoded

    * properties for character k.

    *

    * Method buildMap manages the transformation from the sparse representation to

    * the dense representation.  It calls method buildOne to handle the encoding

    * of character property data from a single UnicodeSpec object into 32 bits.

    * For undefined characters, method buildOne is not called and the map entry for

    * that character is set to UnicodeSpec.UNASSIGNED.

    *

    * @param data       character property data from the Unicode specification file

    * @return   an array of length 65536 with one entry for every possible char value

    *

    * @see GenerateCharacter#buildOne

    */

    static long[] buildMap(UnicodeSpec[] data, SpecialCaseMap[] specialMaps, PropList propList)

    {

        long[] result;

        if (bLatin1 == true) {

            result = new long[256];

        } else {

            result = new long[1<<16];

        }

        int k=0;

        int codePoint = plane<<16;

        UnicodeSpec nonCharSpec = new UnicodeSpec();

        for (int j = 0; j < data.length && k < result.length; j++) {

            if (data[j].codePoint == codePoint) {

                result[k] = buildOne(codePoint, data[j], specialMaps);

                ++k;

                ++codePoint;

            }

            else if(data[j].codePoint > codePoint) {

                if (data[j].name.endsWith("Last>")) {

                    // build map data for all chars except last in range

                    while (codePoint < data[j].codePoint && k < result.length) {

                        result[k] = buildOne(codePoint, data[j], specialMaps);

                        ++k;

                        ++codePoint;

                    }

                }

                else {

                    // we have a few unassigned chars before data[j].codePoint

                    while (codePoint < data[j].codePoint && k < result.length) {

                        result[k] = buildOne(codePoint, nonCharSpec, specialMaps);

                        ++k;

                        ++codePoint;

                    }

                }

                k = data[j].codePoint & 0xFFFF;

                codePoint = data[j].codePoint;

                result[k] = buildOne(codePoint, data[j], specialMaps);

                ++k;

                ++codePoint;

            }

            else {

                System.out.println("An error has occured during spec mapping.");

                System.exit(0);

            }

        }

        // if there are still unprocessed chars, process them

        // as unassigned/undefined.

        codePoint = (plane<<16) | k;

        while (k < result.length) {

            result[k] = buildOne(codePoint, nonCharSpec, specialMaps);

            ++k;

            ++codePoint;

        }

        // now add all extra supported properties from PropList, to the

        // upper 16-bit

        addExProp(result, propList, "Other_Lowercase", maskOtherLowercase);

        addExProp(result, propList, "Other_Uppercase", maskOtherUppercase);

        addExProp(result, propList, "Other_Alphabetic", maskOtherAlphabetic);

        addExProp(result, propList, "Ideographic", maskIdeographic);

        //addExProp(result, propList, "Other_Math", maskOtherMath);

        //addExProp(result, propList, "Noncharacter_CodePoint", maskNoncharacterCP);

        return result;

    }

    // The maximum and minimum offsets found while scanning the database

    static int maxOffsetSeen = 0;

    static int minOffsetSeen = 0;

    /**

     * Some Unicode separator characters are not considered Java whitespace.

     * @param c character to test

     * @return true if c in an invalid Java whitespace character, false otherwise.

     */

    static boolean isInvalidJavaWhiteSpace(int c) {

        int[] exceptions = {0x00A0, 0x2007, 0x202F, 0xFEFF};

        boolean retValue = false;

        for(int x=0;x<exceptions.length;x++) {

            if(c == exceptions[x]) {

                retValue = true;

                break;

            }

        }

        return retValue;

    }

    /**

    * Given the character property data for one Unicode character, encode the data

    * of interest into a single long integer value.  (Right now only 32 bits

    * of this long value are used, but type long is used rather than int to facilitate

    * future extensions of this source code generator that might require more than

    * 32 bits to encode relevant character properties.)

    *

    * @param c   the character code for which to encode property data

    * @param us  property data record from the Unicode specification file

    *            (its character code might not be equal to c if it specifies data

    *            for a range of characters)

    * @return   an encoded long value that contains the properties for a single char

    *

    * @see GenerateCharacter#buildMap

    */

    static long buildOne(int c, UnicodeSpec us, SpecialCaseMap[] specialMaps) {

        long resultA = 0;

        // record the general category

        resultA |= us.generalCategory;

        // record the numeric properties

        NUMERIC: {

        STRANGE: {

            int val = 0;

            // c is A-Z

            if ((c >= 0x0041) && (c <= 0x005A)) {

                val = c - 0x0041;

                resultA |= valueJavaSupradecimal;

            // c is a-z

            } else if ((c >= 0x0061) && (c <= 0x007A)) {

                val = c - 0x0061;

                resultA |= valueJavaSupradecimal;

            // c is a full-width A-Z

            } else if ((c >= 0xFF21) && (c <= 0xFF3A)) {

                val = c - 0xFF21;

                resultA |= valueJavaSupradecimal;

            // c is a full-width a-z

            } else if ((c >= 0xFF41) && (c <= 0xFF5A)) {

                val = c - 0xFF41;

                resultA |= valueJavaSupradecimal;

            } else if (us.isDecimalValue()) {

                val = us.decimalValue;

                resultA |= valueDigit;

            } else if (us.isDigitValue()) {

                val = us.digitValue;

                resultA |= valueDigit;

            } else {

                if (us.numericValue.length() == 0) {

                    break NUMERIC;                      // no numeric value at all

                } else {

                    try {

                        val = Integer.parseInt(us.numericValue);

                        if (val >= 32 || val < 0) break STRANGE;

                        if (c == 0x215F) break STRANGE;

                    } catch(NumberFormatException e) {

                        break STRANGE;

                    }

                    resultA |= valueDigit;

                }

            }

            if (val >= 32 || val < 0) break STRANGE;

            resultA |= ((val - c & maskDigit) << shiftDigitOffset);

            break NUMERIC;

        } // end STRANGE

        resultA |= valueStrangeNumeric;

        } // end NUMERIC

        // record case mapping

        int offset = 0;

        // might have a 1:M mapping

        int specialMap = SpecialCaseMap.find(c, specialCaseMaps);

        boolean bHasUpper = (us.hasUpperMap()) || (specialMap != -1);

        if (bHasUpper) {

            resultA |= maskUpperCase;

        }

        if (specialMap != -1) {

            // has mapping, but cannot record the

            // proper offset; can only flag it and provide special case

            // code in Character.java

            offset = -1;

        }

        else if (us.hasUpperMap())  {

            offset = c - us.upperMap;

        }

        if (us.hasLowerMap()) {

            resultA |= maskLowerCase;

            if (offset == 0)

                offset = us.lowerMap - c;

            else if (offset != (us.lowerMap - c)) {

                if (DEBUG) {

                FAIL("Character " + hex(c) +

                " has incompatible lowercase and uppercase mappings");

                }

            }

        }

        if ((us.hasTitleMap() && us.titleMap != us.upperMap) ||

            (bHasUpper && us.hasLowerMap())) {

            resultA |= maskTitleCase;

        }

        if (bHasUpper && !us.hasLowerMap() && !us.hasTitleMap() && verbose) {

            System.out.println("Warning: Character " + hex4(c) + " has upper but " +

                               "no title case; Java won't know this");

        }

        if (offset < minOffsetSeen) minOffsetSeen = offset;

        if (offset > maxOffsetSeen) maxOffsetSeen = offset;

        if (offset > maxOffset || offset < minOffset) {

            if (DEBUG) {

            FAIL("Case offset " + offset + " for character " + hex4(c) + " must be handled as a special case");

            }

            offset = maskCase;

        }

        resultA |= ((offset & maskCase) << shiftCaseOffset);

        // record lexical info about this character

        if (us.generalCategory == UnicodeSpec.LOWERCASE_LETTER

                || us.generalCategory == UnicodeSpec.UPPERCASE_LETTER

                || us.generalCategory == UnicodeSpec.TITLECASE_LETTER

                || us.generalCategory == UnicodeSpec.MODIFIER_LETTER

                || us.generalCategory == UnicodeSpec.OTHER_LETTER

                || us.generalCategory == UnicodeSpec.LETTER_NUMBER) {

            resultA |= valueJavaUnicodeStart;

        }

        else if (us.generalCategory == UnicodeSpec.COMBINING_SPACING_MARK