const pcre_uint32 PRIV(ucd_caseless_sets)[] = {0};

const pcre_uint32 PRIV(ucd_caseless_sets)[] = {0};

#else

#else

/* When recompiling tables with a new Unicode version, please check the

/* When recompiling tables with a new Unicode version, please check the

types in this structure definition from pcre_internal.h (the actual

types in this structure definition from pcre_internal.h (the actual

field names will be different):

field names will be different):

extern const pcre_uint16 PRIV(ucd_stage2)[];

extern const pcre_uint16 PRIV(ucd_stage2)[];

extern const pcre_uint32 PRIV(ucp_gentype)[];

extern const pcre_uint32 PRIV(ucp_gentype)[];

extern const pcre_uint32 PRIV(ucp_gbtable)[];

extern const pcre_uint32 PRIV(ucp_gbtable)[];

#ifdef SUPPORT_JIT

#ifdef SUPPORT_JIT

extern const int         PRIV(ucp_typerange)[];

extern const int         PRIV(ucp_typerange)[];

#endif

#endif

/* UCD access macros */

/* UCD access macros */

#define UCD_BLOCK_SIZE 128

#define UCD_BLOCK_SIZE 128

        PRIV(ucd_stage2)[PRIV(ucd_stage1)[(int)(ch) / UCD_BLOCK_SIZE] * \

        PRIV(ucd_stage2)[PRIV(ucd_stage1)[(int)(ch) / UCD_BLOCK_SIZE] * \

        UCD_BLOCK_SIZE + (int)(ch) % UCD_BLOCK_SIZE])

        UCD_BLOCK_SIZE + (int)(ch) % UCD_BLOCK_SIZE])

#define UCD_CHARTYPE(ch)    GET_UCD(ch)->chartype

#define UCD_CHARTYPE(ch)    GET_UCD(ch)->chartype

#define UCD_SCRIPT(ch)      GET_UCD(ch)->script

#define UCD_SCRIPT(ch)      GET_UCD(ch)->script

1.  Fixed typo in CMakeLists.txt (wrong number of arguments for 

1.  Fixed typo in CMakeLists.txt (wrong number of arguments for 

PCRE_STATIC_RUNTIME (affects MSVC only).

PCRE_STATIC_RUNTIME (affects MSVC only).

mode with --only-matching matched several lines, it restarted scanning at the 

mode with --only-matching matched several lines, it restarted scanning at the 

next line instead of moving on to the end of the matched string, which can be 

next line instead of moving on to the end of the matched string, which can be 

several lines after the start.

several lines after the start.

    (a) Check for values < 256 when calling isprint() in pcretest.

    (a) Check for values < 256 when calling isprint() in pcretest.

    (b) Give an error for too big a number after \O. 

    (b) Give an error for too big a number after \O. 

Version 8.40 11-January-2017

Version 8.40 11-January-2017

#! /usr/bin/python

#! /usr/bin/python

# Multistage table builder

# Multistage table builder

# (c) Peter Kankowski, 2008

# (c) Peter Kankowski, 2008

# ./MultiStage2.py >../pcre_ucd.c

# ./MultiStage2.py >../pcre_ucd.c

#

#

# It requires four Unicode data tables, DerivedGeneralCategory.txt,

# It requires four Unicode data tables, DerivedGeneralCategory.txt,

# UCD directory.

# UCD directory.

#

#

# Minor modifications made to this script:

# Minor modifications made to this script:

#  code scans CaseFolding.txt instead of UnicodeData.txt.

#  code scans CaseFolding.txt instead of UnicodeData.txt.

#

#

# The main tables generated by this script are used by macros defined in

# The main tables generated by this script are used by macros defined in

# sequences of code that contains no branches, which makes for greater speed.

# sequences of code that contains no branches, which makes for greater speed.

#

#

# Conceptually, there is a table of records (of type ucd_record), containing a

# Conceptually, there is a table of records (of type ucd_record), containing a

# Example: lowercase "a" (U+0061) is in block 0

# Example: lowercase "a" (U+0061) is in block 0

#          lookup 0 in stage1 table yields 0

#          lookup 0 in stage1 table yields 0

#          lookup 97 in the first table in stage2 yields 16

#          lookup 97 in the first table in stage2 yields 16

#            33 = ucp_Latin   => Latin script

#            33 = ucp_Latin   => Latin script

#             5 = ucp_Ll      => Lower case letter

#             5 = ucp_Ll      => Lower case letter

#            11 = ucp_gbOther => Grapheme break property "Other"

#            11 = ucp_gbOther => Grapheme break property "Other"

#             0               => not part of a caseless set

#             0               => not part of a caseless set

#           -32               => Other case is U+0041

#           -32               => Other case is U+0041

# Almost all lowercase latin characters resolve to the same record. One or two

# Almost all lowercase latin characters resolve to the same record. One or two

# are different because they are part of a multi-character caseless set (for

# are different because they are part of a multi-character caseless set (for

# example, k, K and the Kelvin symbol are such a set).

# example, k, K and the Kelvin symbol are such a set).

# Example: hiragana letter A (U+3042) is in block 96 (0x60)

# Example: hiragana letter A (U+3042) is in block 96 (0x60)

#          lookup 96 in stage1 table yields 88

#          lookup 96 in stage1 table yields 88

#          lookup 66 in the 88th table in stage2 yields 467

#          lookup 66 in the 88th table in stage2 yields 467

#            26 = ucp_Hiragana => Hiragana script

#            26 = ucp_Hiragana => Hiragana script

#             7 = ucp_Lo       => Other letter

#             7 = ucp_Lo       => Other letter

#            11 = ucp_gbOther  => Grapheme break property "Other"

#            11 = ucp_gbOther  => Grapheme break property "Other"

#             0                => not part of a caseless set

#             0                => not part of a caseless set

#

#

# In these examples, no other blocks resolve to the same "virtual" block, as it

# In these examples, no other blocks resolve to the same "virtual" block, as it

# happens, but plenty of other blocks do share "virtual" blocks.

# happens, but plenty of other blocks do share "virtual" blocks.

#

#

# individual character types such as ucp_Cc to the general types like ucp_C.

# individual character types such as ucp_Cc to the general types like ucp_C.

#

#

#  Philip Hazel, 03 July 2008

#  Philip Hazel, 03 July 2008

# 01-March-2010:     Updated list of scripts for Unicode 5.2.0

# 01-March-2010:     Updated list of scripts for Unicode 5.2.0

# 30-April-2011:     Updated list of scripts for Unicode 6.0.0

# 30-April-2011:     Updated list of scripts for Unicode 6.0.0

#     July-2012:     Updated list of scripts for Unicode 6.1.0

#     July-2012:     Updated list of scripts for Unicode 6.1.0

#                      structure had a hole in it, so the resulting table is

#                      structure had a hole in it, so the resulting table is

#                      not much bigger than before.

#                      not much bigger than before.

# 18-September-2012: Added code for multiple caseless sets. This uses the

# 18-September-2012: Added code for multiple caseless sets. This uses the

                if m.group(3) is None:

                if m.group(3) is None:

                        last = char

                        last = char

                else:

                else:

                for i in range(char, last + 1):

                for i in range(char, last + 1):

                        # It is important not to overwrite a previously set

                        # It is important not to overwrite a previously set

                        # value because in the CaseFolding file there are lines

                        # value because in the CaseFolding file there are lines

                          table[i] = value

                          table[i] = value

        file.close()

        file.close()

        return table

        return table

                        stage2 += block

                        stage2 += block

                        blocks[block] = start

                        blocks[block] = start

                stage1.append(start)

                stage1.append(start)

        return stage1, stage2

        return stage1, stage2

# Print a table

# Print a table

def print_table(table, table_name, block_size = None):

def print_table(table, table_name, block_size = None):

        type, size = get_type_size(table)

        type, size = get_type_size(table)

        ELEMS_PER_LINE = 16

        ELEMS_PER_LINE = 16

        s = "const %s %s[] = { /* %d bytes" % (type, table_name, size * len(table))

        s = "const %s %s[] = { /* %d bytes" % (type, table_name, size * len(table))

        if block_size:

        if block_size:

                s += ", block = %d" % block_size

                s += ", block = %d" % block_size

                size = (size + slice_size - 1) & -slice_size

                size = (size + slice_size - 1) & -slice_size

                size += slice_size

                size += slice_size

                structure += '%s property_%d;\n' % (slice_type, i)

                structure += '%s property_%d;\n' % (slice_type, i)

        # round up to the first item of the next structure in array

        # round up to the first item of the next structure in array

        record_slice = map(lambda record: record[0], records)

        record_slice = map(lambda record: record[0], records)

        slice_type, slice_size = get_type_size(record_slice)

        slice_type, slice_size = get_type_size(record_slice)

        size = (size + slice_size - 1) & -slice_size

        size = (size + slice_size - 1) & -slice_size

        structure += '} ucd_record;\n*/\n\n'

        structure += '} ucd_record;\n*/\n\n'

        return size, structure

        return size, structure

def test_record_size():

def test_record_size():

        tests = [ \

        tests = [ \

          ( [(3,), (6,), (6,), (1,)], 1 ), \

          ( [(3,), (6,), (6,), (1,)], 1 ), \

 'Old_North_Arabian', 'Old_Permic', 'Pahawh_Hmong', 'Palmyrene', 'Psalter_Pahlavi',

 'Old_North_Arabian', 'Old_Permic', 'Pahawh_Hmong', 'Palmyrene', 'Psalter_Pahlavi',

 'Pau_Cin_Hau', 'Siddham', 'Tirhuta', 'Warang_Citi'

 'Pau_Cin_Hau', 'Siddham', 'Tirhuta', 'Warang_Citi'

 ]

 ]

category_names = ['Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Ll', 'Lm', 'Lo', 'Lt', 'Lu',

category_names = ['Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Ll', 'Lm', 'Lo', 'Lt', 'Lu',

  'Mc', 'Me', 'Mn', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Pe', 'Pf', 'Pi', 'Po', 'Ps',

  'Mc', 'Me', 'Mn', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Pe', 'Pf', 'Pi', 'Po', 'Ps',

  'Sc', 'Sk', 'Sm', 'So', 'Zl', 'Zp', 'Zs' ]

  'Sc', 'Sk', 'Sm', 'So', 'Zl', 'Zp', 'Zs' ]

other_case = read_table('Unicode.tables/CaseFolding.txt', get_other_case, 0)

other_case = read_table('Unicode.tables/CaseFolding.txt', get_other_case, 0)

# offsets in the table that are inserted into the main table.

# offsets in the table that are inserted into the main table.

# The CaseFolding.txt file lists pairs, but the common logic for reading data

# The CaseFolding.txt file lists pairs, but the common logic for reading data

# offsets for those that are not already set.

# offsets for those that are not already set.

for c in range(0x10ffff):

for c in range(0x10ffff):

  if other_case[c] != 0 and other_case[c + other_case[c]] == 0:

  if other_case[c] != 0 and other_case[c + other_case[c]] == 0:

# Now scan again and create equivalence sets.

# Now scan again and create equivalence sets.

  o = c + other_case[c]

  o = c + other_case[c]

  # Trigger when this character's other case does not point back here. We

  # Trigger when this character's other case does not point back here. We

  if other_case[o] != -other_case[c]:

  if other_case[o] != -other_case[c]:

    t = o + other_case[o]

    t = o + other_case[o]

    # part of a set. If so, unite the existing set with the new set.

    # part of a set. If so, unite the existing set with the new set.

    for s in sets:

    for s in sets:

      for x in s:

      for x in s:

        if x == c or x == o or x == t:

        if x == c or x == o or x == t:

          found = 1

          found = 1

      # Add new characters to an existing set

      # Add new characters to an existing set

      if found:

      if found:

        for y in [c, o, t]:

        for y in [c, o, t]:

          for x in s:

          for x in s:

            if x == y:

            if x == y:

          if not found:

          if not found:

            s.append(y)

            s.append(y)

        appended = 1

        appended = 1

    # If we have not added to an existing set, create a new one.

    # If we have not added to an existing set, create a new one.

      sets.append([c, o, t])

      sets.append([c, o, t])

# End of loop looking for caseless sets.

# End of loop looking for caseless sets.

offset = 1;

offset = 1;

for s in sets:

for s in sets:

    caseless_offsets[x] = offset

    caseless_offsets[x] = offset

  offset += len(s) + 1

  offset += len(s) + 1

# Combine the tables

# Combine the tables

  caseless_offsets, other_case)

  caseless_offsets, other_case)

record_size, record_struct = get_record_size_struct(records.keys())

record_size, record_struct = get_record_size_struct(records.keys())

print "const pcre_uint32 PRIV(ucd_caseless_sets)[] = {0};"

print "const pcre_uint32 PRIV(ucd_caseless_sets)[] = {0};"

print "#else"

print "#else"

print

print

print record_struct

print record_struct

# --- Added by PH: output the table of caseless character sets ---

# --- Added by PH: output the table of caseless character sets ---

  s = sorted(s)

  s = sorted(s)

  for x in s:

  for x in s:

    print '  0x%04x,' % x,

    print '  0x%04x,' % x,

print '};'

print '};'

print

print