# # Copyright (C) 2002-2010, International Business Machines Corporation # and others. All Rights Reserved. # # file: word.txt # # ICU Word Break Rules # See Unicode Standard Annex #29. # These rules are based on UAX-29 Revision 16 for Unicode 6.0 # # Note: Updates to word.txt will usually need to be merged into # word_POSIX.txt and word_ja.txt also. ############################################################################## # # Character class definitions from TR 29 # ############################################################################## !!chain; # # Character Class Definitions. # $CR = [\p{Word_Break = CR}]; $LF = [\p{Word_Break = LF}]; $Newline = [\p{Word_Break = Newline}]; $Extend = [\p{Word_Break = Extend}]; $Format = [\p{Word_Break = Format}]; $Katakana = [\p{Word_Break = Katakana}]; $ALetter = [\p{Word_Break = ALetter}]; $MidNumLet = [\p{Word_Break = MidNumLet}]; $MidLetter = [\p{Word_Break = MidLetter}]; $MidNum = [\p{Word_Break = MidNum}]; $Numeric = [\p{Word_Break = Numeric}]; $ExtendNumLet = [\p{Word_Break = ExtendNumLet}]; # Dictionary character set, for triggering language-based break engines. Currently # limited to LineBreak=Complex_Context. Note that this set only works in Unicode # 5.0 or later as the definition of Complex_Context was corrected to include all # characters requiring dictionary break. $dictionary = [:LineBreak = Complex_Context:]; $Control = [\p{Grapheme_Cluster_Break = Control}]; $ALetterPlus = [$ALetter [$dictionary-$Extend-$Control]]; # Note: default ALetter does not # include the dictionary characters. # # Rules 4 Ignore Format and Extend characters, # except when they appear at the beginning of a region of text. # $KatakanaEx = $Katakana ($Extend | $Format)*; $ALetterEx = $ALetterPlus ($Extend | $Format)*; $MidNumLetEx = $MidNumLet ($Extend | $Format)*; $MidLetterEx = $MidLetter ($Extend | $Format)*; $MidNumEx = $MidNum ($Extend | $Format)*; $NumericEx = $Numeric ($Extend | $Format)*; $ExtendNumLetEx = $ExtendNumLet ($Extend | $Format)*; $Hiragana = [\p{script=Hiragana}]; $Ideographic = [\p{Ideographic}]; $HiraganaEx = $Hiragana ($Extend | $Format)*; $IdeographicEx = $Ideographic ($Extend | $Format)*; ## ------------------------------------------------- !!forward; # Rule 3 - CR x LF # $CR $LF; # Rule 4 - ignore Format and Extend characters, except when they appear at the beginning # of a region of Text. The rule here comes into play when the start of text # begins with a group of Format chars, or with a "word" consisting of a single # char that is not in any of the listed word break categories followed by # format char(s). [^$CR $LF $Newline]? ($Extend | $Format)+; $NumericEx {100}; $ALetterEx {200}; $KatakanaEx {300}; # note: these status values override those from rule 5 $HiraganaEx {300}; # by virtual of being numerically larger. $IdeographicEx {400}; # # # rule 5 # Do not break between most letters. # $ALetterEx $ALetterEx {200}; # rule 6 and 7 $ALetterEx ($MidLetterEx | $MidNumLetEx) $ALetterEx {200}; # rule 8 $NumericEx $NumericEx {100}; # rule 9 $ALetterEx $NumericEx {200}; # rule 10 $NumericEx $ALetterEx {200}; # rule 11 and 12 $NumericEx ($MidNumEx | $MidNumLetEx) $NumericEx {100}; # rule 13 $KatakanaEx $KatakanaEx {300}; # rule 13a/b $ALetterEx $ExtendNumLetEx {200}; # (13a) $NumericEx $ExtendNumLetEx {100}; # (13a) $KatakanaEx $ExtendNumLetEx {300}; # (13a) $ExtendNumLetEx $ExtendNumLetEx {200}; # (13a) $ExtendNumLetEx $ALetterEx {200}; # (13b) $ExtendNumLetEx $NumericEx {100}; # (13b) $ExtendNumLetEx $KatakanaEx {300}; # (13b) ## ------------------------------------------------- !!reverse; $BackALetterEx = ($Format | $Extend)* $ALetterPlus; $BackMidNumLetEx = ($Format | $Extend)* $MidNumLet; $BackNumericEx = ($Format | $Extend)* $Numeric; $BackMidNumEx = ($Format | $Extend)* $MidNum; $BackMidLetterEx = ($Format | $Extend)* $MidLetter; $BackKatakanaEx = ($Format | $Extend)* $Katakana; $BackExtendNumLetEx= ($Format | $Extend)* $ExtendNumLet; # rule 3 $LF $CR; # rule 4 ($Format | $Extend)* [^$CR $LF $Newline]?; # rule 5 $BackALetterEx $BackALetterEx; # rule 6 and 7 $BackALetterEx ($BackMidLetterEx | $BackMidNumLetEx) $BackALetterEx; # rule 8 $BackNumericEx $BackNumericEx; # rule 9 $BackNumericEx $BackALetterEx; # rule 10 $BackALetterEx $BackNumericEx; # rule 11 and 12 $BackNumericEx ($BackMidNumEx | $BackMidNumLetEx) $BackNumericEx; # rule 13 $BackKatakanaEx $BackKatakanaEx; # rules 13 a/b # $BackExtendNumLetEx ($BackALetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx); ($BackALetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx; ## ------------------------------------------------- !!safe_reverse; # rule 3 ($Extend | $Format)+ .?; # rule 6 ($MidLetter | $MidNumLet) $BackALetterEx; # rule 11 ($MidNum | $MidNumLet) $BackNumericEx; # For dictionary-based break $dictionary $dictionary; ## ------------------------------------------------- !!safe_forward; # rule 4 ($Extend | $Format)+ .?; # rule 6 ($MidLetterEx | $MidNumLetEx) $ALetterEx; # rule 11 ($MidNumEx | $MidNumLetEx) $NumericEx; # For dictionary-based break $dictionary $dictionary;