UTF8Handler implements Unicode aware operations for strings, these operations will be used by the Chars proxy when $KCODE is set to ‘UTF8’.

Methods
Constants
HANGUL_SBASE = 0xAC00
  Hangul character boundaries and properties
HANGUL_LBASE = 0x1100
HANGUL_VBASE = 0x1161
HANGUL_TBASE = 0x11A7
HANGUL_LCOUNT = 19
HANGUL_VCOUNT = 21
HANGUL_TCOUNT = 28
HANGUL_NCOUNT = HANGUL_VCOUNT * HANGUL_TCOUNT
HANGUL_SCOUNT = 11172
HANGUL_SLAST = HANGUL_SBASE + HANGUL_SCOUNT
HANGUL_JAMO_FIRST = 0x1100
HANGUL_JAMO_LAST = 0x11FF
UNICODE_WHITESPACE = [ (0x0009..0x000D).to_a, # White_Space # Cc [5] <control-0009>..<control-000D> 0x0020, # White_Space # Zs SPACE 0x0085, # White_Space # Cc <control-0085> 0x00A0, # White_Space # Zs NO-BREAK SPACE 0x1680, # White_Space # Zs OGHAM SPACE MARK 0x180E, # White_Space # Zs MONGOLIAN VOWEL SEPARATOR (0x2000..0x200A).to_a, # White_Space # Zs [11] EN QUAD..HAIR SPACE 0x2028, # White_Space # Zl LINE SEPARATOR 0x2029, # White_Space # Zp PARAGRAPH SEPARATOR 0x202F, # White_Space # Zs NARROW NO-BREAK SPACE 0x205F, # White_Space # Zs MEDIUM MATHEMATICAL SPACE 0x3000, # White_Space # Zs IDEOGRAPHIC SPACE ].flatten.freeze
  All the unicode whitespace
UNICODE_LEADERS_AND_TRAILERS = UNICODE_WHITESPACE + [65279]
  BOM (byte order mark) can also be seen as whitespace, it‘s a non-rendering character used to distinguish between little and big endian. This is not an issue in utf-8, so it must be ignored.
UTF8_PAT = /\A(?: [\x00-\x7f] | [\xc2-\xdf] [\x80-\xbf] | \xe0 [\xa0-\xbf] [\x80-\xbf] | [\xe1-\xef] [\x80-\xbf] [\x80-\xbf] | \xf0 [\x90-\xbf] [\x80-\xbf] [\x80-\xbf] | [\xf1-\xf3] [\x80-\xbf] [\x80-\xbf] [\x80-\xbf] | \xf4 [\x80-\x8f] [\x80-\xbf] [\x80-\xbf] )*\z/xn
  Borrowed from the Kconv library by Shinji KONO - (also as seen on the W3C site)
UNICODE_TRAILERS_PAT = /(#{codepoints_to_pattern(UNICODE_LEADERS_AND_TRAILERS)})+\Z/
UNICODE_LEADERS_PAT = /\A(#{codepoints_to_pattern(UNICODE_LEADERS_AND_TRAILERS)})+/
UCD = UnicodeDatabase.new
  UniCode Database
Public Class methods
capitalize(str)

Returns a copy of str with the first character converted to uppercase and the remainder to lowercase

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 193
193:       def capitalize(str)
194:         upcase(slice(str, 0..0)) + downcase(slice(str, 1..-1) || '')
195:       end
compose(str)

Perform composition on the characters in the string

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 229
229:       def compose(str)
230:         compose_codepoints u_unpack(str).pack('U*')
231:       end
consumes?(str)

Checks if the string is valid UTF8.

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 259
259:       def consumes?(str)
260:         # Unpack is a little bit faster than regular expressions
261:         begin
262:           str.unpack('U*')
263:           true
264:         rescue ArgumentError
265:           false
266:         end
267:       end
decompose(str)

Perform decomposition on the characters in the string

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 224
224:       def decompose(str)
225:         decompose_codepoints(:canonical, u_unpack(str)).pack('U*')
226:       end
downcase(str)

Convert characters in the string to lowercase

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 190
190:       def downcase(str); to_case :lowercase_mapping, str; end
g_length(str)

Returns the number of grapheme clusters in the string. This method is very likely to be moved or renamed in future versions.

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 271
271:       def g_length(str)
272:         g_unpack(str).length
273:       end
index(str, *args)

Returns the position of the passed argument in the string, counting in codepoints

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 139
139:       def index(str, *args)
140:         bidx = str.index(*args)
141:         bidx ? (u_unpack(str.slice(0...bidx)).size) : nil
142:       end
insert(str, offset, fragment)

Inserts the passed string at specified codepoint offsets

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 129
129:       def insert(str, offset, fragment)
130:         str.replace(
131:           u_unpack(str).insert(
132:             offset,
133:             u_unpack(fragment)
134:           ).flatten.pack('U*')
135:         )
136:       end
lstrip(str)

Does Unicode-aware lstrip

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 150
150:       def lstrip(str)
151:         str.gsub(UNICODE_LEADERS_PAT, '')
152:       end
normalize(str, form=ActiveSupport::Multibyte::DEFAULT_NORMALIZATION_FORM)

Returns the KC normalization of the string by default. NFKC is considered the best normalization form for passing strings to databases and validations.

  • str: The string to perform normalization on.
  • form: The form you want to normalize in. Should be one of the following: :c, :kc, :d or :kd.
     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 206
206:       def normalize(str, form=ActiveSupport::Multibyte::DEFAULT_NORMALIZATION_FORM)
207:         # See http://www.unicode.org/reports/tr15, Table 1
208:         codepoints = u_unpack(str)
209:         case form
210:           when :d
211:             reorder_characters(decompose_codepoints(:canonical, codepoints))
212:           when :c
213:             compose_codepoints reorder_characters(decompose_codepoints(:canonical, codepoints))
214:           when :kd
215:             reorder_characters(decompose_codepoints(:compatability, codepoints))
216:           when :kc
217:             compose_codepoints reorder_characters(decompose_codepoints(:compatability, codepoints))
218:           else
219:             raise ArgumentError, "#{form} is not a valid normalization variant", caller
220:         end.pack('U*')
221:       end
reverse(str)

Reverses codepoints in the string.

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 166
166:       def reverse(str)
167:         u_unpack(str).reverse.pack('U*')
168:       end
rstrip(str)

Does Unicode-aware rstrip

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 145
145:       def rstrip(str)
146:         str.gsub(UNICODE_TRAILERS_PAT, '')
147:       end
size(str)

Returns the number of codepoints in the string

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 160
160:       def size(str)
161:         u_unpack(str).size
162:       end
slice(str, *args)

Implements Unicode-aware slice with codepoints. Slicing on one point returns the codepoints for that character.

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 172
172:       def slice(str, *args)
173:         if (args.size == 2 && args.first.is_a?(Range))
174:           raise TypeError, 'cannot convert Range into Integer' # Do as if we were native
175:         elsif args[0].kind_of? Range
176:           cps = u_unpack(str).slice(*args)
177:           cps.nil? ? nil : cps.pack('U*')
178:         elsif args.size == 1 && args[0].kind_of?(Numeric)
179:           u_unpack(str)[args[0]]
180:         else
181:           u_unpack(str).slice(*args).pack('U*')
182:         end
183:       end
strip(str)

Removed leading and trailing whitespace

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 155
155:       def strip(str)
156:         str.gsub(UNICODE_LEADERS_PAT, '').gsub(UNICODE_TRAILERS_PAT, '')
157:       end
tidy_bytes(str)

Replaces all the non-utf-8 bytes by their iso-8859-1 or cp1252 equivalent resulting in a valid utf-8 string

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 276
276:       def tidy_bytes(str)
277:         str.split(//u).map do |c|
278:           if !UTF8_PAT.match(c)
279:             n = c.unpack('C')[0]
280:             n < 128 ? n.chr :
281:             n < 160 ? [UCD.cp1252[n] || n].pack('U') :
282:             n < 192 ? "\xC2" + n.chr : "\xC3" + (n-64).chr
283:           else
284:             c
285:           end
286:         end.join
287:       end
translate_offset(str, byte_offset)

Used to translate an offset from bytes to characters, for instance one received from a regular expression match

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 238
238:       def translate_offset(str, byte_offset)
239:         return 0 if str == ''
240:         return nil if byte_offset.nil?
241:         chunk = str[0..byte_offset]
242:         begin
243:           begin
244:             chunk.unpack('U*').length - 1
245:           rescue ArgumentError => e
246:             chunk = str[0..(byte_offset+=1)]
247:             # Stop retrying at the end of the string
248:             raise e unless byte_offset < chunk.length 
249:             # We damaged a character, retry
250:             retry
251:           end
252:         # Catch the ArgumentError so we can throw our own
253:         rescue ArgumentError 
254:           raise EncodingError.new('malformed UTF-8 character')
255:         end
256:       end
upcase(str)

Convert characters in the string to uppercase

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 187
187:       def upcase(str); to_case :uppercase_mapping, str; end
Protected Class methods
compose_codepoints(codepoints)

Compose decomposed characters to the composed form

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 393
393:       def compose_codepoints(codepoints)
394:         pos = 0
395:         eoa = codepoints.length - 1
396:         starter_pos = 0
397:         starter_char = codepoints[0]
398:         previous_combining_class = -1
399:         while pos < eoa
400:           pos += 1
401:           lindex = starter_char - HANGUL_LBASE
402:           # -- Hangul
403:           if 0 <= lindex and lindex < HANGUL_LCOUNT
404:             vindex = codepoints[starter_pos+1] - HANGUL_VBASE rescue vindex = -1
405:             if 0 <= vindex and vindex < HANGUL_VCOUNT
406:               tindex = codepoints[starter_pos+2] - HANGUL_TBASE rescue tindex = -1
407:               if 0 <= tindex and tindex < HANGUL_TCOUNT
408:                 j = starter_pos + 2
409:                 eoa -= 2
410:               else
411:                 tindex = 0
412:                 j = starter_pos + 1
413:                 eoa -= 1
414:               end
415:               codepoints[starter_pos..j] = (lindex * HANGUL_VCOUNT + vindex) * HANGUL_TCOUNT + tindex + HANGUL_SBASE
416:             end
417:             starter_pos += 1
418:             starter_char = codepoints[starter_pos]
419:           # -- Other characters
420:           else
421:             current_char = codepoints[pos]
422:             current = UCD[current_char]
423:             if current.combining_class > previous_combining_class
424:               if ref = UCD.composition_map[starter_char]
425:                 composition = ref[current_char]
426:               else
427:                 composition = nil
428:               end
429:               unless composition.nil?
430:                 codepoints[starter_pos] = composition
431:                 starter_char = composition
432:                 codepoints.delete_at pos
433:                 eoa -= 1
434:                 pos -= 1
435:                 previous_combining_class = -1
436:               else
437:                 previous_combining_class = current.combining_class
438:               end
439:             else
440:               previous_combining_class = current.combining_class
441:             end
442:             if current.combining_class == 0
443:               starter_pos = pos
444:               starter_char = codepoints[pos]
445:             end
446:           end
447:         end
448:         codepoints
449:       end
decompose_codepoints(type, codepoints)

Decompose composed characters to the decomposed form

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 372
372:       def decompose_codepoints(type, codepoints)
373:         codepoints.inject([]) do |decomposed, cp|
374:           # if it's a hangul syllable starter character
375:           if HANGUL_SBASE <= cp and cp < HANGUL_SLAST
376:             sindex = cp - HANGUL_SBASE
377:             ncp = [] # new codepoints
378:             ncp << HANGUL_LBASE + sindex / HANGUL_NCOUNT
379:             ncp << HANGUL_VBASE + (sindex % HANGUL_NCOUNT) / HANGUL_TCOUNT
380:             tindex = sindex % HANGUL_TCOUNT
381:             ncp << (HANGUL_TBASE + tindex) unless tindex == 0
382:             decomposed.concat ncp
383:           # if the codepoint is decomposable in with the current decomposition type
384:           elsif (ncp = UCD[cp].decomp_mapping) and (!UCD[cp].decomp_type || type == :compatability)
385:             decomposed.concat decompose_codepoints(type, ncp.dup)
386:           else
387:             decomposed << cp
388:           end
389:         end
390:       end
g_pack(unpacked)

Reverse operation of g_unpack

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 338
338:       def g_pack(unpacked)
339:         unpacked.flatten
340:       end
g_unpack(str)

Unpack the string at grapheme boundaries instead of codepoint boundaries

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 307
307:       def g_unpack(str)
308:         codepoints = u_unpack(str)
309:         unpacked = []
310:         pos = 0
311:         marker = 0
312:         eoc = codepoints.length
313:         while(pos < eoc)
314:           pos += 1
315:           previous = codepoints[pos-1]
316:           current = codepoints[pos]
317:           if (
318:               # CR X LF
319:               one = ( previous == UCD.boundary[:cr] and current == UCD.boundary[:lf] ) or
320:               # L X (L|V|LV|LVT)
321:               two = ( UCD.boundary[:l] === previous and in_char_class?(current, [:l,:v,:lv,:lvt]) ) or
322:               # (LV|V) X (V|T)
323:               three = ( in_char_class?(previous, [:lv,:v]) and in_char_class?(current, [:v,:t]) ) or
324:               # (LVT|T) X (T)
325:               four = ( in_char_class?(previous, [:lvt,:t]) and UCD.boundary[:t] === current ) or
326:               # X Extend
327:               five = (UCD.boundary[:extend] === current)
328:             )
329:           else
330:             unpacked << codepoints[marker..pos-1]
331:             marker = pos
332:           end
333:         end 
334:         unpacked
335:       end
in_char_class?(codepoint, classes)

Detect whether the codepoint is in a certain character class. Primarily used by the grapheme cluster support.

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 293
293:       def in_char_class?(codepoint, classes)
294:         classes.detect { |c| UCD.boundary[c] === codepoint } ? true : false
295:       end
reorder_characters(codepoints)

Re-order codepoints so the string becomes canonical

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 356
356:       def reorder_characters(codepoints)
357:         length = codepoints.length- 1
358:         pos = 0
359:         while pos < length do
360:           cp1, cp2 = UCD[codepoints[pos]], UCD[codepoints[pos+1]]
361:           if (cp1.combining_class > cp2.combining_class) && (cp2.combining_class > 0)
362:             codepoints[pos..pos+1] = cp2.code, cp1.code
363:             pos += (pos > 0 ? -1 : 1)
364:           else
365:             pos += 1
366:           end
367:         end
368:         codepoints
369:       end
to_case(way, str)

Convert characters to a different case

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 343
343:       def to_case(way, str)
344:         u_unpack(str).map do |codepoint|
345:           cp = UCD[codepoint] 
346:           unless cp.nil?
347:             ncp = cp.send(way)
348:             ncp > 0 ? ncp : codepoint
349:           else
350:             codepoint
351:           end
352:         end.pack('U*')
353:       end
u_unpack(str)

Unpack the string at codepoints boundaries

     # File vendor/rails/activesupport/lib/active_support/multibyte/handlers/utf8_handler.rb, line 298
298:       def u_unpack(str)
299:         begin
300:           str.unpack 'U*'
301:         rescue ArgumentError
302:           raise EncodingError.new('malformed UTF-8 character')
303:         end
304:       end