Edit File: display.py
# -*- coding: utf-8 -*- # # Copyright (c) 2010 Red Hat, Inc. # Copyright (c) 2010 Ville Skyttä # Copyright (c) 2009 Tim Lauridsen # Copyright (c) 2007 Marcus Kuhn # # kitchen is free software; you can redistribute it and/or modify it under the # terms of the GNU Lesser General Public License as published by the Free # Software Foundation; either version 2.1 of the License, or (at your option) # any later version. # # kitchen is distributed in the hope that it will be useful, but WITHOUT ANY # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for # more details. # # You should have received a copy of the GNU Lesser General Public License # along with kitchen; if not, see <http://www.gnu.org/licenses/> # # Authors: # James Antill <james@fedoraproject.org> # Marcus Kuhn # Toshio Kuratomi <toshio@fedoraproject.org> # Tim Lauridsen # Ville Skyttä # # Portions of this are from yum/i18n.py ''' ----------------------- Format Text for Display ----------------------- Functions related to displaying unicode text. Unicode characters don't all have the same width so we need helper functions for displaying them. .. versionadded:: 0.2 kitchen.display API 1.0.0 ''' import itertools import unicodedata from kitchen import b_ from kitchen.text.converters import to_unicode, to_bytes from kitchen.text.exceptions import ControlCharError # This is ported from ustr_utf8_* which I got from: # http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c # I've tried to leave it close to the original C (same names etc.) so that # it is easy to read/compare both versions... James Antilles # # Reimplemented quite a bit of this for speed. Use the bzr log or annotate # commands to see what I've changed since importing this file.-Toshio Kuratomi # ----------------------------- BEG utf8 ------------------to----------- # This is an implementation of wcwidth() and wcswidth() (defined in # IEEE Std 1002.1-2001) for Unicode. # # http://www.opengroup.org/onlinepubs/007904975/functions/wcwidth.html # http://www.opengroup.org/onlinepubs/007904975/functions/wcswidth.html # # In fixed-width output devices, Latin characters all occupy a single # "cell" position of equal width, whereas ideographic CJK characters # occupy two such cells. Interoperability between terminal-line # applications and (teletype-style) character terminals using the # UTF-8 encoding requires agreement on which character should advance # the cursor by how many cell positions. No established formal # standards exist at present on which Unicode character shall occupy # how many cell positions on character terminals. These routines are # a first attempt of defining such behavior based on simple rules # applied to data provided by the Unicode Consortium. # # [...] # # Markus Kuhn -- 2007-05-26 (Unicode 5.0) # # Permission to use, copy, modify, and distribute this software # for any purpose and without fee is hereby granted. The author # disclaims all warranties with regard to this software. # # Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c # Renamed but still pretty much JA's port of MK's code def _interval_bisearch(value, table): '''Binary search in an interval table. :arg value: numeric value to search for :arg table: Ordered list of intervals. This is a list of two-tuples. The elements of the two-tuple define an interval's start and end points. :returns: If :attr:`value` is found within an interval in the :attr:`table` return :data:`True`. Otherwise, :data:`False` This function checks whether a numeric value is present within a table of intervals. It checks using a binary search algorithm, dividing the list of values in half and checking against the values until it determines whether the value is in the table. ''' minimum = 0 maximum = len(table) - 1 if value < table[minimum][0] or value > table[maximum][1]: return False while maximum >= minimum: mid = (minimum + maximum) / 2 if value > table[mid][1]: minimum = mid + 1 elif value < table[mid][0]: maximum = mid - 1 else: return True return False _COMBINING = ( (0x300, 0x36f), (0x483, 0x489), (0x591, 0x5bd), (0x5bf, 0x5bf), (0x5c1, 0x5c2), (0x5c4, 0x5c5), (0x5c7, 0x5c7), (0x600, 0x603), (0x610, 0x61a), (0x64b, 0x65e), (0x670, 0x670), (0x6d6, 0x6e4), (0x6e7, 0x6e8), (0x6ea, 0x6ed), (0x70f, 0x70f), (0x711, 0x711), (0x730, 0x74a), (0x7a6, 0x7b0), (0x7eb, 0x7f3), (0x816, 0x819), (0x81b, 0x823), (0x825, 0x827), (0x829, 0x82d), (0x901, 0x902), (0x93c, 0x93c), (0x941, 0x948), (0x94d, 0x94d), (0x951, 0x954), (0x962, 0x963), (0x981, 0x981), (0x9bc, 0x9bc), (0x9c1, 0x9c4), (0x9cd, 0x9cd), (0x9e2, 0x9e3), (0xa01, 0xa02), (0xa3c, 0xa3c), (0xa41, 0xa42), (0xa47, 0xa48), (0xa4b, 0xa4d), (0xa70, 0xa71), (0xa81, 0xa82), (0xabc, 0xabc), (0xac1, 0xac5), (0xac7, 0xac8), (0xacd, 0xacd), (0xae2, 0xae3), (0xb01, 0xb01), (0xb3c, 0xb3c), (0xb3f, 0xb3f), (0xb41, 0xb43), (0xb4d, 0xb4d), (0xb56, 0xb56), (0xb82, 0xb82), (0xbc0, 0xbc0), (0xbcd, 0xbcd), (0xc3e, 0xc40), (0xc46, 0xc48), (0xc4a, 0xc4d), (0xc55, 0xc56), (0xcbc, 0xcbc), (0xcbf, 0xcbf), (0xcc6, 0xcc6), (0xccc, 0xccd), (0xce2, 0xce3), (0xd41, 0xd43), (0xd4d, 0xd4d), (0xdca, 0xdca), (0xdd2, 0xdd4), (0xdd6, 0xdd6), (0xe31, 0xe31), (0xe34, 0xe3a), (0xe47, 0xe4e), (0xeb1, 0xeb1), (0xeb4, 0xeb9), (0xebb, 0xebc), (0xec8, 0xecd), (0xf18, 0xf19), (0xf35, 0xf35), (0xf37, 0xf37), (0xf39, 0xf39), (0xf71, 0xf7e), (0xf80, 0xf84), (0xf86, 0xf87), (0xf90, 0xf97), (0xf99, 0xfbc), (0xfc6, 0xfc6), (0x102d, 0x1030), (0x1032, 0x1032), (0x1036, 0x1037), (0x1039, 0x103a), (0x1058, 0x1059), (0x108d, 0x108d), (0x1160, 0x11ff), (0x135f, 0x135f), (0x1712, 0x1714), (0x1732, 0x1734), (0x1752, 0x1753), (0x1772, 0x1773), (0x17b4, 0x17b5), (0x17b7, 0x17bd), (0x17c6, 0x17c6), (0x17c9, 0x17d3), (0x17dd, 0x17dd), (0x180b, 0x180d), (0x18a9, 0x18a9), (0x1920, 0x1922), (0x1927, 0x1928), (0x1932, 0x1932), (0x1939, 0x193b), (0x1a17, 0x1a18), (0x1a60, 0x1a60), (0x1a75, 0x1a7c), (0x1a7f, 0x1a7f), (0x1b00, 0x1b03), (0x1b34, 0x1b34), (0x1b36, 0x1b3a), (0x1b3c, 0x1b3c), (0x1b42, 0x1b42), (0x1b44, 0x1b44), (0x1b6b, 0x1b73), (0x1baa, 0x1baa), (0x1c37, 0x1c37), (0x1cd0, 0x1cd2), (0x1cd4, 0x1ce0), (0x1ce2, 0x1ce8), (0x1ced, 0x1ced), (0x1dc0, 0x1de6), (0x1dfd, 0x1dff), (0x200b, 0x200f), (0x202a, 0x202e), (0x2060, 0x2063), (0x206a, 0x206f), (0x20d0, 0x20f0), (0x2cef, 0x2cf1), (0x2de0, 0x2dff), (0x302a, 0x302f), (0x3099, 0x309a), (0xa66f, 0xa66f), (0xa67c, 0xa67d), (0xa6f0, 0xa6f1), (0xa806, 0xa806), (0xa80b, 0xa80b), (0xa825, 0xa826), (0xa8c4, 0xa8c4), (0xa8e0, 0xa8f1), (0xa92b, 0xa92d), (0xa953, 0xa953), (0xa9b3, 0xa9b3), (0xa9c0, 0xa9c0), (0xaab0, 0xaab0), (0xaab2, 0xaab4), (0xaab7, 0xaab8), (0xaabe, 0xaabf), (0xaac1, 0xaac1), (0xabed, 0xabed), (0xfb1e, 0xfb1e), (0xfe00, 0xfe0f), (0xfe20, 0xfe26), (0xfeff, 0xfeff), (0xfff9, 0xfffb), (0x101fd, 0x101fd), (0x10a01, 0x10a03), (0x10a05, 0x10a06), (0x10a0c, 0x10a0f), (0x10a38, 0x10a3a), (0x10a3f, 0x10a3f), (0x110b9, 0x110ba), (0x1d165, 0x1d169), (0x1d16d, 0x1d182), (0x1d185, 0x1d18b), (0x1d1aa, 0x1d1ad), (0x1d242, 0x1d244), (0xe0001, 0xe0001), (0xe0020, 0xe007f), (0xe0100, 0xe01ef), ) ''' Internal table, provided by this module to list :term:`code points` which combine with other characters and therefore should have no :term:`textual width`. This is a sorted :class:`tuple` of non-overlapping intervals. Each interval is a :class:`tuple` listing a starting :term:`code point` and ending :term:`code point`. Every :term:`code point` between the two end points is a combining character. .. seealso:: :func:`~kitchen.text.display._generate_combining_table` for how this table is generated This table was last regenerated on python-2.7.0 with :data:`unicodedata.unidata_version` 5.1.0 ''' # New function from Toshio Kuratomi (LGPLv2+) def _generate_combining_table(): '''Combine Markus Kuhn's data with :mod:`unicodedata` to make combining char list :rtype: :class:`tuple` of tuples :returns: :class:`tuple` of intervals of :term:`code points` that are combining character. Each interval is a 2-:class:`tuple` of the starting :term:`code point` and the ending :term:`code point` for the combining characters. In normal use, this function serves to tell how we're generating the combining char list. For speed reasons, we use this to generate a static list and just use that later. Markus Kuhn's list of combining characters is more complete than what's in the python :mod:`unicodedata` library but the python :mod:`unicodedata` is synced against later versions of the unicode database This is used to generate the :data:`~kitchen.text.display._COMBINING` table. ''' # Marcus Kuhn's sorted list of non-overlapping intervals of non-spacing # characters generated ifrom Unicode 5.0 data by: # "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" markus_kuhn_combining_5_0 = ( ( 0x0300, 0x036F ), ( 0x0483, 0x0486 ), ( 0x0488, 0x0489 ), ( 0x0591, 0x05BD ), ( 0x05BF, 0x05BF ), ( 0x05C1, 0x05C2 ), ( 0x05C4, 0x05C5 ), ( 0x05C7, 0x05C7 ), ( 0x0600, 0x0603 ), ( 0x0610, 0x0615 ), ( 0x064B, 0x065E ), ( 0x0670, 0x0670 ), ( 0x06D6, 0x06E4 ), ( 0x06E7, 0x06E8 ), ( 0x06EA, 0x06ED ), ( 0x070F, 0x070F ), ( 0x0711, 0x0711 ), ( 0x0730, 0x074A ), ( 0x07A6, 0x07B0 ), ( 0x07EB, 0x07F3 ), ( 0x0901, 0x0902 ), ( 0x093C, 0x093C ), ( 0x0941, 0x0948 ), ( 0x094D, 0x094D ), ( 0x0951, 0x0954 ), ( 0x0962, 0x0963 ), ( 0x0981, 0x0981 ), ( 0x09BC, 0x09BC ), ( 0x09C1, 0x09C4 ), ( 0x09CD, 0x09CD ), ( 0x09E2, 0x09E3 ), ( 0x0A01, 0x0A02 ), ( 0x0A3C, 0x0A3C ), ( 0x0A41, 0x0A42 ), ( 0x0A47, 0x0A48 ), ( 0x0A4B, 0x0A4D ), ( 0x0A70, 0x0A71 ), ( 0x0A81, 0x0A82 ), ( 0x0ABC, 0x0ABC ), ( 0x0AC1, 0x0AC5 ), ( 0x0AC7, 0x0AC8 ), ( 0x0ACD, 0x0ACD ), ( 0x0AE2, 0x0AE3 ), ( 0x0B01, 0x0B01 ), ( 0x0B3C, 0x0B3C ), ( 0x0B3F, 0x0B3F ), ( 0x0B41, 0x0B43 ), ( 0x0B4D, 0x0B4D ), ( 0x0B56, 0x0B56 ), ( 0x0B82, 0x0B82 ), ( 0x0BC0, 0x0BC0 ), ( 0x0BCD, 0x0BCD ), ( 0x0C3E, 0x0C40 ), ( 0x0C46, 0x0C48 ), ( 0x0C4A, 0x0C4D ), ( 0x0C55, 0x0C56 ), ( 0x0CBC, 0x0CBC ), ( 0x0CBF, 0x0CBF ), ( 0x0CC6, 0x0CC6 ), ( 0x0CCC, 0x0CCD ), ( 0x0CE2, 0x0CE3 ), ( 0x0D41, 0x0D43 ), ( 0x0D4D, 0x0D4D ), ( 0x0DCA, 0x0DCA ), ( 0x0DD2, 0x0DD4 ), ( 0x0DD6, 0x0DD6 ), ( 0x0E31, 0x0E31 ), ( 0x0E34, 0x0E3A ), ( 0x0E47, 0x0E4E ), ( 0x0EB1, 0x0EB1 ), ( 0x0EB4, 0x0EB9 ), ( 0x0EBB, 0x0EBC ), ( 0x0EC8, 0x0ECD ), ( 0x0F18, 0x0F19 ), ( 0x0F35, 0x0F35 ), ( 0x0F37, 0x0F37 ), ( 0x0F39, 0x0F39 ), ( 0x0F71, 0x0F7E ), ( 0x0F80, 0x0F84 ), ( 0x0F86, 0x0F87 ), ( 0x0F90, 0x0F97 ), ( 0x0F99, 0x0FBC ), ( 0x0FC6, 0x0FC6 ), ( 0x102D, 0x1030 ), ( 0x1032, 0x1032 ), ( 0x1036, 0x1037 ), ( 0x1039, 0x1039 ), ( 0x1058, 0x1059 ), ( 0x1160, 0x11FF ), ( 0x135F, 0x135F ), ( 0x1712, 0x1714 ), ( 0x1732, 0x1734 ), ( 0x1752, 0x1753 ), ( 0x1772, 0x1773 ), ( 0x17B4, 0x17B5 ), ( 0x17B7, 0x17BD ), ( 0x17C6, 0x17C6 ), ( 0x17C9, 0x17D3 ), ( 0x17DD, 0x17DD ), ( 0x180B, 0x180D ), ( 0x18A9, 0x18A9 ), ( 0x1920, 0x1922 ), ( 0x1927, 0x1928 ), ( 0x1932, 0x1932 ), ( 0x1939, 0x193B ), ( 0x1A17, 0x1A18 ), ( 0x1B00, 0x1B03 ), ( 0x1B34, 0x1B34 ), ( 0x1B36, 0x1B3A ), ( 0x1B3C, 0x1B3C ), ( 0x1B42, 0x1B42 ), ( 0x1B6B, 0x1B73 ), ( 0x1DC0, 0x1DCA ), ( 0x1DFE, 0x1DFF ), ( 0x200B, 0x200F ), ( 0x202A, 0x202E ), ( 0x2060, 0x2063 ), ( 0x206A, 0x206F ), ( 0x20D0, 0x20EF ), ( 0x302A, 0x302F ), ( 0x3099, 0x309A ), ( 0xA806, 0xA806 ), ( 0xA80B, 0xA80B ), ( 0xA825, 0xA826 ), ( 0xFB1E, 0xFB1E ), ( 0xFE00, 0xFE0F ), ( 0xFE20, 0xFE23 ), ( 0xFEFF, 0xFEFF ), ( 0xFFF9, 0xFFFB ), ( 0x10A01, 0x10A03 ), ( 0x10A05, 0x10A06 ), ( 0x10A0C, 0x10A0F ), ( 0x10A38, 0x10A3A ), ( 0x10A3F, 0x10A3F ), ( 0x1D167, 0x1D169 ), ( 0x1D173, 0x1D182 ), ( 0x1D185, 0x1D18B ), ( 0x1D1AA, 0x1D1AD ), ( 0x1D242, 0x1D244 ), ( 0xE0001, 0xE0001 ), ( 0xE0020, 0xE007F ), ( 0xE0100, 0xE01EF )) combining = [] in_interval = False interval = [] for codepoint in xrange (0, 0xFFFFF + 1): if _interval_bisearch(codepoint, markus_kuhn_combining_5_0) or \ unicodedata.combining(unichr(codepoint)): if not in_interval: # Found first part of an interval interval = [codepoint] in_interval = True else: if in_interval: in_interval = False interval.append(codepoint - 1) combining.append(interval) if in_interval: # If we're at the end and the interval is open, close it. # :W0631: We looped through a static range so we know codepoint is # defined here #pylint:disable-msg=W0631 interval.append(codepoint) combining.append(interval) return tuple(itertools.imap(tuple, combining)) # New function from Toshio Kuratomi (LGPLv2+) def _print_combining_table(): '''Print out a new :data:`_COMBINING` table This will print a new :data:`_COMBINING` table in the format used in :file:`kitchen/text/display.py`. It's useful for updating the :data:`_COMBINING` table with updated data from a new python as the format won't change from what's already in the file. ''' table = _generate_combining_table() entries = 0 print '_COMBINING = (' for pair in table: if entries >= 3: entries = 0 print if entries == 0: print ' ', entries += 1 entry = '(0x%x, 0x%x),' % pair print entry, print ')' # Handling of control chars rewritten. Rest is JA's port of MK's C code. # -Toshio Kuratomi def _ucp_width(ucs, control_chars='guess'): '''Get the :term:`textual width` of a ucs character :arg ucs: integer representing a single unicode :term:`code point` :kwarg control_chars: specify how to deal with :term:`control characters`. Possible values are: :guess: (default) will take a guess for :term:`control character` widths. Most codes will return zero width. ``backspace``, ``delete``, and ``clear delete`` return -1. ``escape`` currently returns -1 as well but this is not guaranteed as it's not always correct :strict: will raise :exc:`~kitchen.text.exceptions.ControlCharError` if a :term:`control character` is encountered :raises ControlCharError: if the :term:`code point` is a unicode :term:`control character` and :attr:`control_chars` is set to 'strict' :returns: :term:`textual width` of the character. .. note:: It's important to remember this is :term:`textual width` and not the number of characters or bytes. ''' # test for 8-bit control characters if ucs < 32 or (ucs < 0xa0 and ucs >= 0x7f): # Control character detected if control_chars == 'strict': raise ControlCharError(b_('_ucp_width does not understand how to' ' assign a width value to control characters.')) if ucs in (0x08, 0x07F, 0x94): # Backspace, delete, and clear delete remove a single character return -1 if ucs == 0x1b: # Excape is tricky. It removes some number of characters that # come after it but the amount is dependent on what is # interpreting the code. # So this is going to often be wrong but other values will be # wrong as well. return -1 # All other control characters get 0 width return 0 if _interval_bisearch(ucs, _COMBINING): # Combining characters return 0 width as they will be combined with # the width from other characters return 0 # if we arrive here, ucs is not a combining or C0/C1 control character return (1 + (ucs >= 0x1100 and (ucs <= 0x115f or # Hangul Jamo init. consonants ucs == 0x2329 or ucs == 0x232a or (ucs >= 0x2e80 and ucs <= 0xa4cf and ucs != 0x303f) or # CJK ... Yi (ucs >= 0xac00 and ucs <= 0xd7a3) or # Hangul Syllables (ucs >= 0xf900 and ucs <= 0xfaff) or # CJK Compatibility Ideographs (ucs >= 0xfe10 and ucs <= 0xfe19) or # Vertical forms (ucs >= 0xfe30 and ucs <= 0xfe6f) or # CJK Compatibility Forms (ucs >= 0xff00 and ucs <= 0xff60) or # Fullwidth Forms (ucs >= 0xffe0 and ucs <= 0xffe6) or (ucs >= 0x20000 and ucs <= 0x2fffd) or (ucs >= 0x30000 and ucs <= 0x3fffd)))) # Wholly rewritten by me (LGPLv2+) -Toshio Kuratomi def textual_width(msg, control_chars='guess', encoding='utf-8', errors='replace'): '''Get the :term:`textual width` of a string :arg msg: :class:`unicode` string or byte :class:`str` to get the width of :kwarg control_chars: specify how to deal with :term:`control characters`. Possible values are: :guess: (default) will take a guess for :term:`control character` widths. Most codes will return zero width. ``backspace``, ``delete``, and ``clear delete`` return -1. ``escape`` currently returns -1 as well but this is not guaranteed as it's not always correct :strict: will raise :exc:`kitchen.text.exceptions.ControlCharError` if a :term:`control character` is encountered :kwarg encoding: If we are given a byte :class:`str` this is used to decode it into :class:`unicode` string. Any characters that are not decodable in this encoding will get a value dependent on the :attr:`errors` parameter. :kwarg errors: How to treat errors encoding the byte :class:`str` to :class:`unicode` string. Legal values are the same as for :func:`kitchen.text.converters.to_unicode`. The default value of ``replace`` will cause undecodable byte sequences to have a width of one. ``ignore`` will have a width of zero. :raises ControlCharError: if :attr:`msg` contains a :term:`control character` and :attr:`control_chars` is ``strict``. :returns: :term:`Textual width` of the :attr:`msg`. This is the amount of space that the string will consume on a monospace display. It's measured in the number of cell positions or columns it will take up on a monospace display. This is **not** the number of glyphs that are in the string. .. note:: This function can be wrong sometimes because Unicode does not specify a strict width value for all of the :term:`code points`. In particular, we've found that some Tamil characters take up to four character cells but we return a lesser amount. ''' # On python 2.6.4, x86_64, I've benchmarked a few alternate # implementations:: # # timeit.repeat('display.textual_width(data)', # 'from __main__ import display, data', number=100) # I varied data by size and content (1MB of ascii, a few words, 43K utf8, # unicode type # # :this implementation: fastest across the board # # :list comprehension: 6-16% slower # return sum([_ucp_width(ord(c), control_chars=control_chars) # for c in msg]) # # :generator expression: 9-18% slower # return sum((_ucp_width(ord(c), control_chars=control_chars) for c in # msg)) # # :lambda: 10-19% slower # return sum(itertools.imap(lambda x: _ucp_width(ord(x), control_chars), # msg)) # # :partial application: 13-22% slower # func = functools.partial(_ucp_width, control_chars=control_chars) # return sum(itertools.imap(func, itertools.imap(ord, msg))) # # :the original code: 4-38% slower # The 4% was for the short, ascii only string. All the other pieces of # data yielded over 30% slower times. # Non decodable data is just assigned a single cell width msg = to_unicode(msg, encoding=encoding, errors=errors) # Add the width of each char return sum( # calculate width of each char itertools.starmap(_ucp_width, # Setup the arguments to _ucp_width itertools.izip( # int value of each char itertools.imap(ord, msg), # control_chars arg in a form that izip will deal with itertools.repeat(control_chars)))) # Wholly rewritten by me -Toshio Kuratomi def textual_width_chop(msg, chop, encoding='utf-8', errors='replace'): '''Given a string, return it chopped to a given :term:`textual width` :arg msg: :class:`unicode` string or byte :class:`str` to chop :arg chop: Chop :attr:`msg` if it exceeds this :term:`textual width` :kwarg encoding: If we are given a byte :class:`str`, this is used to decode it into a :class:`unicode` string. Any characters that are not decodable in this encoding will be assigned a width of one. :kwarg errors: How to treat errors encoding the byte :class:`str` to :class:`unicode`. Legal values are the same as for :func:`kitchen.text.converters.to_unicode` :rtype: :class:`unicode` string :returns: :class:`unicode` string of the :attr:`msg` chopped at the given :term:`textual width` This is what you want to use instead of ``%.*s``, as it does the "right" thing with regard to :term:`UTF-8` sequences, :term:`control characters`, and characters that take more than one cell position. Eg:: >>> # Wrong: only displays 8 characters because it is operating on bytes >>> print "%.*s" % (10, 'café ñunru!') café ñun >>> # Properly operates on graphemes >>> '%s' % (textual_width_chop('café ñunru!', 10)) café ñunru >>> # takes too many columns because the kanji need two cell positions >>> print '1234567890\\n%.*s' % (10, u'一二三四五六七八九十') 1234567890 一二三四五六七八九十 >>> # Properly chops at 10 columns >>> print '1234567890\\n%s' % (textual_width_chop(u'一二三四五六七八九十', 10)) 1234567890 一二三四五 ''' msg = to_unicode(msg, encoding=encoding, errors=errors) width = textual_width(msg) if width <= chop: return msg maximum = len(msg) if maximum > chop * 2: # A character can take at most 2 cell positions so this is the actual # maximum maximum = chop * 2 minimum = 0 eos = maximum if eos > chop: eos = chop width = textual_width(msg[:eos]) while True: # if current width is high, if width > chop: # calculate new midpoint mid = minimum + (eos - minimum) / 2 if mid == eos: break if (eos - chop) < (eos - mid): while width > chop: width = width - _ucp_width(ord(msg[eos-1])) eos -= 1 return msg[:eos] # subtract distance between eos and mid from width width = width - textual_width(msg[mid:eos]) maximum = eos eos = mid # if current width is low, elif width < chop: # Note: at present, the if (eos - chop) < (eos - mid): # short-circuit above means that we never use this branch. # calculate new midpoint mid = eos + (maximum - eos) / 2 if mid == eos: break if (chop - eos) < (mid - eos): while width < chop: new_width = _ucp_width(ord(msg[eos])) width = width + new_width eos += 1 return msg[:eos] # add distance between eos and new mid to width width = width + textual_width(msg[eos:mid]) minimum = eos eos = mid if eos > maximum: eos = maximum break # if current is just right else: return msg[:eos] return msg[:eos] # I made some adjustments for using unicode but largely unchanged from JA's # port of MK's code -Toshio def textual_width_fill(msg, fill, chop=None, left=True, prefix='', suffix=''): '''Expand a :class:`unicode` string to a specified :term:`textual width` or chop to same :arg msg: :class:`unicode` string to format :arg fill: pad string until the :term:`textual width` of the string is this length :kwarg chop: before doing anything else, chop the string to this length. Default: Don't chop the string at all :kwarg left: If :data:`True` (default) left justify the string and put the padding on the right. If :data:`False`, pad on the left side. :kwarg prefix: Attach this string before the field we're filling :kwarg suffix: Append this string to the end of the field we're filling :rtype: :class:`unicode` string :returns: :attr:`msg` formatted to fill the specified width. If no :attr:`chop` is specified, the string could exceed the fill length when completed. If :attr:`prefix` or :attr:`suffix` are printable characters, the string could be longer than the fill width. .. note:: :attr:`prefix` and :attr:`suffix` should be used for "invisible" characters like highlighting, color changing escape codes, etc. The fill characters are appended outside of any :attr:`prefix` or :attr:`suffix` elements. This allows you to only highlight :attr:`msg` inside of the field you're filling. .. warning:: :attr:`msg`, :attr:`prefix`, and :attr:`suffix` should all be representable as unicode characters. In particular, any escape sequences in :attr:`prefix` and :attr:`suffix` need to be convertible to :class:`unicode`. If you need to use byte sequences here rather than unicode characters, use :func:`~kitchen.text.display.byte_string_textual_width_fill` instead. This function expands a string to fill a field of a particular :term:`textual width`. Use it instead of ``%*.*s``, as it does the "right" thing with regard to :term:`UTF-8` sequences, :term:`control characters`, and characters that take more than one cell position in a display. Example usage:: >>> msg = u'一二三四五六七八九十' >>> # Wrong: This uses 10 characters instead of 10 cells: >>> u":%-*.*s:" % (10, 10, msg[:9]) :一二三四五六七八九 : >>> # This uses 10 cells like we really want: >>> u":%s:" % (textual_width_fill(msg[:9], 10, 10)) :一二三四五: >>> # Wrong: Right aligned in the field, but too many cells >>> u"%20.10s" % (msg) 一二三四五六七八九十 >>> # Correct: Right aligned with proper number of cells >>> u"%s" % (textual_width_fill(msg, 20, 10, left=False)) 一二三四五 >>> # Wrong: Adding some escape characters to highlight the line but too many cells >>> u"%s%20.10s%s" % (prefix, msg, suffix) u'\x1b[7m 一二三四五六七八九十\x1b[0m' >>> # Correct highlight of the line >>> u"%s%s%s" % (prefix, display.textual_width_fill(msg, 20, 10, left=False), suffix) u'\x1b[7m 一二三四五\x1b[0m' >>> # Correct way to not highlight the fill >>> u"%s" % (display.textual_width_fill(msg, 20, 10, left=False, prefix=prefix, suffix=suffix)) u' \x1b[7m一二三四五\x1b[0m' ''' msg = to_unicode(msg) if chop is not None: msg = textual_width_chop(msg, chop) width = textual_width(msg) if width >= fill: if prefix or suffix: msg = u''.join([prefix, msg, suffix]) else: extra = u' ' * (fill - width) if left: msg = u''.join([prefix, msg, suffix, extra]) else: msg = u''.join([extra, prefix, msg, suffix]) return msg def _textual_width_le(width, *args): '''Optimize the common case when deciding which :term:`textual width` is larger :arg width: :term:`textual width` to compare against. :arg \*args: :class:`unicode` strings to check the total :term:`textual width` of :returns: :data:`True` if the total length of :attr:`args` are less than or equal to :attr:`width`. Otherwise :data:`False`. We often want to know "does X fit in Y". It takes a while to use :func:`textual_width` to calculate this. However, we know that the number of canonically composed :class:`unicode` characters is always going to have 1 or 2 for the :term:`textual width` per character. With this we can take the following shortcuts: 1) If the number of canonically composed characters is more than width, the true :term:`textual width` cannot be less than width. 2) If the number of canonically composed characters * 2 is less than the width then the :term:`textual width` must be ok. :term:`textual width` of a canonically composed :class:`unicode` string will always be greater than or equal to the the number of :class:`unicode` characters. So we can first check if the number of composed :class:`unicode` characters is less than the asked for width. If it is we can return :data:`True` immediately. If not, then we must do a full :term:`textual width` lookup. ''' string = ''.join(args) string = unicodedata.normalize('NFC', string) if len(string) > width: return False elif len(string) * 2 <= width: return True elif len(to_bytes(string)) <= width: # Check against bytes. # utf8 has the property of having the same amount or more bytes per # character than textual width. return True else: true_width = textual_width(string) return true_width <= width def wrap(text, width=70, initial_indent=u'', subsequent_indent=u'', encoding='utf-8', errors='replace'): '''Works like we want :func:`textwrap.wrap` to work, :arg text: :class:`unicode` string or byte :class:`str` to wrap :kwarg width: :term:`textual width` at which to wrap. Default: 70 :kwarg initial_indent: string to use to indent the first line. Default: do not indent. :kwarg subsequent_indent: string to use to wrap subsequent lines. Default: do not indent :kwarg encoding: Encoding to use if :attr:`text` is a byte :class:`str` :kwarg errors: error handler to use if :attr:`text` is a byte :class:`str` and contains some undecodable characters. :rtype: :class:`list` of :class:`unicode` strings :returns: list of lines that have been text wrapped and indented. :func:`textwrap.wrap` from the |stdlib|_ has two drawbacks that this attempts to fix: 1. It does not handle :term:`textual width`. It only operates on bytes or characters which are both inadequate (due to multi-byte and double width characters). 2. It malforms lists and blocks. ''' # Tested with: # yum info robodoc gpicview php-pear-Net-Socket wmctrl ustr moreutils # mediawiki-HNP ocspd insight yum mousepad # ...at 120, 80 and 40 chars. # Also, notable among lots of others, searching for "\n ": # exim-clamav, jpackage-utils, tcldom, synaptics, "quake3", # perl-Class-Container, ez-ipupdate, perl-Net-XMPP, "kipi-plugins", # perl-Apache-DBI, netcdf, python-configobj, "translate-toolkit", alpine, # "udunits", "conntrack-tools" # # Note that, we "fail" on: # alsa-plugins-jack, setools*, dblatex, uisp, "perl-Getopt-GUI-Long", # suitesparse, "synce-serial", writer2latex, xenwatch, ltsp-utils def _indent_at_beg(line): '''Return the indent to use for this and (possibly) subsequent lines :arg line: :class:`unicode` line of text to process :rtype: tuple :returns: tuple of count of whitespace before getting to the start of this line followed by a count to the following indent if this block of text is an entry in a list. ''' # Find the first non-whitespace character try: char = line.strip()[0] except IndexError: # All whitespace return 0, 0 else: count = line.find(char) # if we have a bullet character, check for list if char not in u'-*.o\u2022\u2023\u2218': # No bullet; not a list return count, 0 # List: Keep searching until we hit the innermost list nxt = _indent_at_beg(line[count+1:]) nxt = nxt[1] or nxt[0] if nxt: return count, count + 1 + nxt return count, 0 initial_indent = to_unicode(initial_indent, encoding=encoding, errors=errors) subsequent_indent = to_unicode(subsequent_indent, encoding=encoding, errors=errors) subsequent_indent_width = textual_width(subsequent_indent) text = to_unicode(text, encoding=encoding, errors=errors).rstrip(u'\n') lines = text.expandtabs().split(u'\n') ret = [] indent = initial_indent wrap_last = False cur_sab = 0 cur_spc_indent = 0 for line in lines: line = line.rstrip(u' ') (last_sab, last_spc_indent) = (cur_sab, cur_spc_indent) (cur_sab, cur_spc_indent) = _indent_at_beg(line) force_nl = False # We want to stop wrapping under "certain" conditions: if wrap_last and cur_spc_indent: # if line starts a list or force_nl = True if wrap_last and cur_sab == len(line):# is empty line force_nl = True if wrap_last and not last_spc_indent: # if we don't continue a list if cur_sab >= 4 and cur_sab != last_sab: # and is "block indented" force_nl = True if force_nl: ret.append(indent.rstrip(u' ')) indent = subsequent_indent wrap_last = False if cur_sab == len(line): # empty line, remove spaces to make it easier. line = u'' if wrap_last: line = line.lstrip(u' ') cur_spc_indent = last_spc_indent if _textual_width_le(width, indent, line): wrap_last = False ret.append(indent + line) indent = subsequent_indent continue wrap_last = True words = line.split(u' ') line = indent spcs = cur_spc_indent if not spcs and cur_sab >= 4: spcs = cur_sab for word in words: if (not _textual_width_le(width, line, word) and textual_width(line) > subsequent_indent_width): ret.append(line.rstrip(u' ')) line = subsequent_indent + u' ' * spcs line += word line += u' ' indent = line.rstrip(u' ') + u' ' if wrap_last: ret.append(indent.rstrip(u' ')) return ret def fill(text, *args, **kwargs): '''Works like we want :func:`textwrap.fill` to work :arg text: :class:`unicode` string or byte :class:`str` to process :returns: :class:`unicode` string with each line separated by a newline .. seealso:: :func:`kitchen.text.display.wrap` for other parameters that you can give this command. This function is a light wrapper around :func:`kitchen.text.display.wrap`. Where that function returns a :class:`list` of lines, this function returns one string with each line separated by a newline. ''' return u'\n'.join(wrap(text, *args, **kwargs)) # # Byte strings # def byte_string_textual_width_fill(msg, fill, chop=None, left=True, prefix='', suffix='', encoding='utf-8', errors='replace'): '''Expand a byte :class:`str` to a specified :term:`textual width` or chop to same :arg msg: byte :class:`str` encoded in :term:`UTF-8` that we want formatted :arg fill: pad :attr:`msg` until the :term:`textual width` is this long :kwarg chop: before doing anything else, chop the string to this length. Default: Don't chop the string at all :kwarg left: If :data:`True` (default) left justify the string and put the padding on the right. If :data:`False`, pad on the left side. :kwarg prefix: Attach this byte :class:`str` before the field we're filling :kwarg suffix: Append this byte :class:`str` to the end of the field we're filling :rtype: byte :class:`str` :returns: :attr:`msg` formatted to fill the specified :term:`textual width`. If no :attr:`chop` is specified, the string could exceed the fill length when completed. If :attr:`prefix` or :attr:`suffix` are printable characters, the string could be longer than fill width. .. note:: :attr:`prefix` and :attr:`suffix` should be used for "invisible" characters like highlighting, color changing escape codes, etc. The fill characters are appended outside of any :attr:`prefix` or :attr:`suffix` elements. This allows you to only highlight :attr:`msg` inside of the field you're filling. .. seealso:: :func:`~kitchen.text.display.textual_width_fill` For example usage. This function has only two differences. 1. it takes byte :class:`str` for :attr:`prefix` and :attr:`suffix` so you can pass in arbitrary sequences of bytes, not just unicode characters. 2. it returns a byte :class:`str` instead of a :class:`unicode` string. ''' prefix = to_bytes(prefix, encoding=encoding, errors=errors) suffix = to_bytes(suffix, encoding=encoding, errors=errors) if chop is not None: msg = textual_width_chop(msg, chop, encoding=encoding, errors=errors) width = textual_width(msg) msg = to_bytes(msg) if width >= fill: if prefix or suffix: msg = ''.join([prefix, msg, suffix]) else: extra = ' ' * (fill - width) if left: msg = ''.join([prefix, msg, suffix, extra]) else: msg = ''.join([extra, prefix, msg, suffix]) return msg __all__ = ('byte_string_textual_width_fill', 'fill', 'textual_width', 'textual_width_chop', 'textual_width_fill', 'wrap')