Runtime updated

pygments_package/pygments/__init__.py

@@ -12,30 +12,29 @@
     * a wide range of common languages and markup formats is supported
     * special attention is paid to details, increasing quality by a fair amount
     * support for new languages and formats are added easily
-    * a number of output formats, presently HTML, LaTeX, RTF, SVG and ANSI sequences
+    * a number of output formats, presently HTML, LaTeX, RTF, SVG, all image
+      formats that PIL supports, and ANSI sequences
     * it is usable as a command-line tool and as a library
     * ... and it highlights even Brainfuck!
 
     The `Pygments tip`_ is installable with ``easy_install Pygments==dev``.
 
-    .. _Pygments tip: http://dev.pocoo.org/hg/pygments-main/archive/tip.tar.gz#egg=Pygments-dev
+    .. _Pygments tip:
+       http://dev.pocoo.org/hg/pygments-main/archive/tip.tar.gz#egg=Pygments-dev
 
-    :copyright: 2006-2008 by Georg Brandl, Armin Ronacher and others.
-    :license: BSD, see LICENSE for more details.
+    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
 """
 
-__version__ = '1.0'
-__author__ = 'Georg Brandl <g.brandl@gmx.net>'
-__url__ = 'http://pygments.org/'
-__license__ = 'BSD License'
+__version__ = '1.3.1'
 __docformat__ = 'restructuredtext'
 
 __all__ = ['lex', 'format', 'highlight']
 
 
-import sys, os
-from StringIO import StringIO
-from cStringIO import StringIO as CStringIO
+import sys
+
+from pygments.util import StringIO, BytesIO
 
 
 def lex(code, lexer):
@@ -62,8 +61,8 @@ def format(tokens, formatter, outfile=None):
     """
     try:
         if not outfile:
-            # if we want Unicode output, we have to use Python StringIO
-            realoutfile = formatter.encoding and CStringIO() or StringIO()
+            #print formatter, 'using', formatter.encoding
+            realoutfile = formatter.encoding and BytesIO() or StringIO()
             formatter.format(tokens, realoutfile)
             return realoutfile.getvalue()
         else:

pygments_package/pygments/cmdline.py

@@ -5,14 +5,14 @@
 
     Command line interface.
 
-    :copyright: 2006-2008 by Georg Brandl.
-    :license: BSD, see LICENSE for more details.
+    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
 """
 import sys
 import getopt
 from textwrap import dedent
 
-from pygments import __version__, __author__, highlight
+from pygments import __version__, highlight
 from pygments.util import ClassNotFound, OptionError, docstring_headline
 from pygments.lexers import get_all_lexers, get_lexer_by_name, get_lexer_for_filename, \
      find_lexer_class, guess_lexer, TextLexer
@@ -219,7 +219,7 @@ def main(args=sys.argv):
         return 0
 
     if opts.pop('-V', None) is not None:
-        print 'Pygments version %s, (c) 2006-2008 by %s.' % (__version__, __author__)
+        print 'Pygments version %s, (c) 2006-2008 by Georg Brandl.' % __version__
         return 0
 
     # handle ``pygmentize -L``
@@ -359,14 +359,14 @@ def main(args=sys.argv):
 
         infn = args[0]
         try:
-            code = open(infn).read()
+            code = open(infn, 'rb').read()
         except Exception, err:
             print >>sys.stderr, 'Error: cannot read infile:', err
             return 1
 
         if not lexer:
             try:
-                lexer = get_lexer_for_filename(infn, **parsed_opts)
+                lexer = get_lexer_for_filename(infn, code, **parsed_opts)
             except ClassNotFound, err:
                 if '-g' in opts:
                     try:
@@ -402,9 +402,12 @@ def main(args=sys.argv):
             # encoding pass-through
             fmter.encoding = 'latin1'
         else:
-            # use terminal encoding
-            lexer.encoding = getattr(sys.stdin, 'encoding', None) or 'ascii'
-            fmter.encoding = getattr(sys.stdout, 'encoding', None) or 'ascii'
+            if sys.version_info < (3,):
+                # use terminal encoding; Python 3's terminals already do that
+                lexer.encoding = getattr(sys.stdin, 'encoding',
+                                         None) or 'ascii'
+                fmter.encoding = getattr(sys.stdout, 'encoding',
+                                         None) or 'ascii'
 
     # ... and do it!
     try:

pygments_package/pygments/console.py

@@ -5,8 +5,8 @@
 
     Format colored console output.
 
-    :copyright: 2006-2007 by Georg Brandl, Armin Ronacher.
-    :license: BSD, see LICENSE for more details.
+    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
 """
 
 esc = "\x1b["

pygments_package/pygments/filter.py

@@ -5,8 +5,8 @@
 
     Module that implements the default filter.
 
-    :copyright: 2006-2007 by Armin Ronacher.
-    :license: BSD, see LICENSE for more details.
+    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
 """
 
 

pygments_package/pygments/filters/__init__.py

@@ -6,20 +6,17 @@
     Module containing filter lookup functions and default
     filters.
 
-    :copyright: 2006-2007 by Armin Ronacher, Georg Brandl.
-    :license: BSD, see LICENSE for more details.
+    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
 """
-try:
-    set
-except NameError:
-    from sets import Set as set
 
 import re
+
 from pygments.token import String, Comment, Keyword, Name, Error, Whitespace, \
     string_to_tokentype
 from pygments.filter import Filter
-from pygments.util import get_list_opt, get_int_opt, get_bool_opt, get_choice_opt, \
-     ClassNotFound, OptionError
+from pygments.util import get_list_opt, get_int_opt, get_bool_opt, \
+     get_choice_opt, ClassNotFound, OptionError
 from pygments.plugin import find_plugin_filters
 
 
@@ -283,10 +280,78 @@ def replacefunc(wschar):
                 yield ttype, value
 
 
+class GobbleFilter(Filter):
+    """
+    Gobbles source code lines (eats initial characters).
+
+    This filter drops the first ``n`` characters off every line of code.  This
+    may be useful when the source code fed to the lexer is indented by a fixed
+    amount of space that isn't desired in the output.
+
+    Options accepted:
+
+    `n` : int
+       The number of characters to gobble.
+
+    *New in Pygments 1.2.*
+    """
+    def __init__(self, **options):
+        Filter.__init__(self, **options)
+        self.n = get_int_opt(options, 'n', 0)
+
+    def gobble(self, value, left):
+        if left < len(value):
+            return value[left:], 0
+        else:
+            return '', left - len(value)
+
+    def filter(self, lexer, stream):
+        n = self.n
+        left = n # How many characters left to gobble.
+        for ttype, value in stream:
+            # Remove ``left`` tokens from first line, ``n`` from all others.
+            parts = value.split('\n')
+            (parts[0], left) = self.gobble(parts[0], left)
+            for i in range(1, len(parts)):
+                (parts[i], left) = self.gobble(parts[i], n)
+            value = '\n'.join(parts)
+
+            if value != '':
+                yield ttype, value
+
+
+class TokenMergeFilter(Filter):
+    """
+    Merges consecutive tokens with the same token type in the output stream of a
+    lexer.
+
+    *New in Pygments 1.2.*
+    """
+    def __init__(self, **options):
+        Filter.__init__(self, **options)
+
+    def filter(self, lexer, stream):
+        output = []
+        current_type = None
+        current_value = None
+        for ttype, value in stream:
+            if ttype is current_type:
+                current_value += value
+            else:
+                if current_type is not None:
+                    yield current_type, current_value
+                current_type = ttype
+                current_value = value
+        if current_type is not None:
+            yield current_type, current_value
+
+
 FILTERS = {
     'codetagify':     CodeTagFilter,
     'keywordcase':    KeywordCaseFilter,
     'highlight':      NameHighlightFilter,
     'raiseonerror':   RaiseOnErrorTokenFilter,
     'whitespace':     VisibleWhitespaceFilter,
+    'gobble':         GobbleFilter,
+    'tokenmerge':     TokenMergeFilter,
 }

pygments_package/pygments/formatter.py

@@ -5,10 +5,12 @@
 
     Base formatter class.
 
-    :copyright: 2006-2007 by Georg Brandl, Armin Ronacher.
-    :license: BSD, see LICENSE for more details.
+    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
 """
 
+import codecs
+
 from pygments.util import get_bool_opt
 from pygments.styles import get_style_by_name
 
@@ -84,4 +86,7 @@ def format(self, tokensource, outfile):
         Format ``tokensource``, an iterable of ``(tokentype, tokenstring)``
         tuples and write it into ``outfile``.
         """
-        raise NotImplementedError()
+        if self.encoding:
+            # wrap the outfile in a StreamWriter
+            outfile = codecs.lookup(self.encoding)[3](outfile)
+        return self.format_unencoded(tokensource, outfile)

pygments_package/pygments/formatters/__init__.py

@@ -5,15 +5,15 @@
 
     Pygments formatters.
 
-    :copyright: 2006-2007 by Georg Brandl, Armin Ronacher.
-    :license: BSD, see LICENSE for more details.
+    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
 """
 import os.path
 import fnmatch
 
 from pygments.formatters._mapping import FORMATTERS
 from pygments.plugin import find_plugin_formatters
-from pygments.util import docstring_headline, ClassNotFound
+from pygments.util import ClassNotFound
 
 ns = globals()
 for fcls in FORMATTERS:

pygments_package/pygments/formatters/_mapping.py

@@ -9,8 +9,8 @@
 
     Do not alter the FORMATTERS dictionary by hand.
 
-    :copyright: 2006-2007 by Armin Ronacher, Georg Brandl.
-    :license: BSD, see LICENSE for more details.
+    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
 """
 
 from pygments.util import docstring_headline

pygments_package/pygments/formatters/bbcode.py

@@ -5,8 +5,8 @@
 
     BBcode formatter.
 
-    :copyright: 2006-2007 by Lukas Meuser.
-    :license: BSD, see LICENSE for more details.
+    :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
 """
 
 
@@ -76,19 +76,16 @@ def _make_styles(self):
 
             self.styles[ttype] = start, end
 
-    def format(self, tokensource, outfile):
+    def format_unencoded(self, tokensource, outfile):
         if self._code:
             outfile.write('[code]')
         if self._mono:
             outfile.write('[font=monospace]')
 
-        enc = self.encoding
         lastval = ''
         lasttype = None
 
         for ttype, value in tokensource:
-            if enc:
-                value = value.encode(enc)
             while ttype not in self.styles:
                 ttype = ttype.parent
             if ttype == lasttype: