From: JinWang An Date: Tue, 5 Jan 2021 03:14:13 +0000 (+0900) Subject: Imported Upstream version 1.3 X-Git-Tag: upstream/1.3^0 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=e6021a50bbd33b262dbd8b2bd925762cd4958a81;p=platform%2Fupstream%2Fpython3-pygments.git Imported Upstream version 1.3 --- diff --git a/AUTHORS b/AUTHORS index 34cc6be..d102686 100644 --- a/AUTHORS +++ b/AUTHORS @@ -8,9 +8,11 @@ Other contributors, listed alphabetically, are: * Kumar Appaiah -- Debian control lexer * Ali Afshar -- image formatter * Andreas Amann -- AppleScript lexer +* Jeremy Ashkenas -- CoffeeScript lexer * Stefan Matthias Aust -- Smalltalk lexer * Ben Bangert -- Mako lexers * Max Battcher -- Darcs patch lexer +* Paul Baumgart, 280 North, Inc. -- Objective-J lexer * Michael Bayer -- Myghty lexers * Jarrett Billingsley -- MiniD lexer * Adam Blinkinsop -- Haskell, Redcode lexers @@ -18,17 +20,21 @@ Other contributors, listed alphabetically, are: * Pierre Bourdon -- bugfixes * Christopher Creutzig -- MuPAD lexer * Pete Curry -- bugfixes +* Owen Durni -- haXe lexer * Nick Efford -- Python 3 lexer * Artem Egorkine -- terminal256 formatter * Laurent Gautier -- R/S lexer * Krzysiek Goj -- Scala lexer * Matt Good -- Genshi, Cheetah lexers +* Patrick Gotthardt -- PHP namespaces support * Olivier Guibe -- Asymptote lexer * Matthew Harrison -- SVG formatter * Steven Hazel -- Tcl lexer * Aslak Hellesøy -- Gherkin lexer +* David Hess, Fish Software, Inc. -- Objective-J lexer * Varun Hiremath -- Debian control lexer * Dennis Kaarsemaker -- sources.list lexer +* Benjamin Kowarsch -- Modula-2 lexer * Marek Kubica -- Scheme lexer * Jochen Kupperschmidt -- Markdown processor * Gerd Kurzbach -- Modelica lexer @@ -37,7 +43,7 @@ Other contributors, listed alphabetically, are: * Kirk McDonald -- D lexer * Lukas Meuser -- BBCode formatter, Lua lexer * Paulo Moura -- Logtalk lexer -* Ana Nelson -- Ragel, ANTLR lexers +* Ana Nelson -- Ragel, ANTLR, R console lexers * Nam T. Nguyen -- Monokai style * Jesper Noehr -- HTML formatter "anchorlinenos" * Jonas Obrist -- BBCode lexer @@ -50,13 +56,16 @@ Other contributors, listed alphabetically, are: * Mario Ruggier -- Evoque lexers * Stou Sandalski -- NumPy, FORTRAN, tcsh and XSLT lexers * Matteo Sasso -- Common Lisp lexer +* Joe Schafer -- Ada lexer * Ken Schutte -- Matlab lexers * Tassilo Schweyer -- Io, MOOCode lexers * Joerg Sieker -- ABAP lexer * Kirill Simonov -- YAML lexer * Tiberius Teng -- default style overhaul * Jeremy Thurgood -- Erlang, Squid config lexers +* Erick Tryzelaar -- Felix lexer * Whitney Young -- ObjectiveC lexer +* Nathan Weizenbaum -- Haml and Sass lexers * Dietmar Winkler -- Modelica lexer * Nils Winter -- Smalltalk lexer * Davy Wybiral -- Clojure lexer diff --git a/CHANGES b/CHANGES index b81efa0..59d8860 100644 --- a/CHANGES +++ b/CHANGES @@ -3,6 +3,45 @@ Pygments changelog Issue numbers refer to the tracker at http://dev.pocoo.org/projects/pygments/. +Version 1.3 +----------- +(codename Schneeglöckchen, released Mar 01, 2010) + +- Added the ``ensurenl`` lexer option, which can be used to suppress the + automatic addition of a newline to the lexer input. + +- Lexers added: + + * Ada + * Coldfusion + * Modula-2 + * haXe + * R console + * Objective-J + * Haml and Sass + * CoffeeScript + +- Enhanced reStructuredText highlighting. + +- Added support for PHP 5.3 namespaces in the PHP lexer. + +- Added a bash completion script for `pygmentize`, to the external/ + directory (#466). + +- Fixed a bug in `do_insertions()` used for multi-lexer languages. + +- Fixed a Ruby regex highlighting bug (#476). + +- Fixed regex highlighting bugs in Perl lexer (#258). + +- Add small enhancements to the C lexer (#467) and Bash lexer (#469). + +- Small fixes for the Tcl, Debian control file, Nginx config, + Smalltalk, Objective-C, Clojure, Lua lexers. + +- Gherkin lexer: Fixed single apostrophe bug and added new i18n keywords. + + Version 1.2.2 ------------- (bugfix release, released Jan 02, 2010) @@ -33,7 +72,10 @@ Version 1.2 * Go * Gherkin (Cucumber) * CMake - * OOC + * Ooc + * Coldfusion + * haXe + * R console - Added options for rendering LaTeX in source code comments in the LaTeX formatter (#461). diff --git a/MANIFEST.in b/MANIFEST.in index 2a39ef4..8fa0dfe 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -include external/*.py +include external/* include Makefile CHANGES LICENSE AUTHORS TODO ez_setup.py recursive-include tests * recursive-include docs * diff --git a/PKG-INFO b/PKG-INFO index 98c415d..1c8da05 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: Pygments -Version: 1.2.2 +Version: 1.3 Summary: Pygments is a syntax highlighting package written in Python. Home-page: http://pygments.org/ Author: Georg Brandl diff --git a/Pygments.egg-info/PKG-INFO b/Pygments.egg-info/PKG-INFO index 98c415d..1c8da05 100644 --- a/Pygments.egg-info/PKG-INFO +++ b/Pygments.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: Pygments -Version: 1.2.2 +Version: 1.3 Summary: Pygments is a syntax highlighting package written in Python. Home-page: http://pygments.org/ Author: Georg Brandl diff --git a/Pygments.egg-info/SOURCES.txt b/Pygments.egg-info/SOURCES.txt index a885aaa..1e12c02 100644 --- a/Pygments.egg-info/SOURCES.txt +++ b/Pygments.egg-info/SOURCES.txt @@ -5,12 +5,12 @@ MANIFEST.in Makefile TODO ez_setup.py -pygmentize setup.cfg setup.py Pygments.egg-info/PKG-INFO Pygments.egg-info/SOURCES.txt Pygments.egg-info/dependency_links.txt +Pygments.egg-info/entry_points.txt Pygments.egg-info/not-zip-safe Pygments.egg-info/top_level.txt docs/generate.py @@ -57,6 +57,7 @@ docs/src/tokens.txt docs/src/unicode.txt external/markdown-processor.py external/moin-parser.py +external/pygments.bashcomp external/rst-directive-old.py external/rst-directive.py pygments/__init__.py @@ -133,27 +134,16 @@ scripts/vim2pygments.py tests/old_run.py tests/run.py tests/support.py -tests/support.pyc tests/test_basic_api.py -tests/test_basic_api.pyc tests/test_clexer.py -tests/test_clexer.pyc tests/test_cmdline.py -tests/test_cmdline.pyc tests/test_examplefiles.py -tests/test_examplefiles.pyc tests/test_html_formatter.py -tests/test_html_formatter.pyc tests/test_latex_formatter.py -tests/test_latex_formatter.pyc tests/test_regexlexer.py -tests/test_regexlexer.pyc tests/test_token.py -tests/test_token.pyc tests/test_using_api.py -tests/test_using_api.pyc tests/test_util.py -tests/test_util.pyc tests/dtds/HTML4-f.dtd tests/dtds/HTML4-s.dtd tests/dtds/HTML4.dcl @@ -164,19 +154,23 @@ tests/dtds/HTMLspec.ent tests/dtds/HTMLsym.ent tests/examplefiles/ANTLRv3.g tests/examplefiles/AlternatingGroup.mu +tests/examplefiles/CPDictionary.j tests/examplefiles/Constants.mo tests/examplefiles/DancingSudoku.lhs tests/examplefiles/Errors.scala tests/examplefiles/Intro.java tests/examplefiles/Makefile tests/examplefiles/Object.st +tests/examplefiles/OrderedMap.hx tests/examplefiles/RegexMatcher.ns2 tests/examplefiles/SmallCheck.hs +tests/examplefiles/Sorting.mod tests/examplefiles/Sudoku.lhs tests/examplefiles/apache2.conf tests/examplefiles/as3_test.as tests/examplefiles/as3_test2.as tests/examplefiles/as3_test3.as +tests/examplefiles/aspx-cs_example tests/examplefiles/badcase.java tests/examplefiles/batchfile.bat tests/examplefiles/boot-9.scm @@ -186,12 +180,12 @@ tests/examplefiles/classes.dylan tests/examplefiles/condensed_ruby.rb tests/examplefiles/database.pytb tests/examplefiles/de.MoinMoin.po +tests/examplefiles/demo.cfm tests/examplefiles/django_sample.html+django tests/examplefiles/dwarf.cw tests/examplefiles/erl_session tests/examplefiles/escape_semicolon.clj tests/examplefiles/evil_regex.js -tests/examplefiles/example.aspx tests/examplefiles/example.c tests/examplefiles/example.cpp tests/examplefiles/example.lua @@ -202,7 +196,6 @@ tests/examplefiles/example.rhtml tests/examplefiles/example.sh-session tests/examplefiles/example.weechatlog tests/examplefiles/example.xhtml -tests/examplefiles/example.xml tests/examplefiles/example.yaml tests/examplefiles/example2.aspx tests/examplefiles/firefox.mak @@ -244,6 +237,7 @@ tests/examplefiles/pycon_test.pycon tests/examplefiles/pytb_test2.pytb tests/examplefiles/python25-bsd.mak tests/examplefiles/qsort.prolog +tests/examplefiles/r-console-transcript.Rout tests/examplefiles/ragel-cpp_rlscan tests/examplefiles/ragel-cpp_snippet tests/examplefiles/regex.js @@ -258,6 +252,7 @@ tests/examplefiles/sqlite3.sqlite3-console tests/examplefiles/squid.conf tests/examplefiles/string_delimiters.d tests/examplefiles/test.R +tests/examplefiles/test.adb tests/examplefiles/test.asy tests/examplefiles/test.bas tests/examplefiles/test.boo @@ -266,9 +261,11 @@ tests/examplefiles/test.css tests/examplefiles/test.d tests/examplefiles/test.erl tests/examplefiles/test.evoque +tests/examplefiles/test.flx tests/examplefiles/test.html tests/examplefiles/test.java tests/examplefiles/test.jsp +tests/examplefiles/test.mod tests/examplefiles/test.moo tests/examplefiles/test.myt tests/examplefiles/test.pas @@ -281,6 +278,8 @@ tests/examplefiles/test.tcsh tests/examplefiles/test.xsl tests/examplefiles/truncated.pytb tests/examplefiles/type.lisp +tests/examplefiles/underscore.coffee tests/examplefiles/unicode.applescript tests/examplefiles/while.pov +tests/examplefiles/xml_example tests/examplefiles/zmlrpc.f90 \ No newline at end of file diff --git a/Pygments.egg-info/entry_points.txt b/Pygments.egg-info/entry_points.txt new file mode 100644 index 0000000..756d801 --- /dev/null +++ b/Pygments.egg-info/entry_points.txt @@ -0,0 +1,3 @@ +[console_scripts] +pygmentize = pygments.cmdline:main + diff --git a/docs/build/api.html b/docs/build/api.html index 4cf4217..3ca8e31 100644 --- a/docs/build/api.html +++ b/docs/build/api.html @@ -453,6 +453,6 @@ or a tuple, it is returned as a list. - \ No newline at end of file diff --git a/docs/build/authors.html b/docs/build/authors.html index ae566e1..a2ce275 100644 --- a/docs/build/authors.html +++ b/docs/build/authors.html @@ -217,9 +217,11 @@ div.toc h2 {
  • Kumar Appaiah -- Debian control lexer
  • Ali Afshar -- image formatter
  • Andreas Amann -- AppleScript lexer
    • +
  • Jeremy Ashkenas -- CoffeeScript lexer
  • Stefan Matthias Aust -- Smalltalk lexer
  • Ben Bangert -- Mako lexers
  • Max Battcher -- Darcs patch lexer
    • +
  • Paul Baumgart, 280 North, Inc. -- Objective-J lexer
  • Michael Bayer -- Myghty lexers
  • Jarrett Billingsley -- MiniD lexer
  • Adam Blinkinsop -- Haskell, Redcode lexers
    • @@ -227,17 +229,21 @@ div.toc h2 {
  • Pierre Bourdon -- bugfixes
  • Christopher Creutzig -- MuPAD lexer
  • Pete Curry -- bugfixes
    • +
  • Owen Durni -- haXe lexer
  • Nick Efford -- Python 3 lexer
  • Artem Egorkine -- terminal256 formatter
  • Laurent Gautier -- R/S lexer
  • Krzysiek Goj -- Scala lexer
  • Matt Good -- Genshi, Cheetah lexers
    • +
  • Patrick Gotthardt -- PHP namespaces support
  • Olivier Guibe -- Asymptote lexer
  • Matthew Harrison -- SVG formatter
  • Steven Hazel -- Tcl lexer
  • Aslak Hellesøy -- Gherkin lexer
    • +
  • David Hess, Fish Software, Inc. -- Objective-J lexer
  • Varun Hiremath -- Debian control lexer
  • Dennis Kaarsemaker -- sources.list lexer
    • +
  • Benjamin Kowarsch -- Modula-2 lexer
  • Marek Kubica -- Scheme lexer
  • Jochen Kupperschmidt -- Markdown processor
  • Gerd Kurzbach -- Modelica lexer
    • @@ -246,7 +252,7 @@ div.toc h2 {
  • Kirk McDonald -- D lexer
  • Lukas Meuser -- BBCode formatter, Lua lexer
  • Paulo Moura -- Logtalk lexer
    • -
  • Ana Nelson -- Ragel, ANTLR lexers
    • +
  • Ana Nelson -- Ragel, ANTLR, R console lexers
  • Nam T. Nguyen -- Monokai style
  • Jesper Noehr -- HTML formatter "anchorlinenos"
  • Jonas Obrist -- BBCode lexer
    • @@ -259,13 +265,16 @@ div.toc h2 {
  • Mario Ruggier -- Evoque lexers
  • Stou Sandalski -- NumPy, FORTRAN, tcsh and XSLT lexers
  • Matteo Sasso -- Common Lisp lexer
    • +
  • Joe Schafer -- Ada lexer
  • Ken Schutte -- Matlab lexers
  • Tassilo Schweyer -- Io, MOOCode lexers
  • Joerg Sieker -- ABAP lexer
  • Kirill Simonov -- YAML lexer
  • Tiberius Teng -- default style overhaul
  • Jeremy Thurgood -- Erlang, Squid config lexers
    • +
  • Erick Tryzelaar -- Felix lexer
  • Whitney Young -- ObjectiveC lexer
    • +
  • Nathan Weizenbaum -- Haml and Sass lexers
  • Dietmar Winkler -- Modelica lexer
  • Nils Winter -- Smalltalk lexer
  • Davy Wybiral -- Clojure lexer
    • @@ -274,6 +283,6 @@ div.toc h2 { - \ No newline at end of file diff --git a/docs/build/changelog.html b/docs/build/changelog.html index 6e38a8b..0177758 100644 --- a/docs/build/changelog.html +++ b/docs/build/changelog.html @@ -213,6 +213,8 @@ div.toc h2 {

    Contents

  • Added options for rendering LaTeX in source code comments in the @@ -702,6 +737,6 @@ continuations.
    • - \ No newline at end of file diff --git a/docs/build/cmdline.html b/docs/build/cmdline.html index e382e6d..6db3dd6 100644 --- a/docs/build/cmdline.html +++ b/docs/build/cmdline.html @@ -348,6 +348,6 @@ formatter is the terminal encoding (sys.stdout.encoding). - \ No newline at end of file diff --git a/docs/build/filterdevelopment.html b/docs/build/filterdevelopment.html index 97ba876..f6de0c3 100644 --- a/docs/build/filterdevelopment.html +++ b/docs/build/filterdevelopment.html @@ -277,6 +277,6 @@ decorated function for filtering.

    - \ No newline at end of file diff --git a/docs/build/filters.html b/docs/build/filters.html index a05e4b0..e370f46 100644 --- a/docs/build/filters.html +++ b/docs/build/filters.html @@ -407,6 +407,6 @@ code to your styleguide.

    - \ No newline at end of file diff --git a/docs/build/formatterdevelopment.html b/docs/build/formatterdevelopment.html index cc4e9cb..8a27246 100644 --- a/docs/build/formatterdevelopment.html +++ b/docs/build/formatterdevelopment.html @@ -369,6 +369,6 @@ is up to the formatter) and has to return a string or here - \ No newline at end of file diff --git a/docs/build/installation.html b/docs/build/installation.html index 815263c..e7ef9c4 100644 --- a/docs/build/installation.html +++ b/docs/build/installation.html @@ -276,6 +276,6 @@ is run, the sources are updated from Subversion. --> - \ No newline at end of file diff --git a/docs/build/integrate.html b/docs/build/integrate.html index 903d761..06773ec 100644 --- a/docs/build/integrate.html +++ b/docs/build/integrate.html @@ -221,6 +221,8 @@ div.toc h2 {
  • TextMate
    • +
  • Bash completion
    • + @@ -246,10 +248,15 @@ You can copy and adapt it to your liking.

    TextMate

    Antonio Cangiano has created a Pygments bundle for TextMate that allows to colorize code via a simple menu option. It can be found here.

    + +
    +

    Bash completion

    +

    The source distribution contains a file external/pygments.bashcomp that +sets up completion for the pygmentize command in bash.

    - \ No newline at end of file diff --git a/docs/build/lexerdevelopment.html b/docs/build/lexerdevelopment.html index 7602ec3..c7d5207 100644 --- a/docs/build/lexerdevelopment.html +++ b/docs/build/lexerdevelopment.html @@ -686,6 +686,6 @@ the get_tokens_unprocessed() method. The follo - \ No newline at end of file diff --git a/docs/build/lexers.html b/docs/build/lexers.html index 70627a1..84c55e8 100644 --- a/docs/build/lexers.html +++ b/docs/build/lexers.html @@ -442,7 +442,7 @@ language) source.

    Short names:python, py -Filename patterns:*.py, *.pyw, *.sc, SConstruct, SConscript +Filename patterns:*.py, *.pyw, *.sc, SConstruct, SConscript, *.tac Mimetypes:text/x-python, application/x-python @@ -639,6 +639,23 @@ language) source.

    Lexers for compiled languages

    +

    AdaLexer

    +
    +

    For Ada source code.

    +

    New in Pygments 1.3.

    + +++ + + + + + + + +
    Short names:ada, ada95ada2005
    Filename patterns:*.adb, *.ads, *.ada
    Mimetypes:text/x-ada
    +

    CLexer

    For C source code with preprocessor directives.

    @@ -752,6 +769,23 @@ Default is to consider all of them builtin.
    +

    FelixLexer

    +
    +

    For Felix source code.

    +

    New in Pygments 1.2.

    + +++ + + + + + + + +
    Short names:felix, flx
    Filename patterns:*.flx, *.flxh
    Mimetypes:text/x-felix
    +

    FortranLexer

    Lexer for FORTRAN 90 code.

    @@ -818,6 +852,34 @@ Default is to consider all of them builtin.
    +

    Modula2Lexer

    +
    +

    For Modula-2 source code.

    +

    Additional options that determine which keywords are highlighted:

    +
    +
    pim
    +
    Select PIM Modula-2 dialect (default: True).
    +
    iso
    +
    Select ISO Modula-2 dialect (default: False).
    +
    objm2
    +
    Select Objective Modula-2 dialect (default: False).
    +
    gm2ext
    +
    Also highlight GNU extensions (default: False).
    +
    +

    New in Pygments 1.3.

    + +++ + + + + + + + +
    Short names:modula2, m2
    Filename patterns:*.def, *.mod
    Mimetypes:text/x-modula2
    +

    ObjectiveCLexer

    For Objective-C source code with preprocessor directives.

    @@ -1214,6 +1276,22 @@ Contributed by Christopher Creutzig < + + + +Short names:rconsole, rout + +Filename patterns:*.Rout + +Mimetypes:None + + + +

    SLexer

    For S, S-plus, and R source code.

    @@ -1277,7 +1355,7 @@ Contributed by Andreas Amann <     -Short names:asy +Short names:asy, asymptote Filename patterns:*.asy @@ -1288,15 +1366,15 @@ Contributed by Andreas Amann < -Short names:bash, sh +Short names:bash, sh, ksh -Filename patterns:*.sh, *.ebuild, *.eclass +Filename patterns:*.sh, *.ksh, *.bash, *.ebuild, *.eclass Mimetypes:application/x-sh, application/x-shellscript @@ -2025,6 +2103,38 @@ with the XmlLexer.

    +

    ColdfusionHtmlLexer

    +
    +

    Coldfusion markup in html

    + +++ + + + + + + + +
    Short names:cfm
    Filename patterns:*.cfm, *.cfml, *.cfc
    Mimetypes:application/x-coldfusion
    +
    +

    ColdfusionLexer

    +
    +

    Coldfusion statements

    + +++ + + + + + + + +
    Short names:cfs
    Filename patterns:None
    Mimetypes:None
    +

    CssDjangoLexer

    Subclass of the DjangoLexer that highlights unlexed data with the @@ -3093,6 +3203,23 @@ language.

    +

    CoffeeScriptLexer

    +
    +

    For CoffeeScript source code.

    +

    New in Pygments 1.3.

    + +++ + + + + + + + +
    Short names:coffee-script, coffeescript
    Filename patterns:*.coffee
    Mimetypes:text/coffeescript
    +

    CssLexer

    For CSS (Cascading Style Sheets).

    @@ -3109,6 +3236,39 @@ language.

    +

    HamlLexer

    +
    +

    For Haml markup.

    +

    New in Pygments 1.3.

    + +++ + + + + + + + +
    Short names:haml, HAML
    Filename patterns:*.haml
    Mimetypes:text/x-haml
    +
    +

    HaxeLexer

    +
    +

    For haXe source code (http://haxe.org/).

    + +++ + + + + + + + +
    Short names:hx, haXe
    Filename patterns:*.hx
    Mimetypes:text/haxe
    +

    HtmlLexer

    For HTML 4 and XHTML 1 markup. Nested JavaScript and CSS is highlighted @@ -3159,6 +3319,23 @@ Nested AS3 in <script> tags is highlighted by the appropriate lexer.

    +

    ObjectiveJLexer

    +
    +

    For Objective-J source code with preprocessor directives.

    +

    New in Pygments 1.3.

    + +++ + + + + + + + +
    Short names:objective-j, objectivej, obj-j, objj
    Filename patterns:*.j
    Mimetypes:text/x-objective-j
    +

    PhpLexer

    For PHP source code. @@ -3200,6 +3377,23 @@ the php documentation.

    +

    SassLexer

    +
    +

    For Sass stylesheets.

    +

    New in Pygments 1.3.

    + +++ + + + + + + + +
    Short names:sass, SASS
    Filename patterns:*.sass
    Mimetypes:text/x-sass
    +

    XmlLexer

    Generic lexer for XML (eXtensible Markup Language).

    @@ -3255,6 +3449,6 @@ in the form (name, aliases, filetypes, mimetypes) - \ No newline at end of file diff --git a/docs/build/moinmoin.html b/docs/build/moinmoin.html index 007e125..6f3e3c2 100644 --- a/docs/build/moinmoin.html +++ b/docs/build/moinmoin.html @@ -240,6 +240,6 @@ can set the INLINESTYLES option to True.

    - \ No newline at end of file diff --git a/docs/build/plugins.html b/docs/build/plugins.html index ed7484b..95641ad 100644 --- a/docs/build/plugins.html +++ b/docs/build/plugins.html @@ -289,6 +289,6 @@ distribution.

    - \ No newline at end of file diff --git a/docs/build/quickstart.html b/docs/build/quickstart.html index 64cc0f0..92e3a75 100644 --- a/docs/build/quickstart.html +++ b/docs/build/quickstart.html @@ -385,6 +385,6 @@ $ pygmentize -S default -f html > style.css - \ No newline at end of file diff --git a/docs/build/rstdirective.html b/docs/build/rstdirective.html index d7edf6b..bbec08f 100644 --- a/docs/build/rstdirective.html +++ b/docs/build/rstdirective.html @@ -224,6 +224,6 @@ if the `handlecodeblocks` option is true. --> - \ No newline at end of file diff --git a/docs/build/styles.html b/docs/build/styles.html index 059f90c..ee224ef 100644 --- a/docs/build/styles.html +++ b/docs/build/styles.html @@ -336,6 +336,6 @@ a way to iterate over all styles:

    - \ No newline at end of file diff --git a/docs/build/tokens.html b/docs/build/tokens.html index a8c0483..4290e09 100644 --- a/docs/build/tokens.html +++ b/docs/build/tokens.html @@ -536,6 +536,6 @@ highlight a programming language but a patch file.

    - \ No newline at end of file diff --git a/docs/build/unicode.html b/docs/build/unicode.html index 0cb8f52..7a09e8b 100644 --- a/docs/build/unicode.html +++ b/docs/build/unicode.html @@ -244,6 +244,6 @@ input and output encodings.

    - \ No newline at end of file diff --git a/docs/src/integrate.txt b/docs/src/integrate.txt index fb3fa5a..51a3dac 100644 --- a/docs/src/integrate.txt +++ b/docs/src/integrate.txt @@ -35,3 +35,9 @@ Antonio Cangiano has created a Pygments bundle for TextMate that allows to colorize code via a simple menu option. It can be found here_. .. _here: http://antoniocangiano.com/2008/10/28/pygments-textmate-bundle/ + +Bash completion +--------------- + +The source distribution contains a file ``external/pygments.bashcomp`` that +sets up completion for the ``pygmentize`` command in bash. diff --git a/external/pygments.bashcomp b/external/pygments.bashcomp new file mode 100644 index 0000000..1299fdb --- /dev/null +++ b/external/pygments.bashcomp @@ -0,0 +1,38 @@ +#!bash +# +# Bash completion support for Pygments (the 'pygmentize' command). +# + +_pygmentize() +{ + local cur prev + + COMPREPLY=() + cur=`_get_cword` + prev=${COMP_WORDS[COMP_CWORD-1]} + + case "$prev" in + -f) + FORMATTERS=`pygmentize -L formatters | grep '* ' | cut -c3- | sed -e 's/,//g' -e 's/:$//'` + COMPREPLY=( $( compgen -W '$FORMATTERS' -- "$cur" ) ) + return 0 + ;; + -l) + LEXERS=`pygmentize -L lexers | grep '* ' | cut -c3- | sed -e 's/,//g' -e 's/:$//'` + COMPREPLY=( $( compgen -W '$LEXERS' -- "$cur" ) ) + return 0 + ;; + -S) + STYLES=`pygmentize -L styles | grep '* ' | cut -c3- | sed s/:$//` + COMPREPLY=( $( compgen -W '$STYLES' -- "$cur" ) ) + return 0 + ;; + esac + + if [[ "$cur" == -* ]]; then + COMPREPLY=( $( compgen -W '-f -l -S -L -g -O -P -F \ + -N -H -h -V -o' -- "$cur" ) ) + return 0 + fi +} +complete -F _pygmentize -o default pygmentize diff --git a/pygmentize b/pygmentize deleted file mode 100755 index e237919..0000000 --- a/pygmentize +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python - -import sys, pygments.cmdline -try: - sys.exit(pygments.cmdline.main(sys.argv)) -except KeyboardInterrupt: - sys.exit(1) diff --git a/pygments/__init__.py b/pygments/__init__.py index ff17af4..823862e 100644 --- a/pygments/__init__.py +++ b/pygments/__init__.py @@ -26,13 +26,13 @@ :license: BSD, see LICENSE for details. """ -__version__ = '1.2.2' +__version__ = '1.3' __docformat__ = 'restructuredtext' __all__ = ['lex', 'format', 'highlight'] -import sys, os +import sys from pygments.util import StringIO, BytesIO diff --git a/pygments/filters/__init__.py b/pygments/filters/__init__.py index 382933b..504c3e1 100644 --- a/pygments/filters/__init__.py +++ b/pygments/filters/__init__.py @@ -9,17 +9,14 @@ :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -try: - set -except NameError: - from sets import Set as set import re + from pygments.token import String, Comment, Keyword, Name, Error, Whitespace, \ string_to_tokentype from pygments.filter import Filter -from pygments.util import get_list_opt, get_int_opt, get_bool_opt, get_choice_opt, \ - ClassNotFound, OptionError +from pygments.util import get_list_opt, get_int_opt, get_bool_opt, \ + get_choice_opt, ClassNotFound, OptionError from pygments.plugin import find_plugin_filters diff --git a/pygments/formatters/__init__.py b/pygments/formatters/__init__.py index e5802b9..0e02a52 100644 --- a/pygments/formatters/__init__.py +++ b/pygments/formatters/__init__.py @@ -13,7 +13,7 @@ import fnmatch from pygments.formatters._mapping import FORMATTERS from pygments.plugin import find_plugin_formatters -from pygments.util import docstring_headline, ClassNotFound +from pygments.util import ClassNotFound ns = globals() for fcls in FORMATTERS: diff --git a/pygments/formatters/html.py b/pygments/formatters/html.py index 96cde3f..5c0972e 100644 --- a/pygments/formatters/html.py +++ b/pygments/formatters/html.py @@ -8,13 +8,10 @@ :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -import sys, os -import StringIO -try: - set -except NameError: - from sets import Set as set +import os +import sys +import StringIO from pygments.formatter import Formatter from pygments.token import Token, Text, STANDARD_TYPES diff --git a/pygments/lexer.py b/pygments/lexer.py index f28627b..fbcc39a 100644 --- a/pygments/lexer.py +++ b/pygments/lexer.py @@ -10,11 +10,6 @@ """ import re -try: - set -except NameError: - from sets import Set as set - from pygments.filter import apply_filters, Filter from pygments.filters import get_filter_by_name from pygments.token import Error, Text, Other, _TokenType @@ -23,7 +18,7 @@ from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \ __all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer', - 'LexerContext', 'include', 'flags', 'bygroups', 'using', 'this'] + 'LexerContext', 'include', 'bygroups', 'using', 'this'] _default_analyse = staticmethod(lambda x: 0.0) @@ -51,6 +46,10 @@ class Lexer(object): ``stripall`` Strip all leading and trailing whitespace from the input (default: False). + ``ensurenl`` + Make sure that the input ends with a newline (default: True). This + is required for some lexers that consume input linewise. + *New in Pygments 1.3.* ``tabsize`` If given and greater than 0, expand tabs in the input (default: 0). ``encoding`` @@ -82,6 +81,7 @@ class Lexer(object): self.options = options self.stripnl = get_bool_opt(options, 'stripnl', True) self.stripall = get_bool_opt(options, 'stripall', False) + self.ensurenl = get_bool_opt(options, 'ensurenl', True) self.tabsize = get_int_opt(options, 'tabsize', 0) self.encoding = options.get('encoding', 'latin1') # self.encoding = options.get('inencoding', None) or self.encoding @@ -155,7 +155,7 @@ class Lexer(object): text = text.strip('\n') if self.tabsize > 0: text = text.expandtabs(self.tabsize) - if not text.endswith('\n'): + if self.ensurenl and not text.endswith('\n'): text += '\n' def streamer(): @@ -646,9 +646,15 @@ def do_insertions(insertions, tokens): realpos += len(v) - oldi # leftover tokens - if insleft: + while insleft: # no normal tokens, set realpos to zero realpos = realpos or 0 for p, t, v in itokens: yield realpos, t, v realpos += len(v) + try: + index, itokens = insertions.next() + except StopIteration: + insleft = False + break # not strictly necessary + diff --git a/pygments/lexers/__init__.py b/pygments/lexers/__init__.py index c1890a9..cce7e9b 100644 --- a/pygments/lexers/__init__.py +++ b/pygments/lexers/__init__.py @@ -8,16 +8,12 @@ :copyright: Copyright 2006-2010 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ + import sys -import fnmatch import types +import fnmatch from os.path import basename -try: - set -except NameError: - from sets import Set as set - from pygments.lexers._mapping import LEXERS from pygments.plugin import find_plugin_lexers from pygments.util import ClassNotFound, bytes @@ -223,7 +219,6 @@ class _automodule(types.ModuleType): raise AttributeError(name) -import sys oldmod = sys.modules['pygments.lexers'] newmod = _automodule('pygments.lexers') newmod.__dict__.update(oldmod.__dict__) diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index 7e06831..cdaf56a 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -17,6 +17,7 @@ LEXERS = { 'ABAPLexer': ('pygments.lexers.other', 'ABAP', ('abap',), ('*.abap',), ('text/x-abap',)), 'ActionScript3Lexer': ('pygments.lexers.web', 'ActionScript 3', ('as3', 'actionscript3'), ('*.as',), ('application/x-actionscript', 'text/x-actionscript', 'text/actionscript')), 'ActionScriptLexer': ('pygments.lexers.web', 'ActionScript', ('as', 'actionscript'), ('*.as',), ('application/x-actionscript', 'text/x-actionscript', 'text/actionscript')), + 'AdaLexer': ('pygments.lexers.compiled', 'Ada', ('ada', 'ada95ada2005'), ('*.adb', '*.ads', '*.ada'), ('text/x-ada',)), 'AntlrActionScriptLexer': ('pygments.lexers.parsers', 'ANTLR With ActionScript Target', ('antlr-as', 'antlr-actionscript'), ('*.G', '*.g'), ()), 'AntlrCSharpLexer': ('pygments.lexers.parsers', 'ANTLR With C# Target', ('antlr-csharp', 'antlr-c#'), ('*.G', '*.g'), ()), 'AntlrCppLexer': ('pygments.lexers.parsers', 'ANTLR With CPP Target', ('antlr-cpp',), ('*.G', '*.g'), ()), @@ -28,10 +29,10 @@ LEXERS = { 'AntlrRubyLexer': ('pygments.lexers.parsers', 'ANTLR With Ruby Target', ('antlr-ruby', 'antlr-rb'), ('*.G', '*.g'), ()), 'ApacheConfLexer': ('pygments.lexers.text', 'ApacheConf', ('apacheconf', 'aconf', 'apache'), ('.htaccess', 'apache.conf', 'apache2.conf'), ('text/x-apacheconf',)), 'AppleScriptLexer': ('pygments.lexers.other', 'AppleScript', ('applescript',), ('*.applescript',), ()), - 'AsymptoteLexer': ('pygments.lexers.other', 'Asymptote', ('asy',), ('*.asy',), ('text/x-asymptote',)), + 'AsymptoteLexer': ('pygments.lexers.other', 'Asymptote', ('asy', 'asymptote'), ('*.asy',), ('text/x-asymptote',)), 'BBCodeLexer': ('pygments.lexers.text', 'BBCode', ('bbcode',), (), ('text/x-bbcode',)), 'BaseMakefileLexer': ('pygments.lexers.text', 'Makefile', ('basemake',), (), ()), - 'BashLexer': ('pygments.lexers.other', 'Bash', ('bash', 'sh'), ('*.sh', '*.ebuild', '*.eclass'), ('application/x-sh', 'application/x-shellscript')), + 'BashLexer': ('pygments.lexers.other', 'Bash', ('bash', 'sh', 'ksh'), ('*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass'), ('application/x-sh', 'application/x-shellscript')), 'BashSessionLexer': ('pygments.lexers.other', 'Bash Session', ('console',), ('*.sh-session',), ('application/x-shell-session',)), 'BatchLexer': ('pygments.lexers.other', 'Batchfile', ('bat',), ('*.bat', '*.cmd'), ('application/x-dos-batch',)), 'BefungeLexer': ('pygments.lexers.other', 'Befunge', ('befunge',), ('*.befunge',), ('application/x-befunge',)), @@ -47,6 +48,9 @@ LEXERS = { 'CheetahLexer': ('pygments.lexers.templates', 'Cheetah', ('cheetah', 'spitfire'), ('*.tmpl', '*.spt'), ('application/x-cheetah', 'application/x-spitfire')), 'CheetahXmlLexer': ('pygments.lexers.templates', 'XML+Cheetah', ('xml+cheetah', 'xml+spitfire'), (), ('application/xml+cheetah', 'application/xml+spitfire')), 'ClojureLexer': ('pygments.lexers.agile', 'Clojure', ('clojure', 'clj'), ('*.clj',), ('text/x-clojure', 'application/x-clojure')), + 'CoffeeScriptLexer': ('pygments.lexers.web', 'CoffeeScript', ('coffee-script', 'coffeescript'), ('*.coffee',), ('text/coffeescript',)), + 'ColdfusionHtmlLexer': ('pygments.lexers.templates', 'Coldufsion HTML', ('cfm',), ('*.cfm', '*.cfml', '*.cfc'), ('application/x-coldfusion',)), + 'ColdfusionLexer': ('pygments.lexers.templates', 'cfstatement', ('cfs',), (), ()), 'CommonLispLexer': ('pygments.lexers.functional', 'Common Lisp', ('common-lisp', 'cl'), ('*.cl', '*.lisp', '*.el'), ('text/x-common-lisp',)), 'CppLexer': ('pygments.lexers.compiled', 'C++', ('cpp', 'c++'), ('*.cpp', '*.hpp', '*.c++', '*.h++', '*.cc', '*.hh', '*.cxx', '*.hxx'), ('text/x-c++hdr', 'text/x-c++src')), 'CppObjdumpLexer': ('pygments.lexers.asm', 'cpp-objdump', ('cpp-objdump', 'c++-objdumb', 'cxx-objdump'), ('*.cpp-objdump', '*.c++-objdump', '*.cxx-objdump'), ('text/x-cpp-objdump',)), @@ -71,6 +75,7 @@ LEXERS = { 'EvoqueHtmlLexer': ('pygments.lexers.templates', 'HTML+Evoque', ('html+evoque',), ('*.html',), ('text/html+evoque',)), 'EvoqueLexer': ('pygments.lexers.templates', 'Evoque', ('evoque',), ('*.evoque',), ('application/x-evoque',)), 'EvoqueXmlLexer': ('pygments.lexers.templates', 'XML+Evoque', ('xml+evoque',), ('*.xml',), ('application/xml+evoque',)), + 'FelixLexer': ('pygments.lexers.compiled', 'Felix', ('felix', 'flx'), ('*.flx', '*.flxh'), ('text/x-felix',)), 'FortranLexer': ('pygments.lexers.compiled', 'Fortran', ('fortran',), ('*.f', '*.f90'), ('text/x-fortran',)), 'GLShaderLexer': ('pygments.lexers.compiled', 'GLSL', ('glsl',), ('*.vert', '*.frag', '*.geo'), ('text/x-glslsrc',)), 'GasLexer': ('pygments.lexers.asm', 'GAS', ('gas',), ('*.s', '*.S'), ('text/x-gas',)), @@ -81,7 +86,9 @@ LEXERS = { 'GnuplotLexer': ('pygments.lexers.other', 'Gnuplot', ('gnuplot',), ('*.plot', '*.plt'), ('text/x-gnuplot',)), 'GoLexer': ('pygments.lexers.compiled', 'Go', ('go',), ('*.go',), ('text/x-gosrc',)), 'GroffLexer': ('pygments.lexers.text', 'Groff', ('groff', 'nroff', 'man'), ('*.[1234567]', '*.man'), ('application/x-troff', 'text/troff')), + 'HamlLexer': ('pygments.lexers.web', 'Haml', ('haml', 'HAML'), ('*.haml',), ('text/x-haml',)), 'HaskellLexer': ('pygments.lexers.functional', 'Haskell', ('haskell', 'hs'), ('*.hs',), ('text/x-haskell',)), + 'HaxeLexer': ('pygments.lexers.web', 'haXe', ('hx', 'haXe'), ('*.hx',), ('text/haxe',)), 'HtmlDjangoLexer': ('pygments.lexers.templates', 'HTML+Django/Jinja', ('html+django', 'html+jinja'), (), ('text/html+django', 'text/html+jinja')), 'HtmlGenshiLexer': ('pygments.lexers.templates', 'HTML+Genshi', ('html+genshi', 'html+kid'), (), ('text/html+genshi',)), 'HtmlLexer': ('pygments.lexers.web', 'HTML', ('html',), ('*.html', '*.htm', '*.xhtml', '*.xslt'), ('text/html', 'application/xhtml+xml')), @@ -114,6 +121,7 @@ LEXERS = { 'MatlabSessionLexer': ('pygments.lexers.math', 'Matlab session', ('matlabsession',), (), ()), 'MiniDLexer': ('pygments.lexers.agile', 'MiniD', ('minid',), ('*.md',), ('text/x-minidsrc',)), 'ModelicaLexer': ('pygments.lexers.other', 'Modelica', ('modelica',), ('*.mo',), ('text/x-modelica',)), + 'Modula2Lexer': ('pygments.lexers.compiled', 'Modula-2', ('modula2', 'm2'), ('*.def', '*.mod'), ('text/x-modula2',)), 'MoinWikiLexer': ('pygments.lexers.text', 'MoinMoin/Trac Wiki markup', ('trac-wiki', 'moin'), (), ('text/x-trac-wiki',)), 'MuPADLexer': ('pygments.lexers.math', 'MuPAD', ('mupad',), ('*.mu',), ()), 'MxmlLexer': ('pygments.lexers.web', 'MXML', ('mxml',), ('*.mxml',), ()), @@ -129,6 +137,7 @@ LEXERS = { 'NumPyLexer': ('pygments.lexers.math', 'NumPy', ('numpy',), (), ()), 'ObjdumpLexer': ('pygments.lexers.asm', 'objdump', ('objdump',), ('*.objdump',), ('text/x-objdump',)), 'ObjectiveCLexer': ('pygments.lexers.compiled', 'Objective-C', ('objective-c', 'objectivec', 'obj-c', 'objc'), ('*.m',), ('text/x-objective-c',)), + 'ObjectiveJLexer': ('pygments.lexers.web', 'Objective-J', ('objective-j', 'objectivej', 'obj-j', 'objj'), ('*.j',), ('text/x-objective-j',)), 'OcamlLexer': ('pygments.lexers.compiled', 'OCaml', ('ocaml',), ('*.ml', '*.mli', '*.mll', '*.mly'), ('text/x-ocaml',)), 'OcamlLexer': ('pygments.lexers.functional', 'OCaml', ('ocaml',), ('*.ml', '*.mli', '*.mll', '*.mly'), ('text/x-ocaml',)), 'OocLexer': ('pygments.lexers.compiled', 'Ooc', ('ooc',), ('*.ooc',), ('text/x-ooc',)), @@ -139,8 +148,9 @@ LEXERS = { 'Python3Lexer': ('pygments.lexers.agile', 'Python 3', ('python3', 'py3'), (), ('text/x-python3', 'application/x-python3')), 'Python3TracebackLexer': ('pygments.lexers.agile', 'Python 3.0 Traceback', ('py3tb',), ('*.py3tb',), ('text/x-python3-traceback',)), 'PythonConsoleLexer': ('pygments.lexers.agile', 'Python console session', ('pycon',), (), ('text/x-python-doctest',)), - 'PythonLexer': ('pygments.lexers.agile', 'Python', ('python', 'py'), ('*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript'), ('text/x-python', 'application/x-python')), + 'PythonLexer': ('pygments.lexers.agile', 'Python', ('python', 'py'), ('*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac'), ('text/x-python', 'application/x-python')), 'PythonTracebackLexer': ('pygments.lexers.agile', 'Python Traceback', ('pytb',), ('*.pytb',), ('text/x-python-traceback',)), + 'RConsoleLexer': ('pygments.lexers.math', 'RConsole', ('rconsole', 'rout'), ('*.Rout',), ()), 'RagelCLexer': ('pygments.lexers.parsers', 'Ragel in C Host', ('ragel-c',), ('*.rl',), ()), 'RagelCppLexer': ('pygments.lexers.parsers', 'Ragel in CPP Host', ('ragel-cpp',), ('*.rl',), ()), 'RagelDLexer': ('pygments.lexers.parsers', 'Ragel in D Host', ('ragel-d',), ('*.rl',), ()), @@ -157,6 +167,7 @@ LEXERS = { 'RubyConsoleLexer': ('pygments.lexers.agile', 'Ruby irb session', ('rbcon', 'irb'), (), ('text/x-ruby-shellsession',)), 'RubyLexer': ('pygments.lexers.agile', 'Ruby', ('rb', 'ruby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx'), ('text/x-ruby', 'application/x-ruby')), 'SLexer': ('pygments.lexers.math', 'S', ('splus', 's', 'r'), ('*.S', '*.R'), ('text/S-plus', 'text/S', 'text/R')), + 'SassLexer': ('pygments.lexers.web', 'Sass', ('sass', 'SASS'), ('*.sass',), ('text/x-sass',)), 'ScalaLexer': ('pygments.lexers.compiled', 'Scala', ('scala',), ('*.scala',), ('text/x-scala',)), 'SchemeLexer': ('pygments.lexers.functional', 'Scheme', ('scheme', 'scm'), ('*.scm',), ('text/x-scheme', 'application/x-scheme')), 'SmalltalkLexer': ('pygments.lexers.other', 'Smalltalk', ('smalltalk', 'squeak'), ('*.st',), ('text/x-smalltalk',)), diff --git a/pygments/lexers/agile.py b/pygments/lexers/agile.py index dafed5c..bfaf0a6 100644 --- a/pygments/lexers/agile.py +++ b/pygments/lexers/agile.py @@ -10,10 +10,6 @@ """ import re -try: - set -except NameError: - from sets import Set as set from pygments.lexer import Lexer, RegexLexer, ExtendedRegexLexer, \ LexerContext, include, combined, do_insertions, bygroups, using @@ -41,7 +37,7 @@ class PythonLexer(RegexLexer): name = 'Python' aliases = ['python', 'py'] - filenames = ['*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript'] + filenames = ['*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac'] mimetypes = ['text/x-python', 'application/x-python'] tokens = { @@ -659,7 +655,7 @@ class RubyLexer(ExtendedRegexLexer): r'(?<=^match\s)|' r'(?<=^if\s)|' r'(?<=^elsif\s)' - r')(\s*)(/)(?!=)', bygroups(Text, String.Regex), 'multiline-regex'), + r')(\s*)(/)', bygroups(Text, String.Regex), 'multiline-regex'), # multiline regex (in method calls) (r'(?<=\(|,)/', String.Regex, 'multiline-regex'), # multiline regex (this time the funny no whitespace rule) @@ -835,7 +831,6 @@ class PerlLexer(RegexLexer): (r'@(\\\\|\\\@|[^\@])*@[egimosx]*', String.Regex, '#pop'), (r'%(\\\\|\\\%|[^\%])*%[egimosx]*', String.Regex, '#pop'), (r'\$(\\\\|\\\$|[^\$])*\$[egimosx]*', String.Regex, '#pop'), - (r'!(\\\\|\\!|[^!])*![egimosx]*', String.Regex, '#pop'), ], 'root': [ (r'\#.*?$', Comment.Single), @@ -859,6 +854,7 @@ class PerlLexer(RegexLexer): (r's\((\\\\|\\\)|[^\)])*\)\s*', String.Regex, 'balanced-regex'), (r'm?/(\\\\|\\/|[^/\n])*/[gcimosx]*', String.Regex), + (r'm(?=[/!\\{<\[\(@%\$])', String.Regex, 'balanced-regex'), (r'((?<==~)|(?<=\())\s*/(\\\\|\\/|[^/])*/[gcimosx]*', String.Regex), (r'\s+', Text), (r'(abs|accept|alarm|atan2|bind|binmode|bless|caller|chdir|' @@ -906,7 +902,7 @@ class PerlLexer(RegexLexer): (r'(q|qq|qw|qr|qx)\(', String.Other, 'rb-string'), (r'(q|qq|qw|qr|qx)\[', String.Other, 'sb-string'), (r'(q|qq|qw|qr|qx)\<', String.Other, 'lt-string'), - (r'(q|qq|qw|qr|qx)(.)[.\n]*?\1', String.Other), + (r'(q|qq|qw|qr|qx)([^a-zA-Z0-9])(.|\n)*?\2', String.Other), (r'package\s+', Keyword, 'modulename'), (r'sub\s+', Keyword, 'funcname'), (r'(\[\]|\*\*|::|<<|>>|>=|<=|<=>|={3}|!=|=~|' @@ -970,7 +966,7 @@ class PerlLexer(RegexLexer): (r'\\', String.Other), (r'\<', String.Other, 'lt-string'), (r'\>', String.Other, '#pop'), - (r'[^\<\>]]+', String.Other) + (r'[^\<\>]+', String.Other) ], 'end-part': [ (r'.+', Comment.Preproc, '#pop') @@ -1015,6 +1011,11 @@ class LuaLexer(RegexLexer): tokens = { 'root': [ + # lua allows a file to start with a shebang + (r'#!(.*?)$', Comment.Preproc), + (r'', Text, 'base'), + ], + 'base': [ (r'(?s)--\[(=*)\[.*?\]\1\]', Comment.Multiline), ('--.*$', Comment.Single), @@ -1263,6 +1264,7 @@ class TclLexer(RegexLexer): include('command'), include('basic'), include('data'), + (r'}', Keyword), # HACK: somehow we miscounted our braces ], 'command': _gen_command_rules(keyword_cmds_re, builtin_cmds_re), 'command-in-brace': _gen_command_rules(keyword_cmds_re, @@ -1441,7 +1443,7 @@ class ClojureLexer(RegexLexer): # strings, symbols and characters (r'"(\\\\|\\"|[^"])*"', String), (r"'" + valid_name, String.Symbol), - (r"\\([()/'\".'_!§$%& ?;=+-]{1}|[a-zA-Z0-9]+)", String.Char), + (r"\\([()/'\".'_!§$%& ?;=#+-]{1}|[a-zA-Z0-9]+)", String.Char), # constants (r'(#t|#f)', Name.Constant), diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py index e574a59..4740569 100644 --- a/pygments/lexers/asm.py +++ b/pygments/lexers/asm.py @@ -10,10 +10,6 @@ """ import re -try: - set -except NameError: - from sets import Set as set from pygments.lexer import RegexLexer, include, bygroups, using, DelegatingLexer from pygments.lexers.compiled import DLexer, CppLexer, CLexer diff --git a/pygments/lexers/compiled.py b/pygments/lexers/compiled.py index e382a1b..a2543e2 100644 --- a/pygments/lexers/compiled.py +++ b/pygments/lexers/compiled.py @@ -10,10 +10,6 @@ """ import re -try: - set -except NameError: - from sets import Set as set from pygments.scanner import Scanner from pygments.lexer import Lexer, RegexLexer, include, bygroups, using, \ @@ -29,7 +25,8 @@ from pygments.lexers.functional import OcamlLexer __all__ = ['CLexer', 'CppLexer', 'DLexer', 'DelphiLexer', 'JavaLexer', 'ScalaLexer', 'DylanLexer', 'OcamlLexer', 'ObjectiveCLexer', 'FortranLexer', 'GLShaderLexer', 'PrologLexer', 'CythonLexer', - 'ValaLexer', 'OocLexer', 'GoLexer'] + 'ValaLexer', 'OocLexer', 'GoLexer', 'FelixLexer', 'AdaLexer', + 'Modula2Lexer'] class CLexer(RegexLexer): @@ -63,6 +60,7 @@ class CLexer(RegexLexer): (r'0x[0-9a-fA-F]+[Ll]?', Number.Hex), (r'0[0-7]+[Ll]?', Number.Oct), (r'\d+[Ll]?', Number.Integer), + (r'\*/', Error), (r'[~!%^&*+=|?:<>/-]', Operator), (r'[()\[\],.]', Punctuation), (r'\b(case)(.+?)(:)', bygroups(Keyword, using(this), Text)), @@ -187,6 +185,7 @@ class CppLexer(RegexLexer): (r'0x[0-9a-fA-F]+[Ll]?', Number.Hex), (r'0[0-7]+[Ll]?', Number.Oct), (r'\d+[Ll]?', Number.Integer), + (r'\*/', Error), (r'[~!%^&*+=|?:<>/-]', Operator), (r'[()\[\],.;]', Punctuation), (r'(asm|auto|break|case|catch|const|const_cast|continue|' @@ -929,6 +928,7 @@ class JavaLexer(RegexLexer): ], } + class ScalaLexer(RegexLexer): """ For `Scala `_ source code. @@ -1122,14 +1122,14 @@ class ObjectiveCLexer(RegexLexer): (r'__(asm|int8|based|except|int16|stdcall|cdecl|fastcall|int32|' r'declspec|finally|int64|try|leave)\b', Keyword.Reserved), (r'(TRUE|FALSE|nil|NULL)\b', Name.Builtin), - ('[a-zA-Z_][a-zA-Z0-9_]*:(?!:)', Name.Label), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z$_][a-zA-Z0-9$_]*:(?!:)', Name.Label), + ('[a-zA-Z$_][a-zA-Z0-9$_]*', Name), ], 'root': [ include('whitespace'), # functions (r'((?:[a-zA-Z0-9_*\s])+?(?:\s|[*]))' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name + r'([a-zA-Z$_][a-zA-Z0-9$_]*)' # method name r'(\s*\([^;]*?\))' # signature r'(' + _ws + r')({)', bygroups(using(this), Name.Function, @@ -1137,7 +1137,7 @@ class ObjectiveCLexer(RegexLexer): 'function'), # function declarations (r'((?:[a-zA-Z0-9_*\s])+?(?:\s|[*]))' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name + r'([a-zA-Z$_][a-zA-Z0-9$_]*)' # method name r'(\s*\([^;]*?\))' # signature r'(' + _ws + r')(;)', bygroups(using(this), Name.Function, @@ -1151,18 +1151,18 @@ class ObjectiveCLexer(RegexLexer): ], 'classname' : [ # interface definition that inherits - ('([a-zA-Z_][a-zA-Z0-9_]*)(\s*:\s*)([a-zA-Z_][a-zA-Z0-9_]*)?', + ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*:\s*)([a-zA-Z$_][a-zA-Z0-9$_]*)?', bygroups(Name.Class, Text, Name.Class), '#pop'), # interface definition for a category - ('([a-zA-Z_][a-zA-Z0-9_]*)(\s*)(\([a-zA-Z_][a-zA-Z0-9_]*\))', + ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*)(\([a-zA-Z$_][a-zA-Z0-9$_]*\))', bygroups(Name.Class, Text, Name.Label), '#pop'), # simple interface / implementation - ('([a-zA-Z_][a-zA-Z0-9_]*)', Name.Class, '#pop') + ('([a-zA-Z$_][a-zA-Z0-9$_]*)', Name.Class, '#pop') ], 'forward_classname' : [ - ('([a-zA-Z_][a-zA-Z0-9_]*)(\s*,\s*)', + ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*,\s*)', bygroups(Name.Class, Text), 'forward_classname'), - ('([a-zA-Z_][a-zA-Z0-9_]*)(\s*;?)', + ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*;?)', bygroups(Name.Class, Text), '#pop') ], 'statement' : [ @@ -1790,3 +1790,576 @@ class GoLexer(RegexLexer): (r'[a-zA-Z_]\w*', Name), ] } + + +class FelixLexer(RegexLexer): + """ + For `Felix `_ source code. + + *New in Pygments 1.2.* + """ + + name = 'Felix' + aliases = ['felix', 'flx'] + filenames = ['*.flx', '*.flxh'] + mimetypes = ['text/x-felix'] + + preproc = [ + 'elif', 'else', 'endif', 'if', 'ifdef', 'ifndef', + ] + + keywords = [ + '_', '_deref', 'all', 'as', + 'assert', 'attempt', 'call', 'callback', 'case', 'caseno', 'cclass', + 'code', 'compound', 'ctypes', 'do', 'done', 'downto', 'elif', 'else', + 'endattempt', 'endcase', 'endif', 'endmatch', 'enum', 'except', + 'exceptions', 'expect', 'finally', 'for', 'forall', 'forget', 'fork', + 'functor', 'goto', 'ident', 'if', 'incomplete', 'inherit', 'instance', + 'interface', 'jump', 'lambda', 'loop', 'match', 'module', 'namespace', + 'new', 'noexpand', 'nonterm', 'obj', 'of', 'open', 'parse', 'raise', + 'regexp', 'reglex', 'regmatch', 'rename', 'return', 'the', 'then', + 'to', 'type', 'typecase', 'typedef', 'typematch', 'typeof', 'upto', + 'when', 'whilst', 'with', 'yield', + ] + + keyword_directives = [ + '_gc_pointer', '_gc_type', 'body', 'comment', 'const', 'export', + 'header', 'inline', 'lval', 'macro', 'noinline', 'noreturn', + 'package', 'private', 'pod', 'property', 'public', 'publish', + 'requires', 'todo', 'virtual', 'use', + ] + + keyword_declarations = [ + 'def', 'let', 'ref', 'val', 'var', + ] + + keyword_types = [ + 'unit', 'void', 'any', 'bool', + 'byte', 'offset', + 'address', 'caddress', 'cvaddress', 'vaddress', + 'tiny', 'short', 'int', 'long', 'vlong', + 'utiny', 'ushort', 'vshort', 'uint', 'ulong', 'uvlong', + 'int8', 'int16', 'int32', 'int64', + 'uint8', 'uint16', 'uint32', 'uint64', + 'float', 'double', 'ldouble', + 'complex', 'dcomplex', 'lcomplex', + 'imaginary', 'dimaginary', 'limaginary', + 'char', 'wchar', 'uchar', + 'charp', 'charcp', 'ucharp', 'ucharcp', + 'string', 'wstring', 'ustring', + 'cont', + 'array', 'varray', 'list', + 'lvalue', 'opt', 'slice', + ] + + keyword_constants = [ + 'false', 'true', + ] + + operator_words = [ + 'and', 'not', 'in', 'is', 'isin', 'or', 'xor', + ] + + name_builtins = [ + '_svc', 'while', + ] + + name_pseudo = [ + 'root', 'self', 'this', + ] + + decimal_suffixes = '([tTsSiIlLvV]|ll|LL|([iIuU])(8|16|32|64))?' + + tokens = { + 'root': [ + include('whitespace'), + + # Keywords + (r'(axiom|ctor|fun|gen|proc|reduce|union)\b', Keyword, + 'funcname'), + (r'(class|cclass|cstruct|obj|struct)\b', Keyword, 'classname'), + (r'(instance|module|typeclass)\b', Keyword, 'modulename'), + + (r'(%s)\b' % '|'.join(keywords), Keyword), + (r'(%s)\b' % '|'.join(keyword_directives), Name.Decorator), + (r'(%s)\b' % '|'.join(keyword_declarations), Keyword.Declaration), + (r'(%s)\b' % '|'.join(keyword_types), Keyword.Type), + (r'(%s)\b' % '|'.join(keyword_constants), Keyword.Constant), + + # Operators + include('operators'), + + # Float Literal + # -- Hex Float + (r'0[xX]([0-9a-fA-F_]*\.[0-9a-fA-F_]+|[0-9a-fA-F_]+)' + r'[pP][+\-]?[0-9_]+[lLfFdD]?', Number.Float), + # -- DecimalFloat + (r'[0-9_]+(\.[0-9_]+[eE][+\-]?[0-9_]+|' + r'\.[0-9_]*|[eE][+\-]?[0-9_]+)[lLfFdD]?', Number.Float), + (r'\.(0|[1-9][0-9_]*)([eE][+\-]?[0-9_]+)?[lLfFdD]?', + Number.Float), + + # IntegerLiteral + # -- Binary + (r'0[Bb][01_]+%s' % decimal_suffixes, Number), + # -- Octal + (r'0[0-7_]+%s' % decimal_suffixes, Number.Oct), + # -- Hexadecimal + (r'0[xX][0-9a-fA-F_]+%s' % decimal_suffixes, Number.Hex), + # -- Decimal + (r'(0|[1-9][0-9_]*)%s' % decimal_suffixes, Number.Integer), + + # Strings + ('([rR][cC]?|[cC][rR])"""', String, 'tdqs'), + ("([rR][cC]?|[cC][rR])'''", String, 'tsqs'), + ('([rR][cC]?|[cC][rR])"', String, 'dqs'), + ("([rR][cC]?|[cC][rR])'", String, 'sqs'), + ('[cCfFqQwWuU]?"""', String, combined('stringescape', 'tdqs')), + ("[cCfFqQwWuU]?'''", String, combined('stringescape', 'tsqs')), + ('[cCfFqQwWuU]?"', String, combined('stringescape', 'dqs')), + ("[cCfFqQwWuU]?'", String, combined('stringescape', 'sqs')), + + # Punctuation + (r'[\[\]{}:(),;?]', Punctuation), + + # Labels + (r'[a-zA-Z_]\w*:>', Name.Label), + + # Identifiers + (r'(%s)\b' % '|'.join(name_builtins), Name.Builtin), + (r'(%s)\b' % '|'.join(name_pseudo), Name.Builtin.Pseudo), + (r'[a-zA-Z_]\w*', Name), + ], + 'whitespace': [ + (r'\n', Text), + (r'\s+', Text), + + include('comment'), + + # Preprocessor + (r'#\s*if\s+0', Comment.Preproc, 'if0'), + (r'#', Comment.Preproc, 'macro'), + ], + 'operators': [ + (r'(%s)\b' % '|'.join(operator_words), Operator.Word), + (r'!=|==|<<|>>|\|\||&&|[-~+/*%=<>&^|.$]', Operator), + ], + 'comment': [ + (r'//(.*?)\n', Comment.Single), + (r'/[*]', Comment.Multiline, 'comment2'), + ], + 'comment2': [ + (r'[^\/*]', Comment.Multiline), + (r'/[*]', Comment.Multiline, '#push'), + (r'[*]/', Comment.Multiline, '#pop'), + (r'[\/*]', Comment.Multiline), + ], + 'if0': [ + (r'^\s*#if.*?(?]*?>)', + bygroups(Comment.Preproc, Text, String), '#pop'), + (r'(import|include)(\s+)("[^"]*?")', + bygroups(Comment.Preproc, Text, String), '#pop'), + (r"(import|include)(\s+)('[^']*?')", + bygroups(Comment.Preproc, Text, String), '#pop'), + (r'[^/\n]+', Comment.Preproc), + ##(r'/[*](.|\n)*?[*]/', Comment), + ##(r'//.*?\n', Comment, '#pop'), + (r'/', Comment.Preproc), + (r'(?<=\\)\n', Comment.Preproc), + (r'\n', Comment.Preproc, '#pop'), + ], + 'funcname': [ + include('whitespace'), + (r'[a-zA-Z_]\w*', Name.Function, '#pop'), + # anonymous functions + (r'(?=\()', Text, '#pop'), + ], + 'classname': [ + include('whitespace'), + (r'[a-zA-Z_]\w*', Name.Class, '#pop'), + # anonymous classes + (r'(?=\{)', Text, '#pop'), + ], + 'modulename': [ + include('whitespace'), + (r'\[', Punctuation, ('modulename2', 'tvarlist')), + (r'', Error, 'modulename2'), + ], + 'modulename2': [ + include('whitespace'), + (r'([a-zA-Z_]\w*)', Name.Namespace, '#pop:2'), + ], + 'tvarlist': [ + include('whitespace'), + include('operators'), + (r'\[', Punctuation, '#push'), + (r'\]', Punctuation, '#pop'), + (r',', Punctuation), + (r'(with|where)\b', Keyword), + (r'[a-zA-Z_]\w*', Name), + ], + 'stringescape': [ + (r'\\([\\abfnrtv"\']|\n|N{.*?}|u[a-fA-F0-9]{4}|' + r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape) + ], + 'strings': [ + (r'%(\([a-zA-Z0-9]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' + '[hlL]?[diouxXeEfFgGcrs%]', String.Interpol), + (r'[^\\\'"%\n]+', String), + # quotes, percents and backslashes must be parsed one at a time + (r'[\'"\\]', String), + # unhandled string formatting sign + (r'%', String) + # newlines are an error (use "nl" state) + ], + 'nl': [ + (r'\n', String) + ], + 'dqs': [ + (r'"', String, '#pop'), + # included here again for raw strings + (r'\\\\|\\"|\\\n', String.Escape), + include('strings') + ], + 'sqs': [ + (r"'", String, '#pop'), + # included here again for raw strings + (r"\\\\|\\'|\\\n", String.Escape), + include('strings') + ], + 'tdqs': [ + (r'"""', String, '#pop'), + include('strings'), + include('nl') + ], + 'tsqs': [ + (r"'''", String, '#pop'), + include('strings'), + include('nl') + ], + } + + +class AdaLexer(RegexLexer): + """ + For Ada source code. + + *New in Pygments 1.3.* + """ + + name = 'Ada' + aliases = ['ada', 'ada95' 'ada2005'] + filenames = ['*.adb', '*.ads', '*.ada'] + mimetypes = ['text/x-ada'] + + flags = re.MULTILINE | re.I # Ignore case + + _ws = r'(?:\s|//.*?\n|/[*].*?[*]/)+' + + tokens = { + 'root': [ + (r'[^\S\n]+', Text), + (r'--.*?\n', Comment.Single), + (r'[^\S\n]+', Text), + (r'function|procedure|entry', Keyword.Declaration, 'subprogram'), + (r'(subtype|type)(\s+)([a-z0-9_]+)', + bygroups(Keyword.Declaration, Text, Keyword.Type), 'type_def'), + (r'task|protected', Keyword.Declaration), + (r'(subtype)(\s+)', bygroups(Keyword.Declaration, Text)), + (r'(end)(\s+)', bygroups(Keyword.Reserved, Text), 'end'), + (r'(pragma)(\s+)([a-zA-Z0-9_]+)', bygroups(Keyword.Reserved, Text, + Comment.Preproc)), + (r'(true|false|null)\b', Keyword.Constant), + (r'(Byte|Character|Float|Integer|Long_Float|Long_Integer|' + r'Long_Long_Float|Long_Long_Integer|Natural|Positive|Short_Float|' + r'Short_Integer|Short_Short_Float|Short_Short_Integer|String|' + r'Wide_String|Duration)\b', Keyword.Type), + (r'(and(\s+then)?|in|mod|not|or(\s+else)|rem)\b', Operator.Word), + (r'generic|private', Keyword.Declaration), + (r'package', Keyword.Declaration, 'package'), + (r'array\b', Keyword.Reserved, 'array_def'), + (r'(with|use)(\s+)', bygroups(Keyword.Namespace, Text), 'import'), + (r'([a-z0-9_]+)(\s*)(:)(\s*)(constant)', + bygroups(Name.Constant, Text, Punctuation, Text, + Keyword.Reserved)), + (r'<<[a-z0-9_]+>>', Name.Label), + (r'([a-z0-9_]+)(\s*)(:)(\s*)(declare|begin|loop|for|while)', + bygroups(Name.Label, Text, Punctuation, Text, Keyword.Reserved)), + (r'\b(abort|abs|abstract|accept|access|aliased|all|array|at|begin|' + r'body|case|constant|declare|delay|delta|digits|do|else|elsif|end|' + r'entry|exception|exit|interface|for|goto|if|is|limited|loop|new|' + r'null|of|or|others|out|overriding|pragma|protected|raise|range|' + r'record|renames|requeue|return|reverse|select|separate|subtype|' + r'synchronized|task|tagged|terminate|then|type|until|when|while|' + r'xor)\b', + Keyword.Reserved), + (r'"[^"]*"', String), + include('attribute'), + include('numbers'), + (r"'[^']'", String.Character), + (r'([a-z0-9_]+)(\s*|[(,])', bygroups(Name, using(this))), + (r"(<>|=>|:=|[\(\)\|:;,.'])", Punctuation), + (r'[*<>+=/&-]', Operator), + (r'\n+', Text), + ], + 'numbers' : [ + (r'[0-9_]+#[0-9a-f]+#', Number.Hex), + (r'[0-9_]+\.[0-9_]*', Number.Float), + (r'[0-9_]+', Number.Integer), + ], + 'attribute' : [ + (r"(')([a-zA-Z0-9_]+)", bygroups(Punctuation, Name.Attribute)), + ], + 'subprogram' : [ + (r'\(', Punctuation, ('#pop', 'formal_part')), + (r';', Punctuation, '#pop'), + (r'is\b', Keyword.Reserved, '#pop'), + (r'"[^"]+"|[a-z0-9_]+', Name.Function), + include('root'), + ], + 'end' : [ + ('(if|case|record|loop|select)', Keyword.Reserved), + ('"[^"]+"|[a-zA-Z0-9_]+', Name.Function), + ('[\n\s]+', Text), + (';', Punctuation, '#pop'), + ], + 'type_def': [ + (r';', Punctuation, '#pop'), + (r'\(', Punctuation, 'formal_part'), + (r'with|and|use', Keyword.Reserved), + (r'array\b', Keyword.Reserved, ('#pop', 'array_def')), + (r'record\b', Keyword.Reserved, ('formal_part')), + include('root'), + ], + 'array_def' : [ + (r';', Punctuation, '#pop'), + (r'([a-z0-9_]+)(\s+)(range)', bygroups(Keyword.Type, Text, + Keyword.Reserved)), + include('root'), + ], + 'import': [ + (r'[a-z0-9_.]+', Name.Namespace, '#pop'), + ], + 'formal_part' : [ + (r'\)', Punctuation, '#pop'), + (r'([a-z0-9_]+)(\s*)(,|:[^=])', bygroups(Name.Variable, + Text, Punctuation)), + (r'(in|not|null|out|access)\b', Keyword.Reserved), + include('root'), + ], + 'package': [ + ('body', Keyword.Declaration), + ('is\s+new|renames', Keyword.Reserved), + ('is', Keyword.Reserved, '#pop'), + (';', Punctuation, '#pop'), + ('\(', Punctuation, 'package_instantiation'), + ('([a-zA-Z0-9_.]+)', Name.Class), + include('root'), + ], + 'package_instantiation': [ + (r'("[^"]+"|[a-z0-9_]+)(\s+)(=>)', bygroups(Name.Variable, + Text, Punctuation)), + (r'[a-z0-9._\'"]', Text), + (r'\)', Punctuation, '#pop'), + include('root'), + ], + } + + +class Modula2Lexer(RegexLexer): + """ + For `Modula-2 `_ source code. + + Additional options that determine which keywords are highlighted: + + `pim` + Select PIM Modula-2 dialect (default: True). + `iso` + Select ISO Modula-2 dialect (default: False). + `objm2` + Select Objective Modula-2 dialect (default: False). + `gm2ext` + Also highlight GNU extensions (default: False). + + *New in Pygments 1.3.* + """ + name = 'Modula-2' + aliases = ['modula2', 'm2'] + filenames = ['*.def', '*.mod'] + mimetypes = ['text/x-modula2'] + + flags = re.MULTILINE | re.DOTALL + + tokens = { + 'whitespace': [ + (r'\n+', Text), # blank lines + (r'\s+', Text), # whitespace + ], + 'identifiers': [ + (r'([a-zA-Z_\$][a-zA-Z0-9_\$]*)', Name), + ], + 'numliterals': [ + (r'[01]+B', Number.Binary), # binary number (ObjM2) + (r'[0-7]+B', Number.Oct), # octal number (PIM + ISO) + (r'[0-7]+C', Number.Oct), # char code (PIM + ISO) + (r'[0-9A-F]+C', Number.Hex), # char code (ObjM2) + (r'[0-9A-F]+H', Number.Hex), # hexadecimal number + (r'[0-9]+\.[0-9]+E[+-][0-9]+', Number.Float), # real number + (r'[0-9]+\.[0-9]+', Number.Float), # real number + (r'[0-9]+', Number.Integer), # decimal whole number + ], + 'strings': [ + (r"'(\\\\|\\'|[^'])*'", String), # single quoted string + (r'"(\\\\|\\"|[^"])*"', String), # double quoted string + ], + 'operators': [ + (r'[*/+=#~&<>\^-]', Operator), + (r':=', Operator), # assignment + (r'@', Operator), # pointer deref (ISO) + (r'\.\.', Operator), # ellipsis or range + (r'`', Operator), # Smalltalk message (ObjM2) + (r'::', Operator), # type conversion (ObjM2) + ], + 'punctuation': [ + (r'[\(\)\[\]{},.:;|]', Punctuation), + ], + 'comments': [ + (r'//.*?\n', Comment.Single), # ObjM2 + (r'/\*(.*?)\*/', Comment.Multiline), # ObjM2 + (r'\(\*([^\$].*?)\*\)', Comment.Multiline), + # TO DO: nesting of (* ... *) comments + ], + 'pragmas': [ + (r'\(\*\$(.*?)\*\)', Comment.Preproc), # PIM + (r'<\*(.*?)\*>', Comment.Preproc), # ISO + ObjM2 + ], + 'root': [ + include('whitespace'), + include('comments'), + include('pragmas'), + include('identifiers'), + include('numliterals'), + include('strings'), + include('operators'), + include('punctuation'), + ] + } + + pim_reserved_words = [ + # 40 reserved words + 'AND', 'ARRAY', 'BEGIN', 'BY', 'CASE', 'CONST', 'DEFINITION', + 'DIV', 'DO', 'ELSE', 'ELSIF', 'END', 'EXIT', 'EXPORT', 'FOR', + 'FROM', 'IF', 'IMPLEMENTATION', 'IMPORT', 'IN', 'LOOP', 'MOD', + 'MODULE', 'NOT', 'OF', 'OR', 'POINTER', 'PROCEDURE', 'QUALIFIED', + 'RECORD', 'REPEAT', 'RETURN', 'SET', 'THEN', 'TO', 'TYPE', + 'UNTIL', 'VAR', 'WHILE', 'WITH', + ] + + pim_pervasives = [ + # 31 pervasives + 'ABS', 'BITSET', 'BOOLEAN', 'CAP', 'CARDINAL', 'CHAR', 'CHR', 'DEC', + 'DISPOSE', 'EXCL', 'FALSE', 'FLOAT', 'HALT', 'HIGH', 'INC', 'INCL', + 'INTEGER', 'LONGINT', 'LONGREAL', 'MAX', 'MIN', 'NEW', 'NIL', 'ODD', + 'ORD', 'PROC', 'REAL', 'SIZE', 'TRUE', 'TRUNC', 'VAL', + ] + + iso_reserved_words = [ + # 46 reserved words + 'AND', 'ARRAY', 'BEGIN', 'BY', 'CASE', 'CONST', 'DEFINITION', 'DIV', + 'DO', 'ELSE', 'ELSIF', 'END', 'EXCEPT', 'EXIT', 'EXPORT', 'FINALLY', + 'FOR', 'FORWARD', 'FROM', 'IF', 'IMPLEMENTATION', 'IMPORT', 'IN', + 'LOOP', 'MOD', 'MODULE', 'NOT', 'OF', 'OR', 'PACKEDSET', 'POINTER', + 'PROCEDURE', 'QUALIFIED', 'RECORD', 'REPEAT', 'REM', 'RETRY', + 'RETURN', 'SET', 'THEN', 'TO', 'TYPE', 'UNTIL', 'VAR', 'WHILE', + 'WITH', + ] + + iso_pervasives = [ + # 42 pervasives + 'ABS', 'BITSET', 'BOOLEAN', 'CAP', 'CARDINAL', 'CHAR', 'CHR', 'CMPLX', + 'COMPLEX', 'DEC', 'DISPOSE', 'EXCL', 'FALSE', 'FLOAT', 'HALT', 'HIGH', + 'IM', 'INC', 'INCL', 'INT', 'INTEGER', 'INTERRUPTIBLE', 'LENGTH', + 'LFLOAT', 'LONGCOMPLEX', 'LONGINT', 'LONGREAL', 'MAX', 'MIN', 'NEW', + 'NIL', 'ODD', 'ORD', 'PROC', 'PROTECTION', 'RE', 'REAL', 'SIZE', + 'TRUE', 'TRUNC', 'UNINTERRUBTIBLE', 'VAL', + ] + + objm2_reserved_words = [ + # base language, 42 reserved words + 'AND', 'ARRAY', 'BEGIN', 'BY', 'CASE', 'CONST', 'DEFINITION', 'DIV', + 'DO', 'ELSE', 'ELSIF', 'END', 'ENUM', 'EXIT', 'FOR', 'FROM', 'IF', + 'IMMUTABLE', 'IMPLEMENTATION', 'IMPORT', 'IN', 'IS', 'LOOP', 'MOD', + 'MODULE', 'NOT', 'OF', 'OPAQUE', 'OR', 'POINTER', 'PROCEDURE', + 'RECORD', 'REPEAT', 'RETURN', 'SET', 'THEN', 'TO', 'TYPE', + 'UNTIL', 'VAR', 'VARIADIC', 'WHILE', + # OO extensions, 16 reserved words + 'BYCOPY', 'BYREF', 'CLASS', 'CONTINUE', 'CRITICAL', 'INOUT', 'METHOD', + 'ON', 'OPTIONAL', 'OUT', 'PRIVATE', 'PROTECTED', 'PROTOCOL', 'PUBLIC', + 'SUPER', 'TRY', + ] + + objm2_pervasives = [ + # base language, 38 pervasives + 'ABS', 'BITSET', 'BOOLEAN', 'CARDINAL', 'CHAR', 'CHR', 'DISPOSE', + 'FALSE', 'HALT', 'HIGH', 'INTEGER', 'INRANGE', 'LENGTH', 'LONGCARD', + 'LONGINT', 'LONGREAL', 'MAX', 'MIN', 'NEG', 'NEW', 'NEXTV', 'NIL', + 'OCTET', 'ODD', 'ORD', 'PRED', 'PROC', 'READ', 'REAL', 'SUCC', 'TMAX', + 'TMIN', 'TRUE', 'TSIZE', 'UNICHAR', 'VAL', 'WRITE', 'WRITEF', + # OO extensions, 3 pervasives + 'OBJECT', 'NO', 'YES', + ] + + gnu_reserved_words = [ + # 10 additional reserved words + 'ASM', '__ATTRIBUTE__', '__BUILTIN__', '__COLUMN__', '__DATE__', + '__FILE__', '__FUNCTION__', '__LINE__', '__MODULE__', 'VOLATILE', + ] + + gnu_pervasives = [ + # 21 identifiers, actually from pseudo-module SYSTEM + # but we will highlight them as if they were pervasives + 'BITSET8', 'BITSET16', 'BITSET32', 'CARDINAL8', 'CARDINAL16', + 'CARDINAL32', 'CARDINAL64', 'COMPLEX32', 'COMPLEX64', 'COMPLEX96', + 'COMPLEX128', 'INTEGER8', 'INTEGER16', 'INTEGER32', 'INTEGER64', + 'REAL8', 'REAL16', 'REAL32', 'REAL96', 'REAL128', 'THROW', + ] + + def __init__(self, **options): + self.reserved_words = set() + self.pervasives = set() + # ISO Modula-2 + if get_bool_opt(options, 'iso', False): + self.reserved_words.update(self.iso_reserved_words) + self.pervasives.update(self.iso_pervasives) + # Objective Modula-2 + elif get_bool_opt(options, 'objm2', False): + self.reserved_words.update(self.objm2_reserved_words) + self.pervasives.update(self.objm2_pervasives) + # PIM Modula-2 (DEFAULT) + else: + self.reserved_words.update(self.pim_reserved_words) + self.pervasives.update(self.pim_pervasives) + # GNU extensions + if get_bool_opt(options, 'gm2ext', False): + self.reserved_words.update(self.gnu_reserved_words) + self.pervasives.update(self.gnu_pervasives) + # initialise + RegexLexer.__init__(self, **options) + + def get_tokens_unprocessed(self, text): + for index, token, value in \ + RegexLexer.get_tokens_unprocessed(self, text): + # check for reserved words and pervasives + if token is Name: + if value in self.reserved_words: + token = Keyword.Reserved + elif value in self.pervasives: + token = Keyword.Pervasive + # return result + yield index, token, value diff --git a/pygments/lexers/functional.py b/pygments/lexers/functional.py index 8431ba3..ffbd753 100644 --- a/pygments/lexers/functional.py +++ b/pygments/lexers/functional.py @@ -10,10 +10,6 @@ """ import re -try: - set -except NameError: - from sets import Set as set from pygments.lexer import Lexer, RegexLexer, bygroups, include, do_insertions from pygments.token import Text, Comment, Operator, Keyword, Name, \ @@ -474,7 +470,7 @@ class LiterateHaskellLexer(Lexer): style = self.options.get('litstyle') if style is None: - style = (text.lstrip()[0] in '%\\') and 'latex' or 'bird' + style = (text.lstrip()[0:1] in '%\\') and 'latex' or 'bird' code = '' insertions = [] diff --git a/pygments/lexers/math.py b/pygments/lexers/math.py index 79097f2..448e299 100644 --- a/pygments/lexers/math.py +++ b/pygments/lexers/math.py @@ -10,10 +10,6 @@ """ import re -try: - set -except NameError: - from sets import Set as set from pygments.lexer import Lexer, RegexLexer, bygroups, include, do_insertions from pygments.token import Comment, String, Punctuation, Keyword, Name, \ @@ -22,7 +18,7 @@ from pygments.token import Comment, String, Punctuation, Keyword, Name, \ from pygments.lexers.agile import PythonLexer __all__ = ['MuPADLexer', 'MatlabLexer', 'MatlabSessionLexer', 'NumPyLexer', - 'SLexer'] + 'RConsoleLexer', 'SLexer'] class MuPADLexer(RegexLexer): @@ -340,6 +336,52 @@ class NumPyLexer(PythonLexer): yield index, token, value +class RConsoleLexer(Lexer): + """ + For R console transcripts or R CMD BATCH output files. + """ + + name = 'RConsole' + aliases = ['rconsole', 'rout'] + filenames = ['*.Rout'] + + def get_tokens_unprocessed(self, text): + slexer = SLexer(**self.options) + + current_code_block = '' + insertions = [] + + for match in line_re.finditer(text): + line = match.group() + if line.startswith('>') or line.startswith('+'): + # Colorize the prompt as such, + # then put rest of line into current_code_block + insertions.append((len(current_code_block), + [(0, Generic.Prompt, line[:2])])) + current_code_block += line[2:] + else: + # We have reached a non-prompt line! + # If we have stored prompt lines, need to process them first. + if current_code_block: + # Weave together the prompts and highlight code. + for item in do_insertions(insertions, + slexer.get_tokens_unprocessed(current_code_block)): + yield item + # Reset vars for next code block. + current_code_block = '' + insertions = [] + # Now process the actual line itself, this is output from R. + yield match.start(), Generic.Output, line + + # If we happen to end on a code block with nothing after it, need to + # process the last code block. This is neither elegant nor DRY so + # should be changed. + if current_code_block: + for item in do_insertions(insertions, + slexer.get_tokens_unprocessed(current_code_block)): + yield item + + class SLexer(RegexLexer): """ For S, S-plus, and R source code. diff --git a/pygments/lexers/other.py b/pygments/lexers/other.py index 7902435..8ca00fe 100644 --- a/pygments/lexers/other.py +++ b/pygments/lexers/other.py @@ -321,14 +321,14 @@ class BefungeLexer(RegexLexer): class BashLexer(RegexLexer): """ - Lexer for (ba)sh shell scripts. + Lexer for (ba|k|)sh shell scripts. *New in Pygments 0.6.* """ name = 'Bash' - aliases = ['bash', 'sh'] - filenames = ['*.sh', '*.ebuild', '*.eclass'] + aliases = ['bash', 'sh', 'ksh'] + filenames = ['*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass'] mimetypes = ['application/x-sh', 'application/x-shellscript'] tokens = { @@ -633,11 +633,11 @@ class SmalltalkLexer(RegexLexer): ], '_parenth_helper' : [ include('whitespaces'), + (r'(\d+r)?-?\d+(\.\d+)?(e-?\d+)?', Number), (r'[-+*/\\~<>=|&#!?,@%\w+:]+', String.Symbol), # literals (r'\'[^\']*\'', String), (r'\$.', String.Char), - (r'(\d+r)?-?\d+(\.\d+)?(e-?\d+)?', Number), (r'#*\(', String.Symbol, 'inner_parenth'), ], 'parenth' : [ @@ -2107,45 +2107,45 @@ class GherkinLexer(RegexLexer): feature_keywords_regexp = ur'^(기능|機能|功能|フィーチャ|خاصية|תכונה|Функционалност|Функционал|Особина|Могућност|Özellik|Właściwość|Tính năng|Savybė|Požiadavka|Požadavek|Osobina|Ominaisuus|Omadus|OH HAI|Mogućnost|Mogucnost|Jellemző|Fīča|Funzionalità|Funktionalität|Funkcionalnost|Funkcionalitāte|Funcționalitate|Functionaliteit|Functionalitate|Funcionalidade|Fonctionnalité|Fitur|Feature|Egenskap|Egenskab|Crikey|Característica|Arwedd)(:)(.*)$' scenario_keywords_regexp = ur'^(\s*)(시나리오 개요|시나리오|배경|背景|場景大綱|場景|场景大纲|场景|劇本大綱|劇本|テンプレ|シナリオテンプレート|シナリオテンプレ|シナリオアウトライン|シナリオ|سيناريو مخطط|سيناريو|الخلفية|תרחיש|תבנית תרחיש|רקע|Тарих|Сценарио|Сценарий структураси|Сценарий|Структура сценарија|Структура сценария|Скица|Рамка на сценарий|Пример|Предыстория|Предистория|Позадина|Основа|Концепт|Контекст|Założenia|Tình huống|Tausta|Taust|Tapausaihio|Tapaus|Szenariogrundriss|Szenario|Szablon scenariusza|Stsenaarium|Struktura scenarija|Skica|Skenario konsep|Skenario|Situācija|Senaryo taslağı|Senaryo|Scénář|Scénario|Schema dello scenario|Scenārijs pēc parauga|Scenārijs|Scenár|Scenariusz|Scenariul de şablon|Scenariul de sablon|Scenariu|Scenario Outline|Scenario Amlinellol|Scenario|Scenarijus|Scenarijaus šablonas|Scenarij|Scenarie|Rerefons|Raamstsenaarium|Primer|Pozadí|Pozadina|Pozadie|Plan du scénario|Plan du Scénario|Osnova scénáře|Osnova|Náčrt Scénáře|Náčrt Scenáru|Mate|MISHUN SRSLY|MISHUN|Kịch bản|Kontext|Konteksts|Kontekstas|Kontekst|Koncept|Khung tình huống|Khung kịch bản|Háttér|Grundlage|Geçmiş|Forgatókönyv vázlat|Forgatókönyv|Esquema do Cenário|Esquema do Cenario|Esquema del escenario|Esquema de l\'escenari|Escenario|Escenari|Dasar|Contexto|Contexte|Contesto|Condiţii|Conditii|Cenário|Cenario|Cefndir|Bối cảnh|Blokes|Bakgrunn|Bakgrund|Baggrund|Background|B4|Antecedents|Antecedentes|All y\'all|Achtergrond|Abstrakt Scenario|Abstract Scenario)(:)(.*)$' examples_regexp = ur'^(\s*)(예|例子|例|サンプル|امثلة|דוגמאות|Сценарији|Примери|Мисоллар|Значения|Örnekler|Voorbeelden|Variantai|Tapaukset|Scenarios|Scenariji|Scenarijai|Příklady|Példák|Príklady|Przykłady|Primjeri|Primeri|Piemēri|Pavyzdžiai|Paraugs|Juhtumid|Exemplos|Exemples|Exemplele|Exempel|Examples|Esempi|Enghreifftiau|Eksempler|Ejemplos|EXAMPLZ|Dữ liệu|Contoh|Cobber|Beispiele)(:)(.*)$' - step_keywords_regexp = ur'^(\s*)(하지만|조건|만일|그리고|그러면|那麼|那么|而且|當|当|前提|假設|假如|但是|但し|並且|もし|ならば|ただし|しかし|かつ|و |متى |لكن |عندما |ثم |بفرض |اذاً |כאשר |וגם |בהינתן |אזי |אז |אבל |Унда |То |Онда |Но |Лекин |Когато |Када |Кад |К тому же |И |Задато |Задати |Задате |Если |Допустим |Дадено |Ва |Бирок |Аммо |Али |Агар |А |Și |És |anrhegedig a |Zatati |Zakładając |Zadato |Zadate |Zadano |Zadani |Zadan |Yna |Ya know how |Ya gotta |Y |Wtedy |When y\'all |When |Wenn |WEN |Và |Ve |Und |Un |Thì |Then y\'all |Then |Tapi |Tak |Tada |Tad |Så |Soit |Siis |Si |Quando |Quand |Quan |Pryd |Pokud |Pokiaľ |Però |Pero |Pak |Oraz |Onda |Ond |Oletetaan |Og |Och |O zaman |Når |När |Niin |Nhưng |N |Mutta |Men |Mas |Maka |Majd |Mais |Maar |Ma |Lorsque |Lorsqu\'|Kun |Kuid |Kui |Khi |Keď |Ketika |Když |Kai |Kada |Kad |Jeżeli |Ja |Ir |I CAN HAZ |I |Ha |Givet |Given y\'all |Given |Gitt |Gegeven |Gegeben sei |Fakat |Eğer ki |Etant donné |Et |Então |Entonces |Entao |En |Eeldades |E |Duota |Donat |Donada |Diyelim ki |Dengan |De |Dato |Dar |Dann |Dan |Dado |Dacă |Daca |DEN |Când |Cuando |Cho |Cept |Cand |But y\'all |But |Biết |Bet |BUT |Atunci |And y\'all |And |Ama |Als |Alors |Allora |Ali |Aleshores |Ale |Akkor |Aber |AN |A také |A )' + step_keywords_regexp = ur'^(\s*)(하지만|조건|만일|그리고|그러면|那麼|那么|而且|當|当|前提|假設|假如|但是|但し|並且|もし|ならば|ただし|しかし|かつ|و |متى |لكن |عندما |ثم |بفرض |اذاً |כאשר |וגם |בהינתן |אזי |אז |אבל |Унда |То |Онда |Но |Лекин |Когато |Када |Кад |К тому же |И |Задато |Задати |Задате |Если |Допустим |Дадено |Ва |Бирок |Аммо |Али |Агар |А |Și |És |anrhegedig a |Zatati |Zakładając |Zadato |Zadate |Zadano |Zadani |Zadan |Yna |Ya know how |Ya gotta |Y |Wtedy |When y\'all |When |Wenn |WEN |Và |Ve |Und |Un |Thì |Then y\'all |Then |Tapi |Tak |Tada |Tad |Så |Stel |Soit |Siis |Si |Quando |Quand |Quan |Pryd |Pokud |Pokiaľ |Però |Pero |Pak |Oraz |Onda |Ond |Oletetaan |Og |Och |O zaman |Når |När |Niin |Nhưng |N |Mutta |Men |Mas |Maka |Majd |Mais |Maar |Ma |Lorsque |Lorsqu\'|Kun |Kuid |Kui |Khi |Keď |Ketika |Když |Kai |Kada |Kad |Jeżeli |Ja |Ir |I CAN HAZ |I |Ha |Givet |Given y\'all |Given |Gitt |Gegeven |Gegeben sei |Fakat |Eğer ki |Etant donné |Et |Então |Entonces |Entao |En |Eeldades |E |Duota |Donat |Donada |Diyelim ki |Dengan |De |Dato |Dar |Dann |Dan |Dado |Dacă |Daca |DEN |Când |Cuando |Cho |Cept |Cand |But y\'all |But |Biết |Bet |BUT |Atunci |And y\'all |And |Ama |Als |Alors |Allora |Ali |Aleshores |Ale |Akkor |Aber |AN |A také |A |\* )' tokens = { 'comments': [ - (r'#.*$', Comment), + (r'#.*$', Comment) ], 'multiline_descriptions' : [ (step_keywords_regexp, Keyword, "#pop"), include('comments'), - (r"(\s|.)", Name.Constant), + (r"(\s|.)", Name.Constant) ], 'multiline_descriptions_on_stack' : [ (step_keywords_regexp, Keyword, "#pop:2"), include('comments'), - (r"(\s|.)", Name.Constant), + (r"(\s|.)", Name.Constant) ], 'scenario_table_description': [ (r"\s+\|", Text, 'scenario_table_header'), include('comments'), - (r"(\s|.)", Name.Constant), + (r"(\s|.)", Name.Constant) ], 'scenario_table_header': [ (r"\s+\|\s*$", Text, "#pop:2"), (r"(\s+\|\s*)(#.*)$", bygroups(Text, Comment), "#pop:2"), include('comments'), (r"\s+\|", Text), - (r"[^\|]", Name.Variable), + (r"[^\|]", Name.Variable) ], 'scenario_sections_on_stack': [ (scenario_keywords_regexp, bygroups(Text, Name.Class, Name.Class, Name.Constant), - "multiline_descriptions_on_stack"), + "multiline_descriptions_on_stack") ], 'narrative': [ include('scenario_sections_on_stack'), - (r"(\s|.)", Name.Builtin), + (r"(\s|.)", Name.Builtin) ], 'table_vars': [ - (r'(<[^>]*>)', bygroups(Name.Variable)), + (r'(<[^>]*>)', bygroups(Name.Variable)) ], 'string': [ include('table_vars'), @@ -2159,16 +2159,11 @@ class GherkinLexer(RegexLexer): (r'"', String, "#pop"), include('string'), ], - 'single_string': [ - (r"'", String, "#pop"), - include('string'), - ], 'root': [ (r'\n', Text), include('comments'), (r'"""', String, "py_string"), (r'"', String, "double_string"), - (r"'", String, "single_string"), include('table_vars'), (r'@[^@\s]+', Name.Namespace), (step_keywords_regexp, bygroups(Text, Keyword)), @@ -2180,7 +2175,7 @@ class GherkinLexer(RegexLexer): (examples_regexp, bygroups(Text, Name.Class, Name.Class, Name.Constant), "scenario_table_description"), - (r'(\s|.)', Text), + (r'(\s|.)', Text) ] } diff --git a/pygments/lexers/parsers.py b/pygments/lexers/parsers.py index 15d58eb..0ead39b 100644 --- a/pygments/lexers/parsers.py +++ b/pygments/lexers/parsers.py @@ -12,17 +12,15 @@ import re from pygments.lexer import RegexLexer, DelegatingLexer, \ - include, bygroups, using, this -from pygments.token import Error, Punctuation, Generic, Other, \ - Text, Comment, Operator, Keyword, Name, String, Number, Whitespace + include, bygroups, using +from pygments.token import Punctuation, Other, Text, Comment, Operator, \ + Keyword, Name, String, Number, Whitespace from pygments.lexers.compiled import JavaLexer, CLexer, CppLexer, \ ObjectiveCLexer, DLexer from pygments.lexers.dotnet import CSharpLexer from pygments.lexers.agile import RubyLexer, PythonLexer, PerlLexer from pygments.lexers.web import ActionScriptLexer -# Use TextLexer during development to just focus on one part of a delegating -# lexer. -from pygments.lexers.special import TextLexer + __all__ = ['RagelLexer', 'RagelEmbeddedLexer', 'RagelCLexer', 'RagelDLexer', 'RagelCppLexer', 'RagelObjectiveCLexer', 'RagelRubyLexer', diff --git a/pygments/lexers/templates.py b/pygments/lexers/templates.py index 49b843d..eb84745 100644 --- a/pygments/lexers/templates.py +++ b/pygments/lexers/templates.py @@ -10,10 +10,6 @@ """ import re -try: - set -except NameError: - from sets import Set as set from pygments.lexers.web import \ PhpLexer, HtmlLexer, XmlLexer, JavascriptLexer, CssLexer @@ -38,7 +34,8 @@ __all__ = ['HtmlPhpLexer', 'XmlPhpLexer', 'CssPhpLexer', 'MakoHtmlLexer', 'MakoXmlLexer', 'MakoJavascriptLexer', 'MakoCssLexer', 'JspLexer', 'CheetahLexer', 'CheetahHtmlLexer', 'CheetahXmlLexer', 'CheetahJavascriptLexer', - 'EvoqueLexer', 'EvoqueHtmlLexer', 'EvoqueXmlLexer'] + 'EvoqueLexer', 'EvoqueHtmlLexer', 'EvoqueXmlLexer', + 'ColdfusionLexer', 'ColdfusionHtmlLexer'] class ErbLexer(Lexer): @@ -246,7 +243,7 @@ class DjangoLexer(RegexLexer): (r'\.[a-zA-Z0-9_]+', Name.Variable), (r':?"(\\\\|\\"|[^"])*"', String.Double), (r":?'(\\\\|\\'|[^'])*'", String.Single), - (r'([{}()\[\]+\-*/,:]|[><=]=?)', Operator), + (r'([{}()\[\]+\-*/,:~]|[><=]=?)', Operator), (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|" r"0[xX][0-9a-fA-F]+[Ll]?", Number), ], @@ -1294,3 +1291,97 @@ class EvoqueXmlLexer(DelegatingLexer): def __init__(self, **options): super(EvoqueXmlLexer, self).__init__(XmlLexer, EvoqueLexer, **options) + +class ColdfusionLexer(RegexLexer): + """ + Coldfusion statements + """ + name = 'cfstatement' + aliases = ['cfs'] + filenames = [] + mimetypes = [] + flags = re.IGNORECASE | re.MULTILINE + + tokens = { + 'root': [ + (r'//.*', Comment), + (r'\+\+|--', Operator), + (r'[-+*/^&=!]', Operator), + (r'<=|>=|<|>', Operator), + (r'mod\b', Operator), + (r'(eq|lt|gt|lte|gte|not|is|and|or)\b', Operator), + (r'\|\||&&', Operator), + (r'"', String.Double, 'string'), + # There is a special rule for allowing html in single quoted + # strings, evidently. + (r"'.*?'", String.Single), + (r'\d+', Number), + (r'(if|else|len|var|case|default|break|switch)\b', Keyword), + (r'([A-Za-z_$][A-Za-z0-9_.]*)\s*(\()', bygroups(Name.Function, Punctuation)), + (r'[A-Za-z_$][A-Za-z0-9_.]*', Name.Variable), + (r'[()\[\]{};:,.\\]', Punctuation), + (r'\s+', Text), + ], + 'string': [ + (r'""', String.Double), + (r'#.+?#', String.Interp), + (r'[^"#]+', String.Double), + (r'#', String.Double), + (r'"', String.Double, '#pop'), + ], + } + +class ColdfusionMarkupLexer(RegexLexer): + """ + Coldfusion markup only + """ + name = 'Coldfusion' + aliases = ['cf'] + filenames = [] + mimetypes = [] + + tokens = { + 'root': [ + (r'[^<]+', Other), + include('tags'), + (r'<[^<>]*', Other), + ], + 'tags': [ + (r'(?s)', Comment.Multiline), + (r'(?s)', Comment), + (r'', Name.Builtin, 'cfoutput'), + (r'(?s)()(.+?)()', + bygroups(Name.Builtin, using(ColdfusionLexer), Name.Builtin)), + # negative lookbehind is for strings with embedded > + (r'(?s)()', + bygroups(Name.Builtin, using(ColdfusionLexer), Name.Builtin)), + ], + 'cfoutput': [ + (r'[^#<]+', Other), + (r'(#)(.*?)(#)', bygroups(Punctuation, using(ColdfusionLexer), + Punctuation)), + #(r'', Name.Builtin, '#push'), + (r'', Name.Builtin, '#pop'), + include('tags'), + (r'(?s)<[^<>]*', Other), + (r'#', Other), + ], + } + + +class ColdfusionHtmlLexer(DelegatingLexer): + """ + Coldfusion markup in html + """ + name = 'Coldufsion HTML' + aliases = ['cfm'] + filenames = ['*.cfm', '*.cfml', '*.cfc'] + mimetypes = ['application/x-coldfusion'] + + def __init__(self, **options): + super(ColdfusionHtmlLexer, self).__init__(HtmlLexer, ColdfusionMarkupLexer, + **options) + diff --git a/pygments/lexers/text.py b/pygments/lexers/text.py index 1d7f89f..6b22370 100644 --- a/pygments/lexers/text.py +++ b/pygments/lexers/text.py @@ -10,10 +10,6 @@ """ import re -try: - set -except NameError: - from sets import Set as set from bisect import bisect from pygments.lexer import Lexer, LexerContext, RegexLexer, ExtendedRegexLexer, \ @@ -638,7 +634,8 @@ class RstLexer(RegexLexer): tokens = { 'root': [ # Heading with overline - (r'^(=+|-+|`+|:+|\.+|\'+|"+|~+|\^+|_+|\*+|\++|#+)([ \t]*\n)(.+)(\n)(\1)(\n)', + (r'^(=+|-+|`+|:+|\.+|\'+|"+|~+|\^+|_+|\*+|\++|#+)([ \t]*\n)' + r'(.+)(\n)(\1)(\n)', bygroups(Generic.Heading, Text, Generic.Heading, Text, Generic.Heading, Text)), # Plain heading @@ -658,24 +655,33 @@ class RstLexer(RegexLexer): bygroups(Text, Number, using(this, state='inline'))), (r'^(\s*)(\(?[A-Za-z]+\))( .+\n(?:\1 .+\n)+)', bygroups(Text, Number, using(this, state='inline'))), + # Line blocks + (r'^(\s*)(\|)( .+\n(?:\| .+\n)*)', + bygroups(Text, Operator, using(this, state='inline'))), # Sourcecode directives (r'^( *\.\.)(\s*)((?:source)?code)(::)([ \t]*)([^\n]+)' r'(\n[ \t]*\n)([ \t]+)(.*)(\n)((?:(?:\8.*|)\n)+)', _handle_sourcecode), # A directive - (r'^( *\.\.)(\s*)([\w-]+)(::)(?:([ \t]*)(.+))?', - bygroups(Punctuation, Text, Operator.Word, Punctuation, Text, Keyword)), + (r'^( *\.\.)(\s*)([\w:-]+?)(::)(?:([ \t]*)(.*))', + bygroups(Punctuation, Text, Operator.Word, Punctuation, Text, + using(this, state='inline'))), # A reference target (r'^( *\.\.)(\s*)([\w\t ]+:)(.*?)$', bygroups(Punctuation, Text, Name.Tag, using(this, state='inline'))), # A footnote target (r'^( *\.\.)(\s*)(\[.+\])(.*?)$', bygroups(Punctuation, Text, Name.Tag, using(this, state='inline'))), + # A substitution def + (r'^( *\.\.)(\s*)(\|.+\|)(\s*)([\w:-]+?)(::)(?:([ \t]*)(.*))', + bygroups(Punctuation, Text, Name.Tag, Text, Operator.Word, + Punctuation, Text, using(this, state='inline'))), # Comments (r'^ *\.\..*(\n( +.*\n|\n)+)?', Comment.Preproc), # Field list - (r'^( *)(:.*?:)([ \t]+)(.*?)$', bygroups(Text, Name.Class, Text, - Name.Function)), + (r'^( *)(:[a-zA-Z-]+:)(\s*)$', bygroups(Text, Name.Class, Text)), + (r'^( *)(:.*?:)([ \t]+)(.*?)$', + bygroups(Text, Name.Class, Text, Name.Function)), # Definition list (r'^([^ ].*(?)(`__?)', # reference with inline target + bygroups(String, String.Interpol, String)), + (r'`.+?`__?', String), # reference + (r'(`.+?`)(:[a-zA-Z0-9:-]+?:)?', bygroups(Name.Variable, Name.Attribute)), # role - (r'(:[a-zA-Z0-9-]+?:)(`.+?`)', - bygroups(Name.Attribute, Name.Variable)), # user-defined role + (r'(:[a-zA-Z0-9:-]+?:)(`.+?`)', + bygroups(Name.Attribute, Name.Variable)), # role (content first) (r'\*\*.+?\*\*', Generic.Strong), # Strong emphasis (r'\*.+?\*', Generic.Emph), # Emphasis (r'\[.*?\]_', String), # Footnote or citation @@ -1013,6 +1020,7 @@ class DebianControlLexer(RegexLexer): (r'[}]', Text), (r'[^,]$', Name.Function, '#pop'), (r'([\+\.a-zA-Z0-9-][\s\n]*)', Name.Function), + (r'\[.*?\]', Name.Entity), ], 'depend_vers': [ (r'\),', Text, '#pop'), @@ -1503,6 +1511,7 @@ class NginxConfLexer(RegexLexer): (r'[^\s;#{}$]+', String), # catch all (r'/[^\s;#]*', Name), # pathname (r'\s+', Text), + (r'[$;]', Text), # leftover characters ], } diff --git a/pygments/lexers/web.py b/pygments/lexers/web.py index 598611d..ec0b27b 100644 --- a/pygments/lexers/web.py +++ b/pygments/lexers/web.py @@ -10,21 +10,20 @@ """ import re -try: - set -except NameError: - from sets import Set as set -from pygments.lexer import RegexLexer, bygroups, using, include, this +from pygments.lexer import RegexLexer, ExtendedRegexLexer, bygroups, using, \ + include, this from pygments.token import \ Text, Comment, Operator, Keyword, Name, String, Number, Other, Punctuation from pygments.util import get_bool_opt, get_list_opt, looks_like_xml, \ html_doctype_matches +from pygments.lexers.agile import RubyLexer __all__ = ['HtmlLexer', 'XmlLexer', 'JavascriptLexer', 'CssLexer', 'PhpLexer', 'ActionScriptLexer', 'XsltLexer', 'ActionScript3Lexer', - 'MxmlLexer'] + 'MxmlLexer', 'HaxeLexer', 'HamlLexer', 'SassLexer', + 'ObjectiveJLexer', 'CoffeeScriptLexer'] class JavascriptLexer(RegexLexer): @@ -312,7 +311,7 @@ class CssLexer(RegexLexer): r'list-style|margin-bottom|margin-left|margin-right|' r'margin-top|margin|marker-offset|marks|max-height|max-width|' r'min-height|min-width|opacity|orphans|outline|outline-color|' - r'outline-style|outline-width|overflow|padding-bottom|' + r'outline-style|outline-width|overflow(?:-x|-y|)|padding-bottom|' r'padding-left|padding-right|padding-top|padding|page|' r'page-break-after|page-break-before|page-break-inside|' r'pause-after|pause-before|pause|pitch|pitch-range|' @@ -390,6 +389,229 @@ class CssLexer(RegexLexer): } +class ObjectiveJLexer(RegexLexer): + """ + For Objective-J source code with preprocessor directives. + + *New in Pygments 1.3.* + """ + + name = 'Objective-J' + aliases = ['objective-j', 'objectivej', 'obj-j', 'objj'] + filenames = ['*.j'] + mimetypes = ['text/x-objective-j'] + + #: optional Comment or Whitespace + _ws = r'(?:\s|//.*?\n|/[*].*?[*]/)*' + + flags = re.DOTALL | re.MULTILINE + + tokens = { + 'root': [ + include('whitespace'), + + # function definition + (r'^(' + _ws + r'[\+-]' + _ws + r')([\(a-zA-Z_].*?[^\(])(' + _ws + '{)', + bygroups(using(this), using(this, state='function_signature'), + using(this))), + + # class definition + (r'(@interface|@implementation)(\s+)', bygroups(Keyword, Text), + 'classname'), + (r'(@class|@protocol)(\s*)', bygroups(Keyword, Text), + 'forward_classname'), + (r'(\s*)(@end)(\s*)', bygroups(Text, Keyword, Text)), + + include('statements'), + ('[{\(\)}]', Punctuation), + (';', Punctuation), + ], + 'whitespace': [ + (r'(@import)(\s+)("(\\\\|\\"|[^"])*")', + bygroups(Comment.Preproc, Text, String.Double)), + (r'(@import)(\s+)(<(\\\\|\\>|[^>])*>)', + bygroups(Comment.Preproc, Text, String.Double)), + (r'(#(?:include|import))(\s+)("(\\\\|\\"|[^"])*")', + bygroups(Comment.Preproc, Text, String.Double)), + (r'(#(?:include|import))(\s+)(<(\\\\|\\>|[^>])*>)', + bygroups(Comment.Preproc, Text, String.Double)), + + (r'#if\s+0', Comment.Preproc, 'if0'), + (r'#', Comment.Preproc, 'macro'), + + (r'\n', Text), + (r'\s+', Text), + (r'\\\n', Text), # line continuation + (r'//(\n|(.|\n)*?[^\\]\n)', Comment.Single), + (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline), + (r' + + + +Date Functions + + + + + #RightNow#
    + #DateFormat(RightNow)#
    + #DateFormat(RightNow,"mm/dd/yy")#
    + #TimeFormat(RightNow)#
    + #TimeFormat(RightNow,"hh:mm tt")#
    + #IsDate(RightNow)#
    + #IsDate("January 31, 2007")#
    + #IsDate("foo")#
    + #DaysInMonth(RightNow)# +     + + #x# + #y# + #z# + + + + + + + + + + + + + + + diff --git a/tests/examplefiles/example.aspx b/tests/examplefiles/example.aspx deleted file mode 100644 index 01de00e..0000000 --- a/tests/examplefiles/example.aspx +++ /dev/null @@ -1,27 +0,0 @@ -<%@ Page Language="C#" %> - - - - - - - - Sample page - - -
    -
    - The current time is: -
    -
    - - - diff --git a/tests/examplefiles/example.xml b/tests/examplefiles/example.xml deleted file mode 100644 index e657e56..0000000 --- a/tests/examplefiles/example.xml +++ /dev/null @@ -1,1897 +0,0 @@ - - - - - - abort - abs - abstract - accept - access - aliased - all - and - array - at - begin - body - constant - declare - delay - delta - digits - do - else - elsif - end - entry - exception - exit - for - function - generic - goto - in - is - limited - mod - new - not - null - of - or - others - out - package - pragma - private - procedure - protected - raise - range - rem - record - renames - requeue - return - reverse - separate - subtype - tagged - task - terminate - then - type - until - use - when - while - with - xor - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - BEGIN - END - if - else - while - do - for - in - continue - break - print - printf - getline - function - return - next - exit - - - ARGC - ARGV - CONVFMT - ENVIRON - FILENAME - FNR - FS - NF - NR - OFMT - OFS - ORS - RS - RSTART - RLENGTH - SUBSEP - - - gsub - index - length - match - split - sprintf - sub - substr - tolower - toupper - atan2 - cos - exp - int - log - rand - sin - sqrt - srand - close - fflush - system - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - break - case - else - esac - exit - export - for - function - in - return - select - then - until - while - . - done - do - elif - fi - if - - - - cp - date - echo - eval - dcop - dcopstart - dcopfind - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - break - case - continue - default - do - else - enum - extern - for - goto - if - inline - return - sizeof - struct - switch - typedef - union - while - - - auto - char - const - double - float - int - long - register - restrict - short - signed - static - unsigned - void - volatile - _Imaginary - _Complex - _Bool - - - FIXME - TODO - ### - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - aaa - access-list - address - alias - arp - async-bootp - banner - boot - bridge - buffers - busy-message - call-history-mib - cdp - chat-script - class-map - clock - cns - config-register - controller - crypto - default - default-value - dialer - dialer-list - dnsix-dmdp - dnsix-nat - downward-compatible-config - enable - end - exception - exit - file - frame-relay - help - hostname - interface - ip - isdn - isdn-mib - kerberos - key - line - logging - login-string - map-class - map-list - memory-size - menu - modemcap - multilink - netbios - no - ntp - partition - policy-map - priority-list - privilege - process-max-time - prompt - queue-list - resume-string - rlogin - rmon - route-map - router - rtr - scheduler - service - snmp-server - sntp - stackmaker - state-machine - subscriber-policy - tacacs-server - template - terminal-queue - tftp-server - time-range - username - virtual-profile - virtual-template - vpdn - vpdn-group - x25 - x29 - - - accounting - accounting-list - accounting-threshold - accounting-transits - address-pool - as-path - audit - auth-proxy - authentication - authorization - bgp-community - bootp - cef - classless - community-list - default-gateway - default-network - dhcp - dhcp-server - domain-list - domain-lookup - domain-name - dvmrp - exec-callback - extcommunity-list - finger - flow-aggregation - flow-cache - flow-export - forward-protocol - ftp - gratuitous-arps - host - host-routing - hp-host - http - icmp - inspect - local - mrm - mroute - msdp - multicast - multicast-routing - name-server - nat - new-model - ospf - password - password-encryption - pgm - pim - port-map - prefix-list - radius - rcmd - reflexive-list - route - routing - rsvp - rtcp - sap - sdr - security - source-route - subnet-zero - tacacs - tcp - tcp-small-servers - telnet - tftp - timestamps - udp-small-servers - vrf - wccp - - - accounting - accounting-list - accounting-threshold - accounting-transits - address-pool - as-path - audit - auth-proxy - authentication - authorization - bgp-community - bootp - cef - classless - community-list - default-gateway - default-network - dhcp - dhcp-server - domain-list - domain-lookup - domain-name - dvmrp - exec-callback - extcommunity-list - finger - flow-aggregation - flow-cache - flow-export - forward-protocol - ftp - gratuitous-arps - host - host-routing - hp-host - http - icmp - inspect - local - mrm - mroute - msdp - multicast - multicast-routing - name-server - nat - new-model - ospf - password - password-encryption - pgm - pim - port-map - prefix-list - radius - rcmd - reflexive-list - route - routing - rsvp - rtcp - sap - sdr - security - source-route - subnet-zero - tacacs - tcp - tcp-small-servers - telnet - tftp - timestamps - udp-small-servers - vrf - wccp - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - if - else - for - in - while - do - continue - break - with - try - catch - switch - case - new - var - function - return - this - delete - true - false - void - throw - typeof - const - default - - - - - - Anchor - Applet - Area - Array - Boolean - Button - Checkbox - Date - Document - Event - FileUpload - Form - Frame - Function - Hidden - History - Image - Layer - Linke - Location - Math - Navigator - Number - Object - Option - Password - Radio - RegExp - Reset - Screen - Select - String - Submit - Text - Textarea - Window - - - - - - abs - acos - alert - anchor - apply - asin - atan - atan2 - back - blur - call - captureEvents - ceil - charAt - charCodeAt - clearInterval - clearTimeout - click - close - compile - concat - confirm - cos - disableExternalCapture - enableExternalCapture - eval - exec - exp - find - floor - focus - forward - fromCharCode - getDate - getDay - getFullYear - getHours - getMilliseconds - getMinutes - getMonth - getSeconds - getSelection - getTime - getTimezoneOffset - getUTCDate - getUTCDay - getUTCFullYear - getUTCHours - getUTCMilliseconds - getUTCMinutes - getUTCMonth - getUTCSeconds - go - handleEvent - home - indexOf - javaEnabled - join - lastIndexOf - link - load - log - match - max - min - moveAbove - moveBelow - moveBy - moveTo - moveToAbsolute - open - parse - plugins.refresh - pop - pow - preference - print - prompt - push - random - releaseEvents - reload - replace - reset - resizeBy - resizeTo - reverse - round - routeEvent - scrollBy - scrollTo - search - select - setDate - setFullYear - setHours - setInterval - setMilliseconds - setMinutes - setMonth - setSeconds - setTime - setTimeout - setUTCDate - setUTCFullYear - setUTCHours - setUTCMilliseconds - setUTCMinutes - setUTCMonth - setUTCSeconds - shift - sin - slice - sort - splice - split - sqrt - stop - String formatting - submit - substr - substring - taintEnabled - tan - test - toLocaleString - toLowerCase - toSource - toString - toUpperCase - toUTCString - unshift - unwatch - UTC - valueOf - watch - write - writeln - - - - - - break - case - catch - continue - default - do - else - for - function - if - in - return - switch - try - var - while - - - - - - Abs - ACos - ArrayAppend - ArrayAvg - ArrayClear - ArrayDeleteAt - ArrayInsertAt - ArrayIsEmpty - ArrayLen - ArrayMax - ArrayMin - ArrayNew - ArrayPrepend - ArrayResize - ArraySet - ArraySort - ArraySum - ArraySwap - ArrayToList - Asc - ASin - Atn - BitAnd - BitMaskClear - BitMaskRead - BitMaskSet - BitNot - BitOr - BitSHLN - BitSHRN - BitXor - Ceiling - Chr - CJustify - Compare - CompareNoCase - Cos - CreateDate - CreateDateTime - CreateObject - CreateODBCDate - CreateODBCDateTime - CreateODBCTime - CreateTime - CreateTimeSpan - CreateUUID - DateAdd - DateCompare - DateConvert - DateDiff - DateFormat - DatePart - Day - DayOfWeek - DayOfWeekAsString - DayOfYear - DaysInMonth - DaysInYear - DE - DecimalFormat - DecrementValue - Decrypt - DeleteClientVariable - DirectoryExists - DollarFormat - Duplicate - Encrypt - Evaluate - Exp - ExpandPath - FileExists - Find - FindNoCase - FindOneOf - FirstDayOfMonth - Fix - FormatBaseN - GetAuthUser - GetBaseTagData - GetBaseTagList - GetBaseTemplatePath - GetClientVariablesList - GetCurrentTemplatePath - GetDirectoryFromPath - GetException - GetFileFromPath - GetFunctionList - GetHttpRequestData - GetHttpTimeString - GetK2ServerDocCount - GetK2ServerDocCountLimit - GetLocale - GetMetaData - GetMetricData - GetPageContext - GetProfileSections - GetProfileString - GetServiceSettings - GetTempDirectory - GetTempFile - GetTemplatePath - GetTickCount - GetTimeZoneInfo - GetToken - Hash - Hour - HTMLCodeFormat - HTMLEditFormat - IIf - IncrementValue - InputBaseN - Insert - Int - IsArray - IsBinary - IsBoolean - IsCustomFunction - IsDate - IsDebugMode - IsDefined - IsK2ServerABroker - IsK2ServerDocCountExceeded - IsK2ServerOnline - IsLeapYear - IsNumeric - IsNumericDate - IsObject - IsQuery - IsSimpleValue - IsStruct - IsUserInRole - IsWDDX - IsXmlDoc - IsXmlElement - IsXmlRoot - JavaCast - JSStringFormat - LCase - Left - Len - ListAppend - ListChangeDelims - ListContains - ListContainsNoCase - ListDeleteAt - ListFind - ListFindNoCase - ListFirst - ListGetAt - ListInsertAt - ListLast - ListLen - ListPrepend - ListQualify - ListRest - ListSetAt - ListSort - ListToArray - ListValueCount - ListValueCountNoCase - LJustify - Log - Log10 - LSCurrencyFormat - LSDateFormat - LSEuroCurrencyFormat - LSIsCurrency - LSIsDate - LSIsNumeric - LSNumberFormat - LSParseCurrency - LSParseDateTime - LSParseEuroCurrency - LSParseNumber - LSTimeFormat - LTrim - Max - Mid - Min - Minute - Month - MonthAsString - Now - NumberFormat - ParagraphFormat - ParameterExists - ParseDateTime - Pi - PreserveSingleQuotes - Quarter - QueryAddColumn - QueryAddRow - QueryNew - QuerySetCell - QuotedValueList - Rand - Randomize - RandRange - REFind - REFindNoCase - RemoveChars - RepeatString - Replace - ReplaceList - ReplaceNoCase - REReplace - REReplaceNoCase - Reverse - Right - RJustify - Round - RTrim - Second - SetEncoding - SetLocale - SetProfileString - SetVariable - Sgn - Sin - SpanExcluding - SpanIncluding - Sqr - StripCR - StructAppend - StructClear - StructCopy - StructCount - StructDelete - StructFind - StructFindKey - StructFindValue - StructGet - StructInsert - StructIsEmpty - StructKeyArray - StructKeyExists - StructKeyList - StructNew - StructSort - StructUpdate - Tan - TimeFormat - ToBase64 - ToBinary - ToString - Trim - UCase - URLDecode - URLEncodedFormat - URLSessionFormat - Val - ValueList - Week - WriteOutput - XmlChildPos - XmlElemNew - XmlFormat - XmlNew - XmlParse - XmlSearch - XmlTransform - Year - YesNoFormat - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - BEGIN - BY - CASE - CLOSE - CONST - DO - ELSE - ELSIF - END - FOR - IF - IMPORT - LOOP - MODULE - NEW - OF - OUT - PROCEDURE - REPEAT - THEN - TO - TYPE - UNTIL - VAR - WHILE - WITH - - - ASSERT - EXIT - HALT - RETURN - - - ANYPTR - ANYREC - ARRAY - BOOLEAN - SHORTCHAR - CHAR - BYTE - SHORTINT - INTEGER - LONGINT - POINTER - RECORD - SHORTREAL - REAL - SET - - - ABSTRACT - EMPTY - EXTENSIBLE - LIMITED - - - ABS - ASH - BITS - CAP - CHR - DEC - ENTIER - EXCL - INC - INCL - LEN - LONG - MAX - MIN - ODD - ORD - SHORT - SIZE - - - FALSE - INF - NIL - TRUE - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tests/examplefiles/r-console-transcript.Rout b/tests/examplefiles/r-console-transcript.Rout new file mode 100644 index 0000000..d0cf34b --- /dev/null +++ b/tests/examplefiles/r-console-transcript.Rout @@ -0,0 +1,38 @@ + +R version 2.9.2 (2009-08-24) +Copyright (C) 2009 The R Foundation for Statistical Computing +ISBN 3-900051-07-0 + +R is free software and comes with ABSOLUTELY NO WARRANTY. +You are welcome to redistribute it under certain conditions. +Type 'license()' or 'licence()' for distribution details. + + Natural language support but running in an English locale + +R is a collaborative project with many contributors. +Type 'contributors()' for more information and +'citation()' on how to cite R or R packages in publications. + +Type 'demo()' for some demos, 'help()' for on-line help, or +'help.start()' for an HTML browser interface to help. +Type 'q()' to quit R. + +[R.app GUI 1.29 (5464) i386-apple-darwin8.11.1] + +> x <- function {} +Error: syntax error +> x <- function() {} +> x <- function() { ++ cat("hello") ++ cat("world") ++ } +> x +function() { +cat("hello") +cat("world") +} +> x() +helloworld +> 2 + 2 +[1] 4 +> \ No newline at end of file diff --git a/tests/examplefiles/test.adb b/tests/examplefiles/test.adb new file mode 100644 index 0000000..b79f3a5 --- /dev/null +++ b/tests/examplefiles/test.adb @@ -0,0 +1,211 @@ +-- Model IED Simulator +-- COL Gene Ressler, 1 December 2007 +with Ada.Text_IO; + +with Ada.Characters.Latin_1; +use Ada.Characters.Latin_1; + +with Ada.Strings.Fixed; +use Ada.Strings.Fixed; + +with Ada.Strings; +with Ada.Strings.Bounded; + +with Binary_Search; + +with Ada.Containers.Generic_Array_Sort; + +package body Scanner is + + Constant_123 : constant Character := Character'Val (16#00#); + MAX_KEYWORD_LENGTH_C : constant Natural := 24; + + New_Constant : constant New_Type + := 2; + + KEYWORDS_C : constant Keyword_Array_T := + (To_BS("description"), + To_BS("with")); + + procedure Blah; + + procedure blah is + begin + + Declaration: + declare + Joe : Type_Type := Random; + begin + Do_Something; + end Declaration; + Loop_ID: + loop + Loop_Do; + exit when 1=2; + end loop Loop_ID; + if True or else False then + Do_This(); + elsif not False and then True then + Do_That; + else + Panic; + end if; + end blah; + + function "*" (Left, Right : in Integer) return Integer is + begin + <> + goto Goto_Label; + return Left + Right; + end "*"; + + function Function_Specification + (Param_1 : in Blah; + Param2, param3 : in access Blah_Type := 0) + return It_Type; + + package Rename_Check renames Ada.Text_IO; + + type New_Float is delta 0.001 digits 12; + + package Package_Inst is new Ada.Strings.Bounded.Generic_Bounded_Length + (Max => MAX_KEYWORD_LENGTH_C); + + type Array_Decl12 is array (Positive range <>) of SB.Bounded_String; + type Array_Decl3 is array (New_Type range Thing_1 .. Thing_2) of SB.Bounded_String; + + type Boring_Type is + (Start, + End_Error); + + subtype Sub_Type_check is Character range '0' .. '9'; + + Initialized_Array : constant Transistion_Array_T := + (Start => + (Letter_Lower | Letter_Upper => Saw_Alpha, + ' ' | HT | CR | LF => Start, + others => Begin_Error), + + End_Error => (others => Start) + + ); + + type Recorder is record + Advance : Boolean; + Return_Token : Token_T; + end record; + + for Recorder use 8; + + type Null_Record is null record; + + type Discriminated_Record (Size : Natural) is + record + A : String (1 .. Size); + end record; + + pragma Unchecked_Union (Union); + pragma Convention (C, Union); + + type Person is tagged + record + Name : String (1 .. 10); + Gender : Gender_Type; + end record; + + type Programmer is new Person with + record + Skilled_In : Language_List; + Favorite_Langauge : Python_Type; + end record; + + type Programmer is new Person + and Printable + with + record + Skilled_In : Language_List; + Blah : aliased Integer; + end record; + + --------------------- + -- Scan_Next_Token -- + --------------------- + + task Cyclic_Buffer_Task_Type is + entry Insert (An_Item : in Item); + entry Remove (An_Item : out Item); + end Cyclic_Buffer_Task_Type; + + task body Cyclic_Buffer_Task_Type is + Q_Size : constant := 100; + subtype Q_Range is Positive range 1 .. Q_Size; + Length : Natural range 0 .. Q_Size := 0; + Head, Tail : Q_Range := 1; + Data : array (Q_Range) of Item; + begin + loop + select + when Length < Q_Size => + accept Insert (An_Item : in Item) do + Data(Tail) := An_Item; + end Insert; + Tail := Tail mod Q_Size + 1; + Length := Length + 1; + or + when Length > 0 => + accept Remove (An_Item : out Item) do + An_Item := Data(Head); + end Remove; + Head := Head mod Q_Size + 1; + Length := Length - 1; + end select; + end loop; + end Cyclic_Buffer_Task_Type; + + + + procedure Scan_Next_Token + (S : in String; + Start_Index : out Positive; + End_Index : in out Natural; -- Tricky comment + Line_Number : in out Positive; + Token : out Token_T); + + procedure Scan_Next_Token + (S : in String; + Start_Index : out Positive; + End_Index : in out Natural; -- Another comment + Line_Number : in out Positive; + Token : out Token_T) + is + begin + Scanner_Loop: + loop + if New_State = End_Error then + exit Scanner_Loop; + end if; + + if State = Start and New_State /= Start then + Start_Index := Peek_Index; + end if; + end loop Scanner_Loop; + end Scan_Next_Token; + + procedure Advance is + begin + Peek_Index := Peek_Index + 1; + end Advance; + + + -- Eliminate the leading space that Ada puts in front of positive + -- integer images. + function Image(N : in Integer) return String is + S : String := Integer'Image(N); + begin + if S(1) = ' ' then + return S(2 .. S'Last); + end if; + return S; + end Image; + +end Scanner; diff --git a/tests/examplefiles/test.flx b/tests/examplefiles/test.flx new file mode 100644 index 0000000..4c8a667 --- /dev/null +++ b/tests/examplefiles/test.flx @@ -0,0 +1,57 @@ +type tiny = "%i8"; +type int = "%i32"; +typedef bool = 2; +fun add : int*int -> int = "%add"; +fun sub : int*int -> int = "%sub"; +fun eq : int*int -> bool = "%eq"; +fun lnot : bool -> bool = "%lnot"; +proc exit : int = "exit"; + +// comment 1 +/* + /* + foo bar + */ +asdas +*/ + +noinline fun foo (x:int) = { + val y = 6; + return x + y; +} + +noinline proc fake_exit (x:int) { + exit x; + return; +} + +noinline fun bar (x:int) = { + var y = 10; + noinline proc baz () { + y = 20; + return; + } + baz (); + return x + y; +} + +noinline fun x (a:int, b:int, c:tiny) = { + val x1 = a; + val x2 = b; + val x3 = c; + noinline fun y (d:int, e:int, f:tiny) = { + val y1 = x1; + val y2 = x2; + val y3 = f; + noinline fun z (g:int, h:int, i:tiny) = { + val z1 = x1; + val z2 = x2; + val z3 = i; + return z1; + } + return z (y1,y2,y3); + } + return y (x1,x2,x3); +} + +fake_exit $ (foo 2) + (bar 3) + (x (1,2,3t)); diff --git a/tests/examplefiles/test.mod b/tests/examplefiles/test.mod new file mode 100644 index 0000000..ba972e3 --- /dev/null +++ b/tests/examplefiles/test.mod @@ -0,0 +1,374 @@ +(* LIFO Storage Library + * + * @file LIFO.mod + * LIFO implementation + * + * Universal Dynamic Stack + * + * Author: Benjamin Kowarsch + * + * Copyright (C) 2009 Benjamin Kowarsch. All rights reserved. + * + * License: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met + * + * 1) NO FEES may be charged for the provision of the software. The software + * may NOT be published on websites that contain advertising, unless + * specific prior written permission has been obtained. + * + * 2) Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 3) Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and other materials provided with the distribution. + * + * 4) Neither the author's name nor the names of any contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * 5) Where this list of conditions or the following disclaimer, in part or + * as a whole is overruled or nullified by applicable law, no permission + * is granted to use the software. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + *) + + +IMPLEMENTATION (* OF *) MODULE LIFO; + +FROM SYSTEM IMPORT ADDRESS, ADR, TSIZE; +FROM Storage IMPORT ALLOCATE, DEALLOCATE; + + +(* --------------------------------------------------------------------------- +// Private type : ListEntry +// --------------------------------------------------------------------------- +*) +TYPE ListPtr = POINTER TO ListEntry; + +TYPE ListEntry = RECORD + value : DataPtr; + next : ListPtr +END; (* ListEntry *) + + +(* --------------------------------------------------------------------------- +// Opaque type : LIFO.Stack +// --------------------------------------------------------------------------- +// CAUTION: Modula-2 does not support the use of variable length array fields +// in records. VLAs can only be implemented using pointer arithmetic which +// means there is no type checking and no boundary checking on the array. +// It also means that array notation cannot be used on the array which makes +// the code difficult to read and maintain. As a result, Modula-2 is less +// safe and less readable than C when it comes to using VLAs. Great care must +// be taken to make sure that the code accessing VLA fields is safe. Boundary +// checks must be inserted manually. Size checks must be inserted manually to +// compensate for the absence of type checks. *) + +TYPE Stack = POINTER TO StackDescriptor; + +TYPE StackDescriptor = RECORD + overflow : ListPtr; + entryCount : StackSize; + arraySize : StackSize; + array : ADDRESS (* ARRAY OF DataPtr *) +END; (* StackDescriptor *) + + +(* --------------------------------------------------------------------------- +// function: LIFO.new( initial_size, status ) +// --------------------------------------------------------------------------- +// +// Creates and returns a new LIFO stack object with an initial capacity of +// . If zero is passed in for , then the stack +// will be created with an initial capacity of LIFO.defaultStackSize. The +// function fails if a value greater than LIFO.maximumStackSize is passed +// in for or if memory could not be allocated. +// +// The initial capacity of a stack is the number of entries that can be stored +// in the stack without enlargement. +// +// The status of the operation is passed back in . *) + +PROCEDURE new ( initialSize : StackSize; VAR status : Status ) : Stack; + +VAR + newStack : Stack; + +BEGIN + + (* zero size means default *) + IF initialSize = 0 THEN + initialSize := defaultStackSize; + END; (* IF *) + + (* bail out if initial size is too high *) + IF initialSize > maximumStackSize THEN + status := invalidSize; + RETURN NIL; + END; (* IF *) + + (* allocate new stack object *) + ALLOCATE(newStack, TSIZE(Stack) + TSIZE(DataPtr) * (initialSize - 1)); + + (* bail out if allocation failed *) + IF newStack = NIL THEN + status := allocationFailed; + RETURN NIL; + END; (* IF *) + + (* initialise meta data *) + newStack^.arraySize := initialSize; + newStack^.entryCount := 0; + newStack^.overflow := NIL; + + (* pass status and new stack to caller *) + status := success; + RETURN newStack + +END new; + + +(* --------------------------------------------------------------------------- +// function: LIFO.push( stack, value, status ) +// --------------------------------------------------------------------------- +// +// Adds a new entry to the top of stack . The new entry is +// added by reference, no data is copied. However, no entry is added if the +// the stack is full, that is when the number of entries stored in the stack +// has reached LIFO.maximumStackSize. The function fails if NIL is passed in +// for or , or if memory could not be allocated. +// +// New entries are allocated dynamically if the number of entries exceeds the +// initial capacity of the stack. +// +// The status of the operation is passed back in . *) + +PROCEDURE push ( VAR stack : Stack; value : DataPtr; VAR status : Status ); +VAR + newEntry : ListPtr; + valuePtr : POINTER TO DataPtr; + +BEGIN + + (* bail out if stack is NIL *) + IF stack = NIL THEN + status := invalidStack; + RETURN; + END; (* IF *) + + (* bail out if value is NIL *) + IF value = NIL THEN + status := invalidData; + RETURN; + END; (* IF *) + + (* bail out if stack is full *) + IF stack^.entryCount >= maximumStackSize THEN + status := stackFull; + RETURN; + END; (* IF *) + + (* check if index falls within array segment *) + IF stack^.entryCount < stack^.arraySize THEN + + (* store value in array segment *) + + (* stack^.array^[stack^.entryCount] := value; *) + valuePtr := ADR(stack^.array) + TSIZE(DataPtr) * stack^.entryCount; + valuePtr^ := value; + + ELSE (* index falls within overflow segment *) + + (* allocate new entry slot *) + NEW(newEntry); + + (* bail out if allocation failed *) + IF newEntry = NIL THEN + status := allocationFailed; + RETURN; + END; (* IF *) + + (* initialise new entry *) + newEntry^.value := value; + + (* link new entry into overflow list *) + newEntry^.next := stack^.overflow; + stack^.overflow := newEntry; + + END; (* IF *) + + (* update entry counter *) + INC(stack^.entryCount); + + (* pass status to caller *) + status := success; + RETURN + +END push; + + +(* --------------------------------------------------------------------------- +// function: LIFO.pop( stack, status ) +// --------------------------------------------------------------------------- +// +// Removes the top most value from stack and returns it. If the stack +// is empty, that is when the number of entries stored in the stack has +// reached zero, then NIL is returned. +// +// Entries which were allocated dynamically (above the initial capacity) are +// deallocated when their values are popped. +// +// The status of the operation is passed back in . *) + +PROCEDURE pop ( VAR stack : Stack; VAR status : Status ) : DataPtr; + +VAR + thisValue : DataPtr; + thisEntry : ListPtr; + valuePtr : POINTER TO DataPtr; + +BEGIN + + (* bail out if stack is NIL *) + IF stack = NIL THEN + status := invalidStack; + RETURN NIL; + END; (* IF *) + + (* bail out if stack is empty *) + IF stack^.entryCount = 0 THEN + status := stackEmpty; + RETURN NIL; + END; (* IF *) + + DEC(stack^.entryCount); + + (* check if index falls within array segment *) + IF stack^.entryCount < stack^.arraySize THEN + + (* obtain value at index entryCount in array segment *) + + (* thisValue := stack^.array^[stack^.entryCount]; *) + valuePtr := ADR(stack^.array) + TSIZE(DataPtr) * stack^.entryCount; + thisValue := valuePtr^; + + ELSE (* index falls within overflow segment *) + + (* obtain value of first entry in overflow list *) + thisValue := stack^.overflow^.value; + + (* isolate first entry in overflow list *) + thisEntry := stack^.overflow; + stack^.overflow := stack^.overflow^.next; + + (* remove the entry from overflow list *) + DISPOSE(thisEntry); + + END; (* IF *) + + (* return value and status to caller *) + status := success; + RETURN thisValue + +END pop; + + +(* --------------------------------------------------------------------------- +// function: LIFO.stackSize( stack ) +// --------------------------------------------------------------------------- +// +// Returns the current capacity of . The current capacity is the total +// number of allocated entries. Returns zero if NIL is passed in for . +*) +PROCEDURE stackSize( VAR stack : Stack ) : StackSize; + +BEGIN + + (* bail out if stack is NIL *) + IF stack = NIL THEN + RETURN 0; + END; (* IF *) + + IF stack^.entryCount < stack^.arraySize THEN + RETURN stack^.arraySize; + ELSE + RETURN stack^.entryCount; + END; (* IF *) + +END stackSize; + + +(* --------------------------------------------------------------------------- +// function: LIFO.stackEntries( stack ) +// --------------------------------------------------------------------------- +// +// Returns the number of entries stored in stack , returns zero if +// NIL is passed in for . *) + +PROCEDURE stackEntries( VAR stack : Stack ) : StackSize; + +BEGIN + + (* bail out if stack is NIL *) + IF stack = NIL THEN + RETURN 0; + END; (* IF *) + + RETURN stack^.entryCount + +END stackEntries; + + +(* --------------------------------------------------------------------------- +// function: LIFO.dispose( stack ) +// --------------------------------------------------------------------------- +// +// Disposes of LIFO stack object . Returns NIL. *) + +PROCEDURE dispose ( VAR stack : Stack ) : Stack; + +VAR + thisEntry : ListPtr; + +BEGIN + + (* bail out if stack is NIL *) + IF stack = NIL THEN + RETURN NIL; + END; (* IF *) + + (* deallocate any entries in stack's overflow list *) + WHILE stack^.overflow # NIL DO + + (* isolate first entry in overflow list *) + thisEntry := stack^.overflow; + stack^.overflow := stack^.overflow^.next; + + (* deallocate the entry *) + DISPOSE(thisEntry); + + END; (* WHILE *) + + (* deallocate stack object and pass NIL to caller *) + DEALLOCATE(stack, TSIZE(Stack) + TSIZE(DataPtr) * (stack^.arraySize - 1)); + RETURN NIL + +END dispose; + + +END LIFO. diff --git a/tests/examplefiles/test.php b/tests/examplefiles/test.php index 90de747..97e21f7 100644 --- a/tests/examplefiles/test.php +++ b/tests/examplefiles/test.php @@ -1,4 +1,7 @@ + this._wrapped: obj + this + + + # Establish the object that gets thrown to break out of a loop iteration. + breaker: if typeof(StopIteration) is 'undefined' then '__break__' else StopIteration + + + # Create a safe reference to the Underscore object forreference below. + _: root._: (obj) -> new wrapper(obj) + + + # Export the Underscore object for CommonJS. + if typeof(exports) != 'undefined' then exports._: _ + + + # Create quick reference variables for speed access to core prototypes. + slice: Array::slice + unshift: Array::unshift + toString: Object::toString + hasOwnProperty: Object::hasOwnProperty + propertyIsEnumerable: Object::propertyIsEnumerable + + + # Current version. + _.VERSION: '0.5.7' + + + # ------------------------ Collection Functions: --------------------------- + + # The cornerstone, an each implementation. + # Handles objects implementing forEach, arrays, and raw objects. + _.each: (obj, iterator, context) -> + index: 0 + try + return obj.forEach(iterator, context) if obj.forEach + if _.isArray(obj) or _.isArguments(obj) + return iterator.call(context, obj[i], i, obj) for i in [0...obj.length] + iterator.call(context, val, key, obj) for key, val of obj + catch e + throw e if e isnt breaker + obj + + + # Return the results of applying the iterator to each element. Use JavaScript + # 1.6's version of map, if possible. + _.map: (obj, iterator, context) -> + return obj.map(iterator, context) if (obj and _.isFunction(obj.map)) + results: [] + _.each obj, (value, index, list) -> + results.push(iterator.call(context, value, index, list)) + results + + + # Reduce builds up a single result from a list of values. Also known as + # inject, or foldl. Uses JavaScript 1.8's version of reduce, if possible. + _.reduce: (obj, memo, iterator, context) -> + return obj.reduce(_.bind(iterator, context), memo) if (obj and _.isFunction(obj.reduce)) + _.each obj, (value, index, list) -> + memo: iterator.call(context, memo, value, index, list) + memo + + + # The right-associative version of reduce, also known as foldr. Uses + # JavaScript 1.8's version of reduceRight, if available. + _.reduceRight: (obj, memo, iterator, context) -> + return obj.reduceRight(_.bind(iterator, context), memo) if (obj and _.isFunction(obj.reduceRight)) + _.each _.clone(_.toArray(obj)).reverse(), (value, index) -> + memo: iterator.call(context, memo, value, index, obj) + memo + + + # Return the first value which passes a truth test. + _.detect: (obj, iterator, context) -> + result: null + _.each obj, (value, index, list) -> + if iterator.call(context, value, index, list) + result: value + _.breakLoop() + result + + + # Return all the elements that pass a truth test. Use JavaScript 1.6's + # filter(), if it exists. + _.select: (obj, iterator, context) -> + if obj and _.isFunction(obj.filter) then return obj.filter(iterator, context) + results: [] + _.each obj, (value, index, list) -> + results.push(value) if iterator.call(context, value, index, list) + results + + + # Return all the elements for which a truth test fails. + _.reject: (obj, iterator, context) -> + results: [] + _.each obj, (value, index, list) -> + results.push(value) if not iterator.call(context, value, index, list) + results + + + # Determine whether all of the elements match a truth test. Delegate to + # JavaScript 1.6's every(), if it is present. + _.all: (obj, iterator, context) -> + iterator ||= _.identity + return obj.every(iterator, context) if obj and _.isFunction(obj.every) + result: true + _.each obj, (value, index, list) -> + _.breakLoop() unless (result: result and iterator.call(context, value, index, list)) + result + + + # Determine if at least one element in the object matches a truth test. Use + # JavaScript 1.6's some(), if it exists. + _.any: (obj, iterator, context) -> + iterator ||= _.identity + return obj.some(iterator, context) if obj and _.isFunction(obj.some) + result: false + _.each obj, (value, index, list) -> + _.breakLoop() if (result: iterator.call(context, value, index, list)) + result + + + # Determine if a given value is included in the array or object, + # based on '==='. + _.include: (obj, target) -> + return _.indexOf(obj, target) isnt -1 if _.isArray(obj) + for key, val of obj + return true if val is target + false + + + # Invoke a method with arguments on every item in a collection. + _.invoke: (obj, method) -> + args: _.rest(arguments, 2) + (if method then val[method] else val).apply(val, args) for val in obj + + + # Convenience version of a common use case of map: fetching a property. + _.pluck: (obj, key) -> + _.map(obj, ((val) -> val[key])) + + + # Return the maximum item or (item-based computation). + _.max: (obj, iterator, context) -> + return Math.max.apply(Math, obj) if not iterator and _.isArray(obj) + result: {computed: -Infinity} + _.each obj, (value, index, list) -> + computed: if iterator then iterator.call(context, value, index, list) else value + computed >= result.computed and (result: {value: value, computed: computed}) + result.value + + + # Return the minimum element (or element-based computation). + _.min: (obj, iterator, context) -> + return Math.min.apply(Math, obj) if not iterator and _.isArray(obj) + result: {computed: Infinity} + _.each obj, (value, index, list) -> + computed: if iterator then iterator.call(context, value, index, list) else value + computed < result.computed and (result: {value: value, computed: computed}) + result.value + + + # Sort the object's values by a criteria produced by an iterator. + _.sortBy: (obj, iterator, context) -> + _.pluck(((_.map obj, (value, index, list) -> + {value: value, criteria: iterator.call(context, value, index, list)} + ).sort((left, right) -> + a: left.criteria; b: right.criteria + if a < b then -1 else if a > b then 1 else 0 + )), 'value') + + + # Use a comparator function to figure out at what index an object should + # be inserted so as to maintain order. Uses binary search. + _.sortedIndex: (array, obj, iterator) -> + iterator ||= _.identity + low: 0; high: array.length + while low < high + mid: (low + high) >> 1 + if iterator(array[mid]) < iterator(obj) then low: mid + 1 else high: mid + low + + + # Convert anything iterable into a real, live array. + _.toArray: (iterable) -> + return [] if (!iterable) + return iterable.toArray() if (iterable.toArray) + return iterable if (_.isArray(iterable)) + return slice.call(iterable) if (_.isArguments(iterable)) + _.values(iterable) + + + # Return the number of elements in an object. + _.size: (obj) -> _.toArray(obj).length + + + # -------------------------- Array Functions: ------------------------------ + + # Get the first element of an array. Passing "n" will return the first N + # values in the array. Aliased as "head". The "guard" check allows it to work + # with _.map. + _.first: (array, n, guard) -> + if n and not guard then slice.call(array, 0, n) else array[0] + + + # Returns everything but the first entry of the array. Aliased as "tail". + # Especially useful on the arguments object. Passing an "index" will return + # the rest of the values in the array from that index onward. The "guard" + # check allows it to work with _.map. + _.rest: (array, index, guard) -> + slice.call(array, if _.isUndefined(index) or guard then 1 else index) + + + # Get the last element of an array. + _.last: (array) -> array[array.length - 1] + + + # Trim out all falsy values from an array. + _.compact: (array) -> array[i] for i in [0...array.length] when array[i] + + + # Return a completely flattened version of an array. + _.flatten: (array) -> + _.reduce array, [], (memo, value) -> + return memo.concat(_.flatten(value)) if _.isArray(value) + memo.push(value) + memo + + + # Return a version of the array that does not contain the specified value(s). + _.without: (array) -> + values: _.rest(arguments) + val for val in _.toArray(array) when not _.include(values, val) + + + # Produce a duplicate-free version of the array. If the array has already + # been sorted, you have the option of using a faster algorithm. + _.uniq: (array, isSorted) -> + memo: [] + for el, i in _.toArray(array) + memo.push(el) if i is 0 || (if isSorted is true then _.last(memo) isnt el else not _.include(memo, el)) + memo + + + # Produce an array that contains every item shared between all the + # passed-in arrays. + _.intersect: (array) -> + rest: _.rest(arguments) + _.select _.uniq(array), (item) -> + _.all rest, (other) -> + _.indexOf(other, item) >= 0 + + + # Zip together multiple lists into a single array -- elements that share + # an index go together. + _.zip: -> + length: _.max(_.pluck(arguments, 'length')) + results: new Array(length) + for i in [0...length] + results[i]: _.pluck(arguments, String(i)) + results + + + # If the browser doesn't supply us with indexOf (I'm looking at you, MSIE), + # we need this function. Return the position of the first occurence of an + # item in an array, or -1 if the item is not included in the array. + _.indexOf: (array, item) -> + return array.indexOf(item) if array.indexOf + i: 0; l: array.length + while l - i + if array[i] is item then return i else i++ + -1 + + + # Provide JavaScript 1.6's lastIndexOf, delegating to the native function, + # if possible. + _.lastIndexOf: (array, item) -> + return array.lastIndexOf(item) if array.lastIndexOf + i: array.length + while i + if array[i] is item then return i else i-- + -1 + + + # Generate an integer Array containing an arithmetic progression. A port of + # the native Python range() function. See: + # http://docs.python.org/library/functions.html#range + _.range: (start, stop, step) -> + a: arguments + solo: a.length <= 1 + i: start: if solo then 0 else a[0]; + stop: if solo then a[0] else a[1]; + step: a[2] or 1 + len: Math.ceil((stop - start) / step) + return [] if len <= 0 + range: new Array(len) + idx: 0 + while true + return range if (if step > 0 then i - stop else stop - i) >= 0 + range[idx]: i + idx++ + i+= step + + + # ----------------------- Function Functions: ----------------------------- + + # Create a function bound to a given object (assigning 'this', and arguments, + # optionally). Binding with arguments is also known as 'curry'. + _.bind: (func, obj) -> + args: _.rest(arguments, 2) + -> func.apply(obj or root, args.concat(arguments)) + + + # Bind all of an object's methods to that object. Useful for ensuring that + # all callbacks defined on an object belong to it. + _.bindAll: (obj) -> + funcs: if arguments.length > 1 then _.rest(arguments) else _.functions(obj) + _.each(funcs, (f) -> obj[f]: _.bind(obj[f], obj)) + obj + + + # Delays a function for the given number of milliseconds, and then calls + # it with the arguments supplied. + _.delay: (func, wait) -> + args: _.rest(arguments, 2) + setTimeout((-> func.apply(func, args)), wait) + + + # Defers a function, scheduling it to run after the current call stack has + # cleared. + _.defer: (func) -> + _.delay.apply(_, [func, 1].concat(_.rest(arguments))) + + + # Returns the first function passed as an argument to the second, + # allowing you to adjust arguments, run code before and after, and + # conditionally execute the original function. + _.wrap: (func, wrapper) -> + -> wrapper.apply(wrapper, [func].concat(arguments)) + + + # Returns a function that is the composition of a list of functions, each + # consuming the return value of the function that follows. + _.compose: -> + funcs: arguments + -> + args: arguments + for i in [(funcs.length - 1)..0] + args: [funcs[i].apply(this, args)] + args[0] + + + # ------------------------- Object Functions: ---------------------------- + + # Retrieve the names of an object's properties. + _.keys: (obj) -> + return _.range(0, obj.length) if _.isArray(obj) + key for key, val of obj + + + # Retrieve the values of an object's properties. + _.values: (obj) -> + _.map(obj, _.identity) + + + # Return a sorted list of the function names available in Underscore. + _.functions: (obj) -> + _.select(_.keys(obj), (key) -> _.isFunction(obj[key])).sort() + + + # Extend a given object with all of the properties in a source object. + _.extend: (destination, source) -> + for key, val of source + destination[key]: val + destination + + + # Create a (shallow-cloned) duplicate of an object. + _.clone: (obj) -> + return obj.slice(0) if _.isArray(obj) + _.extend({}, obj) + + + # Invokes interceptor with the obj, and then returns obj. + # The primary purpose of this method is to "tap into" a method chain, in order to perform operations on intermediate results within the chain. + _.tap: (obj, interceptor) -> + interceptor(obj) + obj + + + # Perform a deep comparison to check if two objects are equal. + _.isEqual: (a, b) -> + # Check object identity. + return true if a is b + # Different types? + atype: typeof(a); btype: typeof(b) + return false if atype isnt btype + # Basic equality test (watch out for coercions). + return true if `a == b` + # One is falsy and the other truthy. + return false if (!a and b) or (a and !b) + # One of them implements an isEqual()? + return a.isEqual(b) if a.isEqual + # Check dates' integer values. + return a.getTime() is b.getTime() if _.isDate(a) and _.isDate(b) + # Both are NaN? + return true if _.isNaN(a) and _.isNaN(b) + # Compare regular expressions. + if _.isRegExp(a) and _.isRegExp(b) + return a.source is b.source and + a.global is b.global and + a.ignoreCase is b.ignoreCase and + a.multiline is b.multiline + # If a is not an object by this point, we can't handle it. + return false if atype isnt 'object' + # Check for different array lengths before comparing contents. + return false if a.length and (a.length isnt b.length) + # Nothing else worked, deep compare the contents. + aKeys: _.keys(a); bKeys: _.keys(b) + # Different object sizes? + return false if aKeys.length isnt bKeys.length + # Recursive comparison of contents. + # for (var key in a) if (!_.isEqual(a[key], b[key])) return false; + return true + + + # Is a given array or object empty? + _.isEmpty: (obj) -> _.keys(obj).length is 0 + + + # Is a given value a DOM element? + _.isElement: (obj) -> obj and obj.nodeType is 1 + + + # Is a given value an array? + _.isArray: (obj) -> !!(obj and obj.concat and obj.unshift) + + + # Is a given variable an arguments object? + _.isArguments: (obj) -> obj and _.isNumber(obj.length) and not obj.concat and + not obj.substr and not obj.apply and not propertyIsEnumerable.call(obj, 'length') + + + # Is the given value a function? + _.isFunction: (obj) -> !!(obj and obj.constructor and obj.call and obj.apply) + + + # Is the given value a string? + _.isString: (obj) -> !!(obj is '' or (obj and obj.charCodeAt and obj.substr)) + + + # Is a given value a number? + _.isNumber: (obj) -> (obj is +obj) or toString.call(obj) is '[object Number]' + + + # Is a given value a Date? + _.isDate: (obj) -> !!(obj and obj.getTimezoneOffset and obj.setUTCFullYear) + + + # Is the given value a regular expression? + _.isRegExp: (obj) -> !!(obj and obj.exec and (obj.ignoreCase or obj.ignoreCase is false)) + + + # Is the given value NaN -- this one is interesting. NaN != NaN, and + # isNaN(undefined) == true, so we make sure it's a number first. + _.isNaN: (obj) -> _.isNumber(obj) and window.isNaN(obj) + + + # Is a given value equal to null? + _.isNull: (obj) -> obj is null + + + # Is a given variable undefined? + _.isUndefined: (obj) -> typeof obj is 'undefined' + + + # -------------------------- Utility Functions: -------------------------- + + # Run Underscore.js in noConflict mode, returning the '_' variable to its + # previous owner. Returns a reference to the Underscore object. + _.noConflict: -> + root._: previousUnderscore + this + + + # Keep the identity function around for default iterators. + _.identity: (value) -> value + + + # Break out of the middle of an iteration. + _.breakLoop: -> throw breaker + + + # Generate a unique integer id (unique within the entire client session). + # Useful for temporary DOM ids. + idCounter: 0 + _.uniqueId: (prefix) -> + (prefix or '') + idCounter++ + + + # By default, Underscore uses ERB-style template delimiters, change the + # following template settings to use alternative delimiters. + _.templateSettings: { + start: '<%' + end: '%>' + interpolate: /<%=(.+?)%>/g + } + + + # JavaScript templating a-la ERB, pilfered from John Resig's + # "Secrets of the JavaScript Ninja", page 83. + # Single-quotea fix from Rick Strahl's version. + _.template: (str, data) -> + c: _.templateSettings + fn: new Function 'obj', + 'var p=[],print=function(){p.push.apply(p,arguments);};' + + 'with(obj){p.push(\'' + + str.replace(/[\r\t\n]/g, " ") + .replace(new RegExp("'(?=[^"+c.end[0]+"]*"+c.end+")","g"),"\t") + .split("'").join("\\'") + .split("\t").join("'") + .replace(c.interpolate, "',$1,'") + .split(c.start).join("');") + .split(c.end).join("p.push('") + + "');}return p.join('');" + if data then fn(data) else fn + + + # ------------------------------- Aliases ---------------------------------- + + _.forEach: _.each + _.foldl: _.inject: _.reduce + _.foldr: _.reduceRight + _.filter: _.select + _.every: _.all + _.some: _.any + _.head: _.first + _.tail: _.rest + _.methods: _.functions + + + # /*------------------------ Setup the OOP Wrapper: --------------------------*/ + + # Helper function to continue chaining intermediate results. + result: (obj, chain) -> + if chain then _(obj).chain() else obj + + + # Add all of the Underscore functions to the wrapper object. + _.each _.functions(_), (name) -> + method: _[name] + wrapper.prototype[name]: -> + unshift.call(arguments, this._wrapped) + result(method.apply(_, arguments), this._chain) + + + # Add all mutator Array functions to the wrapper. + _.each ['pop', 'push', 'reverse', 'shift', 'sort', 'splice', 'unshift'], (name) -> + method: Array.prototype[name] + wrapper.prototype[name]: -> + method.apply(this._wrapped, arguments) + result(this._wrapped, this._chain) + + + # Add all accessor Array functions to the wrapper. + _.each ['concat', 'join', 'slice'], (name) -> + method: Array.prototype[name] + wrapper.prototype[name]: -> + result(method.apply(this._wrapped, arguments), this._chain) + + + # Start chaining a wrapped Underscore object. + wrapper::chain: -> + this._chain: true + this + + + # Extracts the result from a wrapped and chained object. + wrapper::value: -> this._wrapped diff --git a/tests/examplefiles/xml_example b/tests/examplefiles/xml_example new file mode 100644 index 0000000..e657e56 --- /dev/null +++ b/tests/examplefiles/xml_example @@ -0,0 +1,1897 @@ + + + + + + abort + abs + abstract + accept + access + aliased + all + and + array + at + begin + body + constant + declare + delay + delta + digits + do + else + elsif + end + entry + exception + exit + for + function + generic + goto + in + is + limited + mod + new + not + null + of + or + others + out + package + pragma + private + procedure + protected + raise + range + rem + record + renames + requeue + return + reverse + separate + subtype + tagged + task + terminate + then + type + until + use + when + while + with + xor + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BEGIN + END + if + else + while + do + for + in + continue + break + print + printf + getline + function + return + next + exit + + + ARGC + ARGV + CONVFMT + ENVIRON + FILENAME + FNR + FS + NF + NR + OFMT + OFS + ORS + RS + RSTART + RLENGTH + SUBSEP + + + gsub + index + length + match + split + sprintf + sub + substr + tolower + toupper + atan2 + cos + exp + int + log + rand + sin + sqrt + srand + close + fflush + system + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + break + case + else + esac + exit + export + for + function + in + return + select + then + until + while + . + done + do + elif + fi + if + + + + cp + date + echo + eval + dcop + dcopstart + dcopfind + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + break + case + continue + default + do + else + enum + extern + for + goto + if + inline + return + sizeof + struct + switch + typedef + union + while + + + auto + char + const + double + float + int + long + register + restrict + short + signed + static + unsigned + void + volatile + _Imaginary + _Complex + _Bool + + + FIXME + TODO + ### + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + aaa + access-list + address + alias + arp + async-bootp + banner + boot + bridge + buffers + busy-message + call-history-mib + cdp + chat-script + class-map + clock + cns + config-register + controller + crypto + default + default-value + dialer + dialer-list + dnsix-dmdp + dnsix-nat + downward-compatible-config + enable + end + exception + exit + file + frame-relay + help + hostname + interface + ip + isdn + isdn-mib + kerberos + key + line + logging + login-string + map-class + map-list + memory-size + menu + modemcap + multilink + netbios + no + ntp + partition + policy-map + priority-list + privilege + process-max-time + prompt + queue-list + resume-string + rlogin + rmon + route-map + router + rtr + scheduler + service + snmp-server + sntp + stackmaker + state-machine + subscriber-policy + tacacs-server + template + terminal-queue + tftp-server + time-range + username + virtual-profile + virtual-template + vpdn + vpdn-group + x25 + x29 + + + accounting + accounting-list + accounting-threshold + accounting-transits + address-pool + as-path + audit + auth-proxy + authentication + authorization + bgp-community + bootp + cef + classless + community-list + default-gateway + default-network + dhcp + dhcp-server + domain-list + domain-lookup + domain-name + dvmrp + exec-callback + extcommunity-list + finger + flow-aggregation + flow-cache + flow-export + forward-protocol + ftp + gratuitous-arps + host + host-routing + hp-host + http + icmp + inspect + local + mrm + mroute + msdp + multicast + multicast-routing + name-server + nat + new-model + ospf + password + password-encryption + pgm + pim + port-map + prefix-list + radius + rcmd + reflexive-list + route + routing + rsvp + rtcp + sap + sdr + security + source-route + subnet-zero + tacacs + tcp + tcp-small-servers + telnet + tftp + timestamps + udp-small-servers + vrf + wccp + + + accounting + accounting-list + accounting-threshold + accounting-transits + address-pool + as-path + audit + auth-proxy + authentication + authorization + bgp-community + bootp + cef + classless + community-list + default-gateway + default-network + dhcp + dhcp-server + domain-list + domain-lookup + domain-name + dvmrp + exec-callback + extcommunity-list + finger + flow-aggregation + flow-cache + flow-export + forward-protocol + ftp + gratuitous-arps + host + host-routing + hp-host + http + icmp + inspect + local + mrm + mroute + msdp + multicast + multicast-routing + name-server + nat + new-model + ospf + password + password-encryption + pgm + pim + port-map + prefix-list + radius + rcmd + reflexive-list + route + routing + rsvp + rtcp + sap + sdr + security + source-route + subnet-zero + tacacs + tcp + tcp-small-servers + telnet + tftp + timestamps + udp-small-servers + vrf + wccp + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + if + else + for + in + while + do + continue + break + with + try + catch + switch + case + new + var + function + return + this + delete + true + false + void + throw + typeof + const + default + + + + + + Anchor + Applet + Area + Array + Boolean + Button + Checkbox + Date + Document + Event + FileUpload + Form + Frame + Function + Hidden + History + Image + Layer + Linke + Location + Math + Navigator + Number + Object + Option + Password + Radio + RegExp + Reset + Screen + Select + String + Submit + Text + Textarea + Window + + + + + + abs + acos + alert + anchor + apply + asin + atan + atan2 + back + blur + call + captureEvents + ceil + charAt + charCodeAt + clearInterval + clearTimeout + click + close + compile + concat + confirm + cos + disableExternalCapture + enableExternalCapture + eval + exec + exp + find + floor + focus + forward + fromCharCode + getDate + getDay + getFullYear + getHours + getMilliseconds + getMinutes + getMonth + getSeconds + getSelection + getTime + getTimezoneOffset + getUTCDate + getUTCDay + getUTCFullYear + getUTCHours + getUTCMilliseconds + getUTCMinutes + getUTCMonth + getUTCSeconds + go + handleEvent + home + indexOf + javaEnabled + join + lastIndexOf + link + load + log + match + max + min + moveAbove + moveBelow + moveBy + moveTo + moveToAbsolute + open + parse + plugins.refresh + pop + pow + preference + print + prompt + push + random + releaseEvents + reload + replace + reset + resizeBy + resizeTo + reverse + round + routeEvent + scrollBy + scrollTo + search + select + setDate + setFullYear + setHours + setInterval + setMilliseconds + setMinutes + setMonth + setSeconds + setTime + setTimeout + setUTCDate + setUTCFullYear + setUTCHours + setUTCMilliseconds + setUTCMinutes + setUTCMonth + setUTCSeconds + shift + sin + slice + sort + splice + split + sqrt + stop + String formatting + submit + substr + substring + taintEnabled + tan + test + toLocaleString + toLowerCase + toSource + toString + toUpperCase + toUTCString + unshift + unwatch + UTC + valueOf + watch + write + writeln + + + + + + break + case + catch + continue + default + do + else + for + function + if + in + return + switch + try + var + while + + + + + + Abs + ACos + ArrayAppend + ArrayAvg + ArrayClear + ArrayDeleteAt + ArrayInsertAt + ArrayIsEmpty + ArrayLen + ArrayMax + ArrayMin + ArrayNew + ArrayPrepend + ArrayResize + ArraySet + ArraySort + ArraySum + ArraySwap + ArrayToList + Asc + ASin + Atn + BitAnd + BitMaskClear + BitMaskRead + BitMaskSet + BitNot + BitOr + BitSHLN + BitSHRN + BitXor + Ceiling + Chr + CJustify + Compare + CompareNoCase + Cos + CreateDate + CreateDateTime + CreateObject + CreateODBCDate + CreateODBCDateTime + CreateODBCTime + CreateTime + CreateTimeSpan + CreateUUID + DateAdd + DateCompare + DateConvert + DateDiff + DateFormat + DatePart + Day + DayOfWeek + DayOfWeekAsString + DayOfYear + DaysInMonth + DaysInYear + DE + DecimalFormat + DecrementValue + Decrypt + DeleteClientVariable + DirectoryExists + DollarFormat + Duplicate + Encrypt + Evaluate + Exp + ExpandPath + FileExists + Find + FindNoCase + FindOneOf + FirstDayOfMonth + Fix + FormatBaseN + GetAuthUser + GetBaseTagData + GetBaseTagList + GetBaseTemplatePath + GetClientVariablesList + GetCurrentTemplatePath + GetDirectoryFromPath + GetException + GetFileFromPath + GetFunctionList + GetHttpRequestData + GetHttpTimeString + GetK2ServerDocCount + GetK2ServerDocCountLimit + GetLocale + GetMetaData + GetMetricData + GetPageContext + GetProfileSections + GetProfileString + GetServiceSettings + GetTempDirectory + GetTempFile + GetTemplatePath + GetTickCount + GetTimeZoneInfo + GetToken + Hash + Hour + HTMLCodeFormat + HTMLEditFormat + IIf + IncrementValue + InputBaseN + Insert + Int + IsArray + IsBinary + IsBoolean + IsCustomFunction + IsDate + IsDebugMode + IsDefined + IsK2ServerABroker + IsK2ServerDocCountExceeded + IsK2ServerOnline + IsLeapYear + IsNumeric + IsNumericDate + IsObject + IsQuery + IsSimpleValue + IsStruct + IsUserInRole + IsWDDX + IsXmlDoc + IsXmlElement + IsXmlRoot + JavaCast + JSStringFormat + LCase + Left + Len + ListAppend + ListChangeDelims + ListContains + ListContainsNoCase + ListDeleteAt + ListFind + ListFindNoCase + ListFirst + ListGetAt + ListInsertAt + ListLast + ListLen + ListPrepend + ListQualify + ListRest + ListSetAt + ListSort + ListToArray + ListValueCount + ListValueCountNoCase + LJustify + Log + Log10 + LSCurrencyFormat + LSDateFormat + LSEuroCurrencyFormat + LSIsCurrency + LSIsDate + LSIsNumeric + LSNumberFormat + LSParseCurrency + LSParseDateTime + LSParseEuroCurrency + LSParseNumber + LSTimeFormat + LTrim + Max + Mid + Min + Minute + Month + MonthAsString + Now + NumberFormat + ParagraphFormat + ParameterExists + ParseDateTime + Pi + PreserveSingleQuotes + Quarter + QueryAddColumn + QueryAddRow + QueryNew + QuerySetCell + QuotedValueList + Rand + Randomize + RandRange + REFind + REFindNoCase + RemoveChars + RepeatString + Replace + ReplaceList + ReplaceNoCase + REReplace + REReplaceNoCase + Reverse + Right + RJustify + Round + RTrim + Second + SetEncoding + SetLocale + SetProfileString + SetVariable + Sgn + Sin + SpanExcluding + SpanIncluding + Sqr + StripCR + StructAppend + StructClear + StructCopy + StructCount + StructDelete + StructFind + StructFindKey + StructFindValue + StructGet + StructInsert + StructIsEmpty + StructKeyArray + StructKeyExists + StructKeyList + StructNew + StructSort + StructUpdate + Tan + TimeFormat + ToBase64 + ToBinary + ToString + Trim + UCase + URLDecode + URLEncodedFormat + URLSessionFormat + Val + ValueList + Week + WriteOutput + XmlChildPos + XmlElemNew + XmlFormat + XmlNew + XmlParse + XmlSearch + XmlTransform + Year + YesNoFormat + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + BEGIN + BY + CASE + CLOSE + CONST + DO + ELSE + ELSIF + END + FOR + IF + IMPORT + LOOP + MODULE + NEW + OF + OUT + PROCEDURE + REPEAT + THEN + TO + TYPE + UNTIL + VAR + WHILE + WITH + + + ASSERT + EXIT + HALT + RETURN + + + ANYPTR + ANYREC + ARRAY + BOOLEAN + SHORTCHAR + CHAR + BYTE + SHORTINT + INTEGER + LONGINT + POINTER + RECORD + SHORTREAL + REAL + SET + + + ABSTRACT + EMPTY + EXTENSIBLE + LIMITED + + + ABS + ASH + BITS + CAP + CHR + DEC + ENTIER + EXCL + INC + INCL + LEN + LONG + MAX + MIN + ODD + ORD + SHORT + SIZE + + + FALSE + INF + NIL + TRUE + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/support.pyc b/tests/support.pyc deleted file mode 100644 index 8aff198..0000000 Binary files a/tests/support.pyc and /dev/null differ diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index 44a656b..b51d9b5 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -8,8 +8,8 @@ """ import os -import unittest import random +import unittest from pygments import lexers, formatters, filters, format from pygments.token import _TokenType, Text @@ -25,63 +25,192 @@ test_content = [chr(i) for i in xrange(33, 128)] * 5 random.shuffle(test_content) test_content = ''.join(test_content) + '\n' -class LexersTest(unittest.TestCase): - - def test_import_all(self): - # instantiate every lexer, to see if the token type defs are correct - for x in lexers.LEXERS.keys(): - c = getattr(lexers, x)() - - def test_lexer_classes(self): - a = self.assert_ - ae = self.assertEquals - # test that every lexer class has the correct public API - for lexer in lexers._iter_lexerclasses(): - a(type(lexer.name) is str) - for attr in 'aliases', 'filenames', 'alias_filenames', 'mimetypes': - a(hasattr(lexer, attr)) - a(type(getattr(lexer, attr)) is list, "%s: %s attribute wrong" % - (lexer, attr)) - result = lexer.analyse_text("abc") - a(isinstance(result, float) and 0.0 <= result <= 1.0) - - inst = lexer(opt1="val1", opt2="val2") - if issubclass(lexer, RegexLexer): - if not hasattr(lexer, '_tokens'): - # if there's no "_tokens", the lexer has to be one with - # multiple tokendef variants - a(lexer.token_variants) - for variant in lexer.tokens: - a('root' in lexer.tokens[variant]) - else: - a('root' in lexer._tokens, '%s has no root state' % lexer) - - tokens = list(inst.get_tokens(test_content)) - txt = "" - for token in tokens: - a(isinstance(token, tuple)) - a(isinstance(token[0], _TokenType)) - if isinstance(token[1], str): - print repr(token[1]) - a(isinstance(token[1], unicode)) - txt += token[1] - ae(txt, test_content, "%s lexer roundtrip failed: %r != %r" % - (lexer.name, test_content, txt)) - - def test_get_lexers(self): - a = self.assert_ - ae = self.assertEquals - # test that the lexers functions work - - for func, args in [(lexers.get_lexer_by_name, ("python",)), - (lexers.get_lexer_for_filename, ("test.py",)), - (lexers.get_lexer_for_mimetype, ("text/x-python",)), - (lexers.guess_lexer, ("#!/usr/bin/python -O\nprint",)), - (lexers.guess_lexer_for_filename, ("a.py", "<%= @foo %>")) - ]: - x = func(opt="val", *args) - a(isinstance(x, lexers.PythonLexer)) - ae(x.options["opt"], "val") + +def test_lexer_import_all(): + # instantiate every lexer, to see if the token type defs are correct + for x in lexers.LEXERS.keys(): + c = getattr(lexers, x)() + + +def test_lexer_classes(): + # test that every lexer class has the correct public API + def verify(cls): + assert type(cls.name) is str + for attr in 'aliases', 'filenames', 'alias_filenames', 'mimetypes': + assert hasattr(cls, attr) + assert type(getattr(cls, attr)) is list, \ + "%s: %s attribute wrong" % (cls, attr) + result = cls.analyse_text("abc") + assert isinstance(result, float) and 0.0 <= result <= 1.0 + + inst = cls(opt1="val1", opt2="val2") + if issubclass(cls, RegexLexer): + if not hasattr(cls, '_tokens'): + # if there's no "_tokens", the lexer has to be one with + # multiple tokendef variants + assert cls.token_variants + for variant in cls.tokens: + assert 'root' in cls.tokens[variant] + else: + assert 'root' in cls._tokens, \ + '%s has no root state' % cls + + tokens = list(inst.get_tokens(test_content)) + txt = "" + for token in tokens: + assert isinstance(token, tuple) + assert isinstance(token[0], _TokenType) + if isinstance(token[1], str): + print repr(token[1]) + assert isinstance(token[1], unicode) + txt += token[1] + assert txt == test_content, "%s lexer roundtrip failed: %r != %r" % \ + (cls.name, test_content, txt) + + for lexer in lexers._iter_lexerclasses(): + yield verify, lexer + + +def test_lexer_options(): + # test that the basic options work + def ensure(tokens, output): + concatenated = ''.join(token[1] for token in tokens) + assert concatenated == output, \ + '%s: %r != %r' % (lexer, concatenated, output) + def verify(cls): + inst = cls(stripnl=False) + ensure(inst.get_tokens('a\nb'), 'a\nb\n') + ensure(inst.get_tokens('\n\n\n'), '\n\n\n') + inst = cls(stripall=True) + ensure(inst.get_tokens(' \n b\n\n\n'), 'b\n') + # some lexers require full lines in input + if cls.__name__ not in ( + 'PythonConsoleLexer', 'RConsoleLexer', 'RubyConsoleLexer', + 'SqliteConsoleLexer', 'MatlabSessionLexer', 'ErlangShellLexer', + 'BashSessionLexer', 'LiterateHaskellLexer'): + inst = cls(ensurenl=False) + ensure(inst.get_tokens('a\nb'), 'a\nb') + inst = cls(ensurenl=False, stripall=True) + ensure(inst.get_tokens('a\nb\n\n'), 'a\nb') + + for lexer in lexers._iter_lexerclasses(): + if lexer.__name__ == 'RawTokenLexer': + # this one is special + continue + yield verify, lexer + + +def test_get_lexers(): + # test that the lexers functions work + def verify(func, args): + x = func(opt='val', *args) + assert isinstance(x, lexers.PythonLexer) + assert x.options["opt"] == "val" + + for func, args in [(lexers.get_lexer_by_name, ("python",)), + (lexers.get_lexer_for_filename, ("test.py",)), + (lexers.get_lexer_for_mimetype, ("text/x-python",)), + (lexers.guess_lexer, ("#!/usr/bin/python -O\nprint",)), + (lexers.guess_lexer_for_filename, ("a.py", "<%= @foo %>")) + ]: + yield verify, func, args + + +def test_formatter_public_api(): + ts = list(lexers.PythonLexer().get_tokens("def f(): pass")) + out = StringIO() + # test that every formatter class has the correct public API + def verify(formatter, info): + assert len(info) == 4 + assert info[0], "missing formatter name" + assert info[1], "missing formatter aliases" + assert info[3], "missing formatter docstring" + + if formatter.name == 'Raw tokens': + # will not work with Unicode output file + return + + try: + inst = formatter(opt1="val1") + except (ImportError, FontNotFound): + return + try: + inst.get_style_defs() + except NotImplementedError: + # may be raised by formatters for which it doesn't make sense + pass + inst.format(ts, out) + + for formatter, info in formatters.FORMATTERS.iteritems(): + yield verify, formatter, info + +def test_formatter_encodings(): + from pygments.formatters import HtmlFormatter + + # unicode output + fmt = HtmlFormatter() + tokens = [(Text, u"ä")] + out = format(tokens, fmt) + assert type(out) is unicode + assert u"ä" in out + + # encoding option + fmt = HtmlFormatter(encoding="latin1") + tokens = [(Text, u"ä")] + assert u"ä".encode("latin1") in format(tokens, fmt) + + # encoding and outencoding option + fmt = HtmlFormatter(encoding="latin1", outencoding="utf8") + tokens = [(Text, u"ä")] + assert u"ä".encode("utf8") in format(tokens, fmt) + + +def test_formatter_unicode_handling(): + # test that the formatter supports encoding and Unicode + tokens = list(lexers.PythonLexer(encoding='utf-8'). + get_tokens("def f(): 'ä'")) + + def verify(formatter): + try: + inst = formatter(encoding=None) + except (ImportError, FontNotFound): + # some dependency or font not installed + return + + if formatter.name != 'Raw tokens': + out = format(tokens, inst) + if formatter.unicodeoutput: + assert type(out) is unicode + + inst = formatter(encoding='utf-8') + out = format(tokens, inst) + assert type(out) is bytes, '%s: %r' % (formatter, out) + # Cannot test for encoding, since formatters may have to escape + # non-ASCII characters. + else: + inst = formatter() + out = format(tokens, inst) + assert type(out) is bytes, '%s: %r' % (formatter, out) + + for formatter, info in formatters.FORMATTERS.iteritems(): + yield verify, formatter + + +def test_get_formatters(): + # test that the formatters functions work + x = formatters.get_formatter_by_name("html", opt="val") + assert isinstance(x, formatters.HtmlFormatter) + assert x.options["opt"] == "val" + + x = formatters.get_formatter_for_filename("a.html", opt="val") + assert isinstance(x, formatters.HtmlFormatter) + assert x.options["opt"] == "val" + + +def test_styles(): + # minimal style test + from pygments.formatters import HtmlFormatter + fmt = HtmlFormatter(style="pastie") class FiltersTest(unittest.TestCase): @@ -136,95 +265,3 @@ class FiltersTest(unittest.TestCase): text = u'# DEBUG: text' tokens = list(lx.get_tokens(text)) self.assertEquals('# DEBUG: text', tokens[0][1]) - - -class FormattersTest(unittest.TestCase): - - def test_public_api(self): - a = self.assert_ - ae = self.assertEquals - ts = list(lexers.PythonLexer().get_tokens("def f(): pass")) - out = StringIO() - # test that every formatter class has the correct public API - for formatter, info in formatters.FORMATTERS.iteritems(): - a(len(info) == 4) - a(info[0], "missing formatter name") # name - a(info[1], "missing formatter aliases") # aliases - a(info[3], "missing formatter docstring") # doc - - if formatter.name == 'Raw tokens': - # will not work with Unicode output file - continue - - try: - inst = formatter(opt1="val1") - except (ImportError, FontNotFound): - continue - try: - inst.get_style_defs() - except NotImplementedError: - # may be raised by formatters for which it doesn't make sense - pass - inst.format(ts, out) - - def test_encodings(self): - from pygments.formatters import HtmlFormatter - - # unicode output - fmt = HtmlFormatter() - tokens = [(Text, u"ä")] - out = format(tokens, fmt) - self.assert_(type(out) is unicode) - self.assert_(u"ä" in out) - - # encoding option - fmt = HtmlFormatter(encoding="latin1") - tokens = [(Text, u"ä")] - self.assert_(u"ä".encode("latin1") in format(tokens, fmt)) - - # encoding and outencoding option - fmt = HtmlFormatter(encoding="latin1", outencoding="utf8") - tokens = [(Text, u"ä")] - self.assert_(u"ä".encode("utf8") in format(tokens, fmt)) - - def test_styles(self): - from pygments.formatters import HtmlFormatter - fmt = HtmlFormatter(style="pastie") - - def test_unicode_handling(self): - # test that the formatter supports encoding and Unicode - tokens = list(lexers.PythonLexer(encoding='utf-8'). - get_tokens("def f(): 'ä'")) - for formatter, info in formatters.FORMATTERS.iteritems(): - try: - inst = formatter(encoding=None) - except (ImportError, FontNotFound): - # some dependency or font not installed - continue - - if formatter.name != 'Raw tokens': - out = format(tokens, inst) - if formatter.unicodeoutput: - self.assert_(type(out) is unicode) - - inst = formatter(encoding='utf-8') - out = format(tokens, inst) - self.assert_(type(out) is bytes, '%s: %r' % (formatter, out)) - # Cannot test for encoding, since formatters may have to escape - # non-ASCII characters. - else: - inst = formatter() - out = format(tokens, inst) - self.assert_(type(out) is bytes, '%s: %r' % (formatter, out)) - - def test_get_formatters(self): - a = self.assert_ - ae = self.assertEquals - # test that the formatters functions work - x = formatters.get_formatter_by_name("html", opt="val") - a(isinstance(x, formatters.HtmlFormatter)) - ae(x.options["opt"], "val") - - x = formatters.get_formatter_for_filename("a.html", opt="val") - a(isinstance(x, formatters.HtmlFormatter)) - ae(x.options["opt"], "val") diff --git a/tests/test_basic_api.pyc b/tests/test_basic_api.pyc deleted file mode 100644 index 7369726..0000000 Binary files a/tests/test_basic_api.pyc and /dev/null differ diff --git a/tests/test_clexer.pyc b/tests/test_clexer.pyc deleted file mode 100644 index 57e6655..0000000 Binary files a/tests/test_clexer.pyc and /dev/null differ diff --git a/tests/test_cmdline.pyc b/tests/test_cmdline.pyc deleted file mode 100644 index e44c808..0000000 Binary files a/tests/test_cmdline.pyc and /dev/null differ diff --git a/tests/test_examplefiles.py b/tests/test_examplefiles.py index d56eb7c..691ae92 100644 --- a/tests/test_examplefiles.py +++ b/tests/test_examplefiles.py @@ -8,9 +8,7 @@ """ import os -import unittest -from pygments import highlight from pygments.lexers import get_lexer_for_filename, get_lexer_by_name from pygments.token import Error from pygments.util import ClassNotFound, b diff --git a/tests/test_examplefiles.pyc b/tests/test_examplefiles.pyc deleted file mode 100644 index 9e7f25c..0000000 Binary files a/tests/test_examplefiles.pyc and /dev/null differ diff --git a/tests/test_html_formatter.py b/tests/test_html_formatter.py index 9ff593f..ae54b91 100644 --- a/tests/test_html_formatter.py +++ b/tests/test_html_formatter.py @@ -12,17 +12,19 @@ import re import unittest import StringIO import tempfile -from os.path import join, dirname, isfile, abspath +from os.path import join, dirname, isfile from pygments.lexers import PythonLexer from pygments.formatters import HtmlFormatter, NullFormatter from pygments.formatters.html import escape_html +from pygments.util import uni_open import support TESTFILE, TESTDIR = support.location(__file__) -tokensource = list(PythonLexer(encoding='utf-8').get_tokens(open(TESTFILE).read())) +tokensource = list(PythonLexer().get_tokens( + uni_open(TESTFILE, encoding='utf-8').read())) class HtmlFormatterTest(unittest.TestCase): diff --git a/tests/test_html_formatter.pyc b/tests/test_html_formatter.pyc deleted file mode 100644 index 4776a45..0000000 Binary files a/tests/test_html_formatter.pyc and /dev/null differ diff --git a/tests/test_latex_formatter.pyc b/tests/test_latex_formatter.pyc deleted file mode 100644 index de8f73d..0000000 Binary files a/tests/test_latex_formatter.pyc and /dev/null differ diff --git a/tests/test_regexlexer.pyc b/tests/test_regexlexer.pyc deleted file mode 100644 index 2bcf10d..0000000 Binary files a/tests/test_regexlexer.pyc and /dev/null differ diff --git a/tests/test_token.pyc b/tests/test_token.pyc deleted file mode 100644 index 30c8d69..0000000 Binary files a/tests/test_token.pyc and /dev/null differ diff --git a/tests/test_using_api.pyc b/tests/test_using_api.pyc deleted file mode 100644 index d070c23..0000000 Binary files a/tests/test_using_api.pyc and /dev/null differ diff --git a/tests/test_util.pyc b/tests/test_util.pyc deleted file mode 100644 index ea024d1..0000000 Binary files a/tests/test_util.pyc and /dev/null differ