From: DongHun Kwak Date: Tue, 11 Dec 2018 07:07:59 +0000 (+0900) Subject: Imported Upstream version 2.0.0 X-Git-Tag: upstream/2.0.0 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=refs%2Fchanges%2F59%2F195159%2F1;p=platform%2Fupstream%2Fitstool.git Imported Upstream version 2.0.0 Change-Id: Ie947764e1dfd452834808351cf09e25668007f4d Signed-off-by: DongHun Kwak --- diff --git a/ChangeLog b/ChangeLog index 9efad89..a79976f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,1069 @@ +commit 17a89300affeac556803c23eefefc9f279a82908 +Author: Shaun McCance +Date: Fri Nov 1 11:53:28 2013 -0400 + + docbook*.its: Make info children always not within text + + Some of the children are marked within text because they should be + when appearing in a para. But letting info appear as if it were a + text run hits some strange corner cases that cause locale filter + not to be applied as you might think it should. + + its/docbook.its | 3 +++ + its/docbook5.its | 3 +++ + 2 files changed, 6 insertions(+), 0 deletions(-) + +commit 8e4ccd41ad198ebc816a43419c21714bad7dc748 +Author: Shaun McCance +Date: Fri Nov 1 10:46:21 2013 -0400 + + its: Exclude editor remarks/comments with locale filter + + its/docbook.its | 4 ++-- + its/docbook5.its | 4 ++-- + its/mallard.its | 3 ++- + 3 files changed, 6 insertions(+), 5 deletions(-) + +commit 770a84a13e30f5e5d8218ae495b430d0c3002f56 +Author: Shaun McCance +Date: Fri Nov 1 10:14:42 2013 -0400 + + itstool.in: Allow users to set ITS params + + its/its.its | 1 + + itstool.in | 62 ++++++++++++++++++++++++++++++++++++---------------------- + 2 files changed, 39 insertions(+), 24 deletions(-) + +commit 6ec6088089430eb5e1cd5a2aa82eef3161b11487 +Author: Shaun McCance +Date: Thu Oct 31 14:16:54 2013 -0400 + + its: Switched built-in ITS rules to 2.0 + + its/its.its | 2 +- + its/mallard.its | 2 +- + its/ttml.its | 2 +- + its/xhtml.its | 2 +- + 4 files changed, 4 insertions(+), 4 deletions(-) + +commit 7486a069a39b97d84fb76ff85286b40f4c8e5af0 +Author: Shaun McCance +Date: Thu Oct 31 14:16:31 2013 -0400 + + docbook5.its: Added DocBook 5 support + + Also minor fixed to DocBook 4 I ran across + + its/Makefile.am | 2 +- + its/docbook.its | 7 ++- + its/docbook5.its | 201 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ + 3 files changed, 207 insertions(+), 3 deletions(-) + +commit 844bf80669ffe05ad13c5ce540f4c32d6ed0bb31 +Author: Shaun McCance +Date: Thu Oct 31 09:59:11 2013 -0400 + + docbook.its: Updated preserve space and external resource + + its/docbook.its | 19 +++++++++---------- + 1 files changed, 9 insertions(+), 10 deletions(-) + +commit 95d331264b29d4ad13cd661d2d3d84e2ef09e7ab +Author: Shaun McCance +Date: Thu Oct 31 09:54:44 2013 -0400 + + mallard.its: Updated preserve space and external resource + + its/mallard.its | 6 +++--- + 1 files changed, 3 insertions(+), 3 deletions(-) + +commit 4af2d64a7987ef2dc4121afc1bb6ebf8077230b3 +Author: Shaun McCance +Date: Thu Oct 31 09:49:54 2013 -0400 + + xhtml.its: Updated preserve space and external resource + + its/xhtml.its | 5 ++++- + 1 files changed, 4 insertions(+), 1 deletions(-) + +commit d657543826789e263ac9c8861a722f2527b5b0a3 +Merge: e338a3e 2928d6f +Author: Shaun McCance +Date: Mon Oct 28 12:05:03 2013 -0400 + + Merge branch 'master' into its-2-0 + +commit e338a3e4463d6f8098ef76c45d44851fe62cb126 +Author: Shaun McCance +Date: Mon Oct 28 11:06:00 2013 -0400 + + Support localeFilterType="exclude" + + itstool.in | 64 ++++++++++++++++++++++------- + tests/ITS-2.0-Testsuite/run_tests.sh | 18 ++------ + tests/LocaleFilter/Locale1Xml.fr_CA.po | 4 +- + tests/LocaleFilter/Locale1Xml.fr_CH.po | 4 +- + tests/LocaleFilter/Locale1Xml.fr_FR.po | 4 +- + tests/LocaleFilter/Locale1Xml.pot | 4 +- + tests/LocaleFilter/Locale2Xml.fr_CA.po | 4 +- + tests/LocaleFilter/Locale2Xml.fr_CH.po | 4 +- + tests/LocaleFilter/Locale2Xml.fr_FR.po | 4 +- + tests/LocaleFilter/Locale2Xml.pot | 4 +- + tests/LocaleFilter/Locale3Xml.fr_CA.po | 4 +- + tests/LocaleFilter/Locale3Xml.fr_CH.po | 4 +- + tests/LocaleFilter/Locale3Xml.fr_FR.po | 4 +- + tests/LocaleFilter/Locale3Xml.pot | 4 +- + tests/LocaleFilter/Locale4Xml.fr_CA.po | 4 +- + tests/LocaleFilter/Locale4Xml.fr_CH.po | 4 +- + tests/LocaleFilter/Locale4Xml.fr_FR.po | 4 +- + tests/LocaleFilter/Locale4Xml.pot | 4 +- + tests/LocaleFilter/Locale5Xml.fr_CA.po | 4 +- + tests/LocaleFilter/Locale5Xml.fr_CH.po | 4 +- + tests/LocaleFilter/Locale5Xml.fr_FR.po | 4 +- + tests/LocaleFilter/Locale5Xml.pot | 4 +- + tests/LocaleFilter/Locale6Xml.fr_CA.po | 22 ++++++++++ + tests/LocaleFilter/Locale6Xml.fr_CA.xml | 13 ++++++ + tests/LocaleFilter/Locale6Xml.fr_CH.po | 23 +++++++++++ + tests/LocaleFilter/Locale6Xml.fr_CH.xml | 13 ++++++ + tests/LocaleFilter/Locale6Xml.fr_FR.po | 22 ++++++++++ + tests/LocaleFilter/Locale6Xml.fr_FR.xml | 13 ++++++ + tests/LocaleFilter/Locale6Xml.joined.xml | 22 ++++++++++ + tests/LocaleFilter/Locale6Xml.pot | 23 +++++++++++ + tests/LocaleFilter/Locale6Xml.xml | 20 +++++++++ + tests/run_tests.py | 16 +++++++ + 32 files changed, 279 insertions(+), 70 deletions(-) + +commit 9f85a74071db1fa5ca3a2dd83a3ab187884fb7f7 +Author: Shaun McCance +Date: Sun Oct 27 22:36:56 2013 -0400 + + Update ITS 2.0 test suite + + .../elementswithintext/html/withintext1html.html | 64 ++++++++-------- + .../html/withintext1htmlrules.xml | 8 +- + .../elementswithintext/html/withintext2html.html | 20 +++--- + .../elementswithintext/html/withintext3html.html | 64 ++++++++-------- + .../html/withintext3htmlrules.xml | 10 +- + .../elementswithintext/html/withintext4html.html | 76 ++++++++++---------- + .../elementswithintext/xml/withintext1xml.xml | 24 +++--- + .../elementswithintext/xml/withintext2xml.xml | 50 ++++++------ + .../elementswithintext/xml/withintext2xmlrules.xml | 12 ++-- + .../elementswithintext/xml/withintext3xml.xml | 22 +++--- + .../elementswithintext/xml/withintext4xml.xml | 24 +++--- + .../elementswithintext/xml/withintext5xml.xml | 50 ++++++------ + .../elementswithintext/xml/withintext6xml.xml | 42 +++++----- + .../elementswithintext/xml/withintext6xmlrules.xml | 10 +- + .../html/externalresource1html.html | 30 ++++---- + .../html/externalresource1htmlrules.xml | 8 +- + .../html/externalresource2html.html | 34 ++++---- + .../html/externalresource2htmlrules.xml | 8 +- + .../html/externalresource3html.html | 40 +++++----- + .../externalresource/xml/externalresource1xml.xml | 48 ++++++------ + .../externalresource/xml/externalresource2xml.xml | 44 ++++++------ + .../xml/externalresource2xmlrules.xml | 6 +- + .../externalresource/xml/externalresource3xml.xml | 56 +++++++------- + .../xml/externalresource3xmlrules.xml | 6 +- + .../externalresource/xml/externalresource4xml.xml | 50 ++++++------ + .../externalresource/xml/externalresource5xml.xml | 40 +++++----- + .../xml/externalresource5xmlrules.xml | 10 +- + .../inputdata/idvalue/html/idvalue1html.html | 32 ++++---- + .../inputdata/idvalue/html/idvalue1htmlrules.xml | 12 ++-- + .../inputdata/idvalue/html/idvalue2html.html | 34 ++++---- + .../inputdata/idvalue/html/idvalue2htmlrules.xml | 14 ++-- + .../inputdata/idvalue/html/idvalue3html.html | 44 ++++++------ + .../inputdata/idvalue/xml/idvalue1xml.xml | 40 +++++----- + .../inputdata/idvalue/xml/idvalue2xml.xml | 8 +- + .../inputdata/idvalue/xml/idvalue3xml.xml | 26 +++--- + .../inputdata/idvalue/xml/idvalue3xmlrules.xml | 6 +- + .../inputdata/idvalue/xml/idvalue4xml.xml | 34 ++++---- + .../inputdata/idvalue/xml/idvalue4xmlrules.xml | 8 +- + .../inputdata/idvalue/xml/idvalue5xml.xml | 40 +++++----- + .../inputdata/localefilter/html/locale1html.html | 28 ++++---- + .../localefilter/html/locale1htmlrules.xml | 6 +- + .../inputdata/localefilter/html/locale2html.html | 24 +++--- + .../inputdata/localefilter/html/locale3html.html | 28 ++++---- + .../localefilter/html/locale3htmlrules.xml | 8 +- + .../inputdata/localefilter/html/locale4html.html | 36 +++++----- + .../inputdata/localefilter/html/locale5html.html | 42 +++++----- + .../inputdata/localefilter/xml/locale1xml.xml | 24 +++--- + .../inputdata/localefilter/xml/locale2xml.xml | 18 ++-- + .../inputdata/localefilter/xml/locale3xml.xml | 20 +++--- + .../inputdata/localefilter/xml/locale3xmlrules.xml | 6 +- + .../inputdata/localefilter/xml/locale4xml.xml | 32 ++++---- + .../inputdata/localefilter/xml/locale4xmlrules.xml | 6 +- + .../inputdata/localefilter/xml/locale5xml.xml | 16 ++-- + .../inputdata/localefilter/xml/locale6xml.xml | 26 +++--- + .../inputdata/localefilter/xml/locale7xml.xml | 26 +++--- + .../inputdata/localefilter/xml/locale7xmlrules.xml | 8 +- + .../inputdata/localefilter/xml/locale8xml.xml | 36 +++++----- + .../localizationnote/html/locnote1html.html | 22 +++--- + .../localizationnote/html/locnote1htmlrules.xml | 14 ++-- + .../localizationnote/html/locnote2html.html | 40 +++++----- + .../localizationnote/html/locnote2htmlrules.xml | 8 +- + .../localizationnote/html/locnote3html.html | 24 +++--- + .../localizationnote/html/locnote3htmlrules.xml | 6 +- + .../localizationnote/html/locnote4html.html | 32 ++++---- + .../localizationnote/html/locnote4htmlrules.xml | 6 +- + .../localizationnote/html/locnote5html.html | 42 +++++----- + .../localizationnote/html/locnote5htmlrules.xml | 30 ++++---- + .../localizationnote/html/locnote6html.html | 42 +++++----- + .../localizationnote/html/locnote6htmlrules.xml | 30 ++++---- + .../localizationnote/html/locnote7html.html | 20 +++--- + .../localizationnote/html/locnote8html.html | 42 +++++----- + .../localizationnote/html/locnote8htmlrules.xml | 32 ++++---- + .../localizationnote/html/locnote9html.html | 40 +++++----- + .../localizationnote/xml/locnote10xml.xml | 66 ++++++++-------- + .../localizationnote/xml/locnote11xml.xml | 36 +++++----- + .../localizationnote/xml/locnote11xmlrules.xml | 32 ++++---- + .../inputdata/localizationnote/xml/locnote1xml.xml | 26 +++--- + .../inputdata/localizationnote/xml/locnote2xml.xml | 38 +++++----- + .../inputdata/localizationnote/xml/locnote3xml.xml | 24 +++--- + .../inputdata/localizationnote/xml/locnote4xml.xml | 32 ++++---- + .../inputdata/localizationnote/xml/locnote5xml.xml | 30 ++++---- + .../localizationnote/xml/locnote5xmlrules.xml | 18 ++-- + .../inputdata/localizationnote/xml/locnote6xml.xml | 42 +++++----- + .../inputdata/localizationnote/xml/locnote7xml.xml | 40 +++++----- + .../inputdata/localizationnote/xml/locnote8xml.xml | 34 ++++---- + .../inputdata/localizationnote/xml/locnote9xml.xml | 16 ++-- + .../preservespace/xml/preservespace1xml.xml | 26 +++--- + .../preservespace/xml/preservespace2xml.xml | 16 ++-- + .../preservespace/xml/preservespace3xml.xml | 22 +++--- + .../preservespace/xml/preservespace3xmlrules.xml | 6 +- + .../preservespace/xml/preservespace4xml.xml | 16 ++-- + .../preservespace/xml/preservespace5xml.xml | 28 ++++---- + .../preservespace/xml/preservespace6xml.xml | 18 ++-- + .../preservespace/xml/preservespace6xmlrules.xml | 8 +- + .../inputdata/translate/html/translate1html.html | 34 ++++---- + .../translate/html/translate1htmlrules.xml | 8 +- + .../inputdata/translate/html/translate2html.html | 36 +++++----- + .../inputdata/translate/html/translate3html.html | 46 ++++++------ + .../translate/html/translate3htmlrules.xml | 8 +- + .../inputdata/translate/html/translate4html.html | 46 ++++++------ + .../translate/html/translate4htmlrules.xml | 8 +- + .../inputdata/translate/html/translate5html.html | 68 +++++++++--------- + .../translate/html/translate5htmlrules.xml | 8 +- + .../inputdata/translate/html/translate6html.html | 68 +++++++++--------- + .../translate/html/translate6htmlrules.xml | 12 ++-- + .../inputdata/translate/html/translate7html.html | 38 +++++----- + .../inputdata/translate/xml/translate10xml.xml | 14 ++-- + .../translate/xml/translate10xmlrules.xml | 10 +- + .../inputdata/translate/xml/translate1xml.xml | 78 ++++++++++---------- + .../inputdata/translate/xml/translate2xml.xml | 20 +++--- + .../inputdata/translate/xml/translate2xmlrules.xml | 10 +- + .../inputdata/translate/xml/translate3xml.xml | 26 +++--- + .../inputdata/translate/xml/translate3xmlrules.xml | 12 ++-- + .../inputdata/translate/xml/translate4xml.xml | 22 +++--- + .../inputdata/translate/xml/translate5xml.xml | 26 +++--- + .../inputdata/translate/xml/translate6xml.xml | 38 +++++----- + .../inputdata/translate/xml/translate7xml.xml | 38 +++++----- + .../inputdata/translate/xml/translate8xml.xml | 56 +++++++------- + .../inputdata/translate/xml/translate9xml.xml | 22 +++--- + 119 files changed, 1650 insertions(+), 1650 deletions(-) + +commit dde5a9ac006ab839b67e711e27977fbf074dc04e +Author: Shaun McCance +Date: Sun Oct 27 21:02:46 2013 -0400 + + Add support for its:param + + itstool.in | 12 +++++++++++- + tests/ITS-2.0-Testsuite/run_tests.sh | 16 +--------------- + 2 files changed, 12 insertions(+), 16 deletions(-) + +commit 45872e7e4252aadd0fc304ab3c9d1d740c690b5f +Author: Shaun McCance +Date: Sun Oct 27 13:45:12 2013 -0400 + + Fixed issue with preserveSpace from xml:space attribute + + itstool.in | 15 +++++++-------- + 1 files changed, 7 insertions(+), 8 deletions(-) + +commit 29d6a9ea85c8fb939b4fdaabb92e08eb42bfdefc +Author: Shaun McCance +Date: Sun Oct 27 13:45:00 2013 -0400 + + Updated ITS 2.0 test suite + + .../html/withintext1htmloutput.txt | 14 +++++--- + .../html/withintext3htmloutput.txt | 12 ++++--- + .../html/withintext4htmloutput.txt | 16 +++++---- + .../html/externalresource1htmloutput.txt | 7 ++-- + .../html/externalresource2htmloutput.txt | 21 ++++++------ + .../html/externalresource3htmloutput.txt | 7 ++-- + .../localefilter/html/locale1htmloutput.txt | 24 +++++++------- + .../localefilter/html/locale2htmloutput.txt | 20 ++++++------ + .../localefilter/html/locale3htmloutput.txt | 24 +++++++------- + .../localefilter/html/locale4htmloutput.txt | 22 ++++++------ + .../localefilter/html/locale5htmloutput.txt | 21 ++++++++++++ + .../expected/localefilter/xml/locale1xmloutput.txt | 22 ++++++------ + .../expected/localefilter/xml/locale2xmloutput.txt | 12 +++--- + .../expected/localefilter/xml/locale3xmloutput.txt | 18 +++++----- + .../expected/localefilter/xml/locale4xmloutput.txt | 34 ++++++++++---------- + .../expected/localefilter/xml/locale5xmloutput.txt | 12 +++--- + .../expected/localefilter/xml/locale6xmloutput.txt | 30 +++++++++--------- + .../expected/localefilter/xml/locale7xmloutput.txt | 24 +++++++------- + .../expected/localefilter/xml/locale8xmloutput.txt | 19 +++++++++++ + .../localizationnote/xml/locnote1xmloutput.txt | 1 - + .../translate/html/translate1htmloutput.txt | 7 +++- + .../translate/html/translate2htmloutput.txt | 6 +++ + .../translate/html/translate3htmloutput.txt | 1 + + .../translate/html/translate7htmloutput.txt | 4 ++- + .../expected/translate/xml/translate4xmloutput.txt | 5 +++ + .../elementswithintext/html/withintext1html.html | 12 +++--- + .../html/withintext1htmlrules.xml | 2 - + .../elementswithintext/html/withintext3html.html | 14 ++++---- + .../html/withintext3htmlrules.xml | 5 +-- + .../elementswithintext/html/withintext4html.html | 14 +++----- + .../html/externalresource1html.html | 4 +-- + .../html/externalresource2html.html | 8 ++--- + .../html/externalresource2htmlrules.xml | 2 +- + .../html/externalresource3html.html | 4 +-- + .../inputdata/idvalue/html/idvalue2htmlrules.xml | 9 ++--- + .../inputdata/localefilter/html/locale5html.html | 21 ++++++++++++ + .../inputdata/localefilter/xml/locale8xml.xml | 18 ++++++++++ + .../inputdata/localizationnote/xml/locnote1xml.xml | 3 +- + .../inputdata/translate/html/translate1html.html | 1 + + .../translate/html/translate1htmlrules.xml | 2 +- + .../inputdata/translate/html/translate2html.html | 3 +- + .../inputdata/translate/html/translate3html.html | 2 +- + .../inputdata/translate/html/translate7html.html | 4 +- + .../inputdata/translate/xml/translate4xml.xml | 1 + + tests/ITS-2.0-Testsuite/run_tests.sh | 8 +++++ + 45 files changed, 309 insertions(+), 211 deletions(-) + +commit 2928d6f02a0f30415bd993490d1920fd990ce130 +Author: Shaun McCance +Date: Sat Sep 21 16:24:35 2013 -0400 + + Added an option to retain entity references + + You still have to load the DTD if the entities are defined in + the external subset, because libxml2 checks references even if + it doesn't dereference them. It would be nice if this weren't + necessary. + + itstool.in | 32 +++++++++++++++++++++++++++----- + tests/IT-keep-entities-1.ll.po | 21 +++++++++++++++++++++ + tests/IT-keep-entities-1.ll.xml | 7 +++++++ + tests/IT-keep-entities-1.pot | 21 +++++++++++++++++++++ + tests/IT-keep-entities-1.xml | 7 +++++++ + tests/IT-keep-entities-2.ll.po | 21 +++++++++++++++++++++ + tests/IT-keep-entities-2.ll.xml | 9 +++++++++ + tests/IT-keep-entities-2.pot | 21 +++++++++++++++++++++ + tests/IT-keep-entities-2.xml | 9 +++++++++ + tests/run_tests.py | 20 ++++++++++++++------ + 10 files changed, 157 insertions(+), 11 deletions(-) + +commit 309b2cee105430c5ed387d7862e9405d41e76088 +Author: Shaun McCance +Date: Sat Sep 21 12:17:12 2013 -0400 + + Fix utf8 issue introduced in Sep 20 commit + + itstool.in | 10 ++++++---- + 1 files changed, 6 insertions(+), 4 deletions(-) + +commit cf30d00a9d4802f9d7479d28a4a3ce420bc36aaf +Merge: e4c6ada c9dd17a +Author: Shaun McCance +Date: Wed Aug 21 09:50:32 2013 -0400 + + Merge commit 'refs/merge-requests/5' of gitorious.org:itstool/itstool + +commit e4c6adab00c64141483b322d482a3819968b700e +Merge: 5040a32 34cc26b +Author: Shaun McCance +Date: Wed Aug 21 09:45:39 2013 -0400 + + Merge commit 'refs/merge-requests/4' of gitorious.org:itstool/itstool + +commit c9dd17a6504339eb712e0688b68de01566792b7a +Author: Clement Chauplannaz +Date: Sun Jun 9 16:52:29 2013 +0200 + + Configure: test for python module libxml2 presence + + configure.ac | 10 ++++++++++ + 1 files changed, 10 insertions(+), 0 deletions(-) + +commit ab2f5c7430c640cef0dddb7991dc9cfb77284332 +Author: Shaun McCance +Date: Sat Feb 9 23:03:47 2013 -0500 + + Properly handle loc note inheritance + + itstool.in | 11 +++++++---- + tests/ITS-2.0-Testsuite/run_tests.sh | 5 ----- + 2 files changed, 7 insertions(+), 9 deletions(-) + +commit 2f77297d840b27a3b294fa098c4a9fb20130a954 +Author: Shaun McCance +Date: Sat Feb 9 22:43:07 2013 -0500 + + Add LocNote class to better track localization note info + + itstool.in | 69 ++++++++++++++++++++++++--------- + tests/ITS-2.0-Testsuite/run_tests.sh | 5 -- + 2 files changed, 50 insertions(+), 24 deletions(-) + +commit 64121755e4c515f184acd0f99bab20798a865178 +Author: Shaun McCance +Date: Sat Feb 9 13:51:37 2013 -0500 + + Fix IdValue for attributes and nodes with attributes + + itstool.in | 2 +- + 1 files changed, 1 insertions(+), 1 deletions(-) + +commit a6becc8bbc4ce20217ba6a8bd11016297db2e607 +Author: Shaun McCance +Date: Sat Feb 2 14:15:56 2013 -0500 + + Adding the ITS 2.0 test suite, found here: + + https://github.com/finnle/ITS-2.0-Testsuite/commits/master + + itstool.in | 33 ++++++++--- + tests/ITS-2.0-Testsuite/README | 1 + + .../html/withintext1htmloutput.txt | 24 ++++++++ + .../html/withintext2htmloutput.txt | 10 +++ + .../html/withintext3htmloutput.txt | 24 ++++++++ + .../html/withintext4htmloutput.txt | 24 ++++++++ + .../xml/withintext1xmloutput.txt | 17 +++++ + .../xml/withintext2xmloutput.txt | 28 +++++++++ + .../xml/withintext3xmloutput.txt | 15 +++++ + .../xml/withintext4xmloutput.txt | 15 +++++ + .../xml/withintext5xmloutput.txt | 20 ++++++ + .../xml/withintext6xmloutput.txt | 13 ++++ + .../html/externalresource1htmloutput.txt | 19 ++++++ + .../html/externalresource2htmloutput.txt | 21 +++++++ + .../html/externalresource3htmloutput.txt | 18 ++++++ + .../xml/externalresource1xmloutput.txt | 17 +++++ + .../xml/externalresource2xmloutput.txt | 15 +++++ + .../xml/externalresource3xmloutput.txt | 23 +++++++ + .../xml/externalresource4xmloutput.txt | 19 ++++++ + .../xml/externalresource5xmloutput.txt | 15 +++++ + .../expected/idvalue/html/idvalue1htmloutput.txt | 15 +++++ + .../expected/idvalue/html/idvalue2htmloutput.txt | 17 +++++ + .../expected/idvalue/html/idvalue3htmloutput.txt | 14 ++++ + .../expected/idvalue/xml/idvalue1xmloutput.txt | 18 ++++++ + .../expected/idvalue/xml/idvalue2xmloutput.txt | 6 ++ + .../expected/idvalue/xml/idvalue3xmloutput.txt | 14 ++++ + .../expected/idvalue/xml/idvalue4xmloutput.txt | 18 ++++++ + .../expected/idvalue/xml/idvalue5xmloutput.txt | 22 +++++++ + .../localefilter/html/locale1htmloutput.txt | 12 ++++ + .../localefilter/html/locale2htmloutput.txt | 10 +++ + .../localefilter/html/locale3htmloutput.txt | 12 ++++ + .../localefilter/html/locale4htmloutput.txt | 11 ++++ + .../expected/localefilter/xml/locale1xmloutput.txt | 11 ++++ + .../expected/localefilter/xml/locale2xmloutput.txt | 6 ++ + .../expected/localefilter/xml/locale3xmloutput.txt | 9 +++ + .../expected/localefilter/xml/locale4xmloutput.txt | 17 +++++ + .../expected/localefilter/xml/locale5xmloutput.txt | 6 ++ + .../expected/localefilter/xml/locale6xmloutput.txt | 15 +++++ + .../expected/localefilter/xml/locale7xmloutput.txt | 12 ++++ + .../localizationnote/html/locnote1htmloutput.txt | 11 ++++ + .../localizationnote/html/locnote2htmloutput.txt | 17 +++++ + .../localizationnote/html/locnote3htmloutput.txt | 11 ++++ + .../localizationnote/html/locnote4htmloutput.txt | 17 +++++ + .../localizationnote/html/locnote5htmloutput.txt | 21 +++++++ + .../localizationnote/html/locnote6htmloutput.txt | 21 +++++++ + .../localizationnote/html/locnote7htmloutput.txt | 11 ++++ + .../localizationnote/html/locnote8htmloutput.txt | 21 +++++++ + .../localizationnote/html/locnote9htmloutput.txt | 11 ++++ + .../localizationnote/xml/locnote10xmloutput.txt | 31 ++++++++++ + .../localizationnote/xml/locnote11xmloutput.txt | 19 ++++++ + .../localizationnote/xml/locnote1xmloutput.txt | 12 ++++ + .../localizationnote/xml/locnote2xmloutput.txt | 20 ++++++ + .../localizationnote/xml/locnote3xmloutput.txt | 11 ++++ + .../localizationnote/xml/locnote4xmloutput.txt | 17 +++++ + .../localizationnote/xml/locnote5xmloutput.txt | 23 +++++++ + .../localizationnote/xml/locnote6xmloutput.txt | 29 +++++++++ + .../localizationnote/xml/locnote7xmloutput.txt | 20 ++++++ + .../localizationnote/xml/locnote8xmloutput.txt | 20 ++++++ + .../localizationnote/xml/locnote9xmloutput.txt | 11 ++++ + .../preservespace/xml/preservespace1xmloutput.txt | 9 +++ + .../preservespace/xml/preservespace2xmloutput.txt | 4 + + .../preservespace/xml/preservespace3xmloutput.txt | 7 ++ + .../preservespace/xml/preservespace4xmloutput.txt | 4 + + .../preservespace/xml/preservespace5xmloutput.txt | 12 ++++ + .../preservespace/xml/preservespace6xmloutput.txt | 8 +++ + .../translate/html/translate1htmloutput.txt | 13 ++++ + .../translate/html/translate2htmloutput.txt | 11 ++++ + .../translate/html/translate3htmloutput.txt | 15 +++++ + .../translate/html/translate4htmloutput.txt | 15 +++++ + .../translate/html/translate5htmloutput.txt | 24 ++++++++ + .../translate/html/translate6htmloutput.txt | 24 ++++++++ + .../translate/html/translate7htmloutput.txt | 12 ++++ + .../translate/xml/translate10xmloutput.txt | 17 +++++ + .../expected/translate/xml/translate1xmloutput.txt | 59 ++++++++++++++++++ + .../expected/translate/xml/translate2xmloutput.txt | 11 ++++ + .../expected/translate/xml/translate3xmloutput.txt | 17 +++++ + .../expected/translate/xml/translate4xmloutput.txt | 10 +++ + .../expected/translate/xml/translate5xmloutput.txt | 11 ++++ + .../expected/translate/xml/translate6xmloutput.txt | 22 +++++++ + .../expected/translate/xml/translate7xmloutput.txt | 22 +++++++ + .../expected/translate/xml/translate8xmloutput.txt | 28 +++++++++ + .../expected/translate/xml/translate9xmloutput.txt | 23 +++++++ + .../elementswithintext/html/withintext1html.html | 32 ++++++++++ + .../html/withintext1htmlrules.xml | 6 ++ + .../elementswithintext/html/withintext2html.html | 10 +++ + .../elementswithintext/html/withintext3html.html | 32 ++++++++++ + .../html/withintext3htmlrules.xml | 6 ++ + .../elementswithintext/html/withintext4html.html | 40 ++++++++++++ + .../elementswithintext/xml/withintext1xml.xml | 12 ++++ + .../elementswithintext/xml/withintext2xml.xml | 25 ++++++++ + .../elementswithintext/xml/withintext2xmlrules.xml | 6 ++ + .../elementswithintext/xml/withintext3xml.xml | 11 ++++ + .../elementswithintext/xml/withintext4xml.xml | 12 ++++ + .../elementswithintext/xml/withintext5xml.xml | 25 ++++++++ + .../elementswithintext/xml/withintext6xml.xml | 21 +++++++ + .../elementswithintext/xml/withintext6xmlrules.xml | 5 ++ + .../html/externalresource1html.html | 17 +++++ + .../html/externalresource1htmlrules.xml | 4 + + .../html/externalresource2html.html | 19 ++++++ + .../html/externalresource2htmlrules.xml | 4 + + .../html/externalresource3html.html | 22 +++++++ + .../externalresource/xml/externalresource1xml.xml | 24 ++++++++ + .../externalresource/xml/externalresource2xml.xml | 22 +++++++ + .../xml/externalresource2xmlrules.xml | 3 + + .../externalresource/xml/externalresource3xml.xml | 28 +++++++++ + .../xml/externalresource3xmlrules.xml | 3 + + .../externalresource/xml/externalresource4xml.xml | 25 ++++++++ + .../externalresource/xml/externalresource5xml.xml | 20 ++++++ + .../xml/externalresource5xmlrules.xml | 5 ++ + .../inputdata/idvalue/html/idvalue1html.html | 16 +++++ + .../inputdata/idvalue/html/idvalue1htmlrules.xml | 6 ++ + .../inputdata/idvalue/html/idvalue2html.html | 17 +++++ + .../inputdata/idvalue/html/idvalue2htmlrules.xml | 8 +++ + .../inputdata/idvalue/html/idvalue3html.html | 22 +++++++ + .../inputdata/idvalue/xml/idvalue1xml.xml | 20 ++++++ + .../inputdata/idvalue/xml/idvalue2xml.xml | 4 + + .../inputdata/idvalue/xml/idvalue3xml.xml | 13 ++++ + .../inputdata/idvalue/xml/idvalue3xmlrules.xml | 3 + + .../inputdata/idvalue/xml/idvalue4xml.xml | 17 +++++ + .../inputdata/idvalue/xml/idvalue4xmlrules.xml | 4 + + .../inputdata/idvalue/xml/idvalue5xml.xml | 20 ++++++ + .../inputdata/localefilter/html/locale1html.html | 14 ++++ + .../localefilter/html/locale1htmlrules.xml | 3 + + .../inputdata/localefilter/html/locale2html.html | 12 ++++ + .../inputdata/localefilter/html/locale3html.html | 14 ++++ + .../localefilter/html/locale3htmlrules.xml | 4 + + .../inputdata/localefilter/html/locale4html.html | 18 ++++++ + .../inputdata/localefilter/xml/locale1xml.xml | 12 ++++ + .../inputdata/localefilter/xml/locale2xml.xml | 9 +++ + .../inputdata/localefilter/xml/locale3xml.xml | 10 +++ + .../inputdata/localefilter/xml/locale3xmlrules.xml | 3 + + .../inputdata/localefilter/xml/locale4xml.xml | 16 +++++ + .../inputdata/localefilter/xml/locale4xmlrules.xml | 3 + + .../inputdata/localefilter/xml/locale5xml.xml | 8 +++ + .../inputdata/localefilter/xml/locale6xml.xml | 14 ++++ + .../inputdata/localefilter/xml/locale7xml.xml | 13 ++++ + .../inputdata/localefilter/xml/locale7xmlrules.xml | 4 + + .../localizationnote/html/locnote1html.html | 11 ++++ + .../localizationnote/html/locnote1htmlrules.xml | 7 ++ + .../localizationnote/html/locnote2html.html | 20 ++++++ + .../localizationnote/html/locnote2htmlrules.xml | 4 + + .../localizationnote/html/locnote3html.html | 12 ++++ + .../localizationnote/html/locnote3htmlrules.xml | 3 + + .../localizationnote/html/locnote4html.html | 16 +++++ + .../localizationnote/html/locnote4htmlrules.xml | 3 + + .../localizationnote/html/locnote5html.html | 21 +++++++ + .../localizationnote/html/locnote5htmlrules.xml | 15 +++++ + .../localizationnote/html/locnote6html.html | 21 +++++++ + .../localizationnote/html/locnote6htmlrules.xml | 15 +++++ + .../localizationnote/html/locnote7html.html | 10 +++ + .../localizationnote/html/locnote8html.html | 21 +++++++ + .../localizationnote/html/locnote8htmlrules.xml | 16 +++++ + .../localizationnote/html/locnote9html.html | 20 ++++++ + .../localizationnote/xml/locnote10xml.xml | 33 ++++++++++ + .../localizationnote/xml/locnote11xml.xml | 18 ++++++ + .../localizationnote/xml/locnote11xmlrules.xml | 16 +++++ + .../inputdata/localizationnote/xml/locnote1xml.xml | 14 ++++ + .../inputdata/localizationnote/xml/locnote2xml.xml | 19 ++++++ + .../inputdata/localizationnote/xml/locnote3xml.xml | 12 ++++ + .../inputdata/localizationnote/xml/locnote4xml.xml | 16 +++++ + .../inputdata/localizationnote/xml/locnote5xml.xml | 15 +++++ + .../localizationnote/xml/locnote5xmlrules.xml | 9 +++ + .../inputdata/localizationnote/xml/locnote6xml.xml | 21 +++++++ + .../inputdata/localizationnote/xml/locnote7xml.xml | 20 ++++++ + .../inputdata/localizationnote/xml/locnote8xml.xml | 17 +++++ + .../inputdata/localizationnote/xml/locnote9xml.xml | 8 +++ + .../preservespace/xml/preservespace1xml.xml | 13 ++++ + .../preservespace/xml/preservespace2xml.xml | 8 +++ + .../preservespace/xml/preservespace3xml.xml | 11 ++++ + .../preservespace/xml/preservespace3xmlrules.xml | 3 + + .../preservespace/xml/preservespace4xml.xml | 8 +++ + .../preservespace/xml/preservespace5xml.xml | 14 ++++ + .../preservespace/xml/preservespace6xml.xml | 9 +++ + .../preservespace/xml/preservespace6xmlrules.xml | 4 + + .../inputdata/translate/html/translate1html.html | 16 +++++ + .../translate/html/translate1htmlrules.xml | 4 + + .../inputdata/translate/html/translate2html.html | 17 +++++ + .../inputdata/translate/html/translate3html.html | 23 +++++++ + .../translate/html/translate3htmlrules.xml | 4 + + .../inputdata/translate/html/translate4html.html | 23 +++++++ + .../translate/html/translate4htmlrules.xml | 4 + + .../inputdata/translate/html/translate5html.html | 34 +++++++++++ + .../translate/html/translate5htmlrules.xml | 4 + + .../inputdata/translate/html/translate6html.html | 34 +++++++++++ + .../translate/html/translate6htmlrules.xml | 6 ++ + .../inputdata/translate/html/translate7html.html | 19 ++++++ + .../inputdata/translate/xml/translate10xml.xml | 7 ++ + .../translate/xml/translate10xmlrules.xml | 5 ++ + .../inputdata/translate/xml/translate1xml.xml | 39 ++++++++++++ + .../inputdata/translate/xml/translate2xml.xml | 10 +++ + .../inputdata/translate/xml/translate2xmlrules.xml | 5 ++ + .../inputdata/translate/xml/translate3xml.xml | 13 ++++ + .../inputdata/translate/xml/translate3xmlrules.xml | 6 ++ + .../inputdata/translate/xml/translate4xml.xml | 10 +++ + .../inputdata/translate/xml/translate5xml.xml | 13 ++++ + .../inputdata/translate/xml/translate6xml.xml | 19 ++++++ + .../inputdata/translate/xml/translate7xml.xml | 19 ++++++ + .../inputdata/translate/xml/translate8xml.xml | 28 +++++++++ + .../inputdata/translate/xml/translate9xml.xml | 11 ++++ + tests/ITS-2.0-Testsuite/run_tests.sh | 63 ++++++++++++++++++++ + 200 files changed, 3025 insertions(+), 8 deletions(-) + +commit 24811c8cfcbc14cc480a2fbef2288245019f8a7e +Author: Shaun McCance +Date: Wed Sep 26 07:46:10 2012 -0400 + + Implemented ITS 2.0 "ID Value" data category + + itstool.in | 48 ++++++++++++++++++++++++++++++++++++- + tests/IdValue/idvalue1xml.pot | 23 +++++++++++++++++ + tests/IdValue/idvalue1xml.xml | 12 +++++++++ + tests/IdValue/idvalue2xml.pot | 22 +++++++++++++++++ + tests/IdValue/idvalue2xml.xml | 4 +++ + tests/IdValue/idvalue3XmlRule.xml | 4 +++ + tests/IdValue/idvalue3xml.pot | 23 +++++++++++++++++ + tests/IdValue/idvalue3xml.xml | 11 ++++++++ + tests/run_tests.py | 9 +++++++ + 9 files changed, 155 insertions(+), 1 deletions(-) + +commit 1440e3b495ae502658e5e5ede6c09babce4bc4f9 +Author: Shaun McCance +Date: Fri Sep 21 09:39:42 2012 -0400 + + Support for local withinText + + itstool.in | 12 ++++++++++-- + tests/elementwithintextLocalXml.ll.po | 16 ++++++++++++++++ + tests/elementwithintextLocalXml.ll.xml | 6 ++++++ + tests/elementwithintextLocalXml.pot | 16 ++++++++++++++++ + tests/elementwithintextLocalXml.xml | 8 ++++++++ + tests/elementwithintextlocalitsSpanXml.ll.po | 16 ++++++++++++++++ + tests/elementwithintextlocalitsSpanXml.ll.xml | 6 ++++++ + tests/elementwithintextlocalitsSpanXml.pot | 16 ++++++++++++++++ + tests/elementwithintextlocalitsSpanXml.xml | 7 +++++++ + tests/run_tests.py | 6 ++++++ + 10 files changed, 107 insertions(+), 2 deletions(-) + +commit 2bca336117edee032ad956994d2ef3650f25e322 +Author: Shaun McCance +Date: Thu Sep 20 20:12:36 2012 -0400 + + Fix tests for namespace prefix handling + + itstool.in | 8 ++++---- + tests/IT-prefixes-1.ll.po | 16 +++------------- + tests/IT-prefixes-1.ll.xml | 7 ++++++- + tests/IT-prefixes-1.pot | 16 +++------------- + tests/IT-prefixes-1.xml | 5 +++++ + 5 files changed, 21 insertions(+), 31 deletions(-) + +commit 3a56e53c4cd157da0c5f21c18806047e6b64eddb +Merge: 70f6f5e 5040a32 +Author: Shaun McCance +Date: Thu Sep 20 19:58:18 2012 -0400 + + Merge branch 'master' into its-2-0 + +commit 5040a328ba73ec3dd119f013b9a12a2fdc99a6b9 +Author: Shaun McCance +Date: Thu Sep 20 18:37:44 2012 -0400 + + Support namespace prefixes on elements + + itstool.in | 10 ++++++++-- + tests/IT-prefixes-1.ll.po | 36 ++++++++++++++++++++++++++++++++++++ + tests/IT-prefixes-1.ll.xml | 16 ++++++++++++++++ + tests/IT-prefixes-1.pot | 36 ++++++++++++++++++++++++++++++++++++ + tests/IT-prefixes-1.xml | 19 +++++++++++++++++++ + tests/run_tests.py | 3 +++ + 6 files changed, 118 insertions(+), 2 deletions(-) + +commit 70f6f5ef01dc03e624e47a341b146ac099be4ebd +Author: Shaun McCance +Date: Wed Sep 19 19:58:37 2012 -0400 + + Adding test output for External Resource + + itstool.in | 2 ++ + 1 files changed, 2 insertions(+), 0 deletions(-) + +commit a0880365b8562d59e81889e980429aef840f89c4 +Author: Shaun McCance +Date: Wed Sep 19 16:49:47 2012 -0400 + + Use External Resource implementation for itst:externalRefRule + + itstool.in | 46 +++++------------------------------------- + tests/IT-externalRef1.ll.po | 12 +++++----- + tests/IT-externalRef1.pot | 12 +++++----- + 3 files changed, 18 insertions(+), 52 deletions(-) + +commit fee986b054284b029b7b870eff6ded9fd53e43a0 +Author: Shaun McCance +Date: Wed Sep 19 16:44:37 2012 -0400 + + Implemented ITS 2.0 External Resource data category + + itstool.in | 51 ++++++++++++++++++++ + .../Attr/ExternalResource1AttrXml.pot | 41 ++++++++++++++++ + .../Attr/ExternalResource1AttrXml.xml | 24 +++++++++ + .../Attr/ExternalResource2AttrRule.xml | 3 + + .../Attr/ExternalResource2AttrXml.pot | 41 ++++++++++++++++ + .../Attr/ExternalResource2AttrXml.xml | 22 ++++++++ + .../Attr/ExternalResource3AttrRule.xml | 3 + + .../Attr/ExternalResource3AttrXml.pot | 51 ++++++++++++++++++++ + .../Attr/ExternalResource3AttrXml.xml | 28 +++++++++++ + tests/ExternalResource/Attr/movie-frame.gif | 1 + + tests/ExternalResource/Attr/movie.avi | 1 + + tests/ExternalResource/Attr/movie.mp3 | 1 + + tests/ExternalResource/ExternalResource1Xml.pot | 41 ++++++++++++++++ + tests/ExternalResource/ExternalResource1Xml.xml | 24 +++++++++ + tests/ExternalResource/ExternalResource2Rule.xml | 3 + + tests/ExternalResource/ExternalResource2Xml.pot | 41 ++++++++++++++++ + tests/ExternalResource/ExternalResource2Xml.xml | 22 ++++++++ + tests/ExternalResource/ExternalResource3Rule.xml | 3 + + tests/ExternalResource/ExternalResource3Xml.pot | 51 ++++++++++++++++++++ + tests/ExternalResource/ExternalResource3Xml.xml | 28 +++++++++++ + tests/ExternalResource/movie-frame.gif | 1 + + tests/ExternalResource/movie.avi | 1 + + tests/ExternalResource/movie.mp3 | 1 + + tests/run_tests.py | 18 +++++++ + 24 files changed, 501 insertions(+), 0 deletions(-) + +commit bec881523df59fb1380f82ab0a789d67c34a58ed +Author: Shaun McCance +Date: Wed Sep 19 14:10:24 2012 -0400 + + Adding a test for itst:externalRefRule + + Don't know why I never had a test for this before. I'm + going to implement the ITS 2.0 External Resource data + category, and retool this extension on top, so I need + to make sure I don't break it. + + tests/IT-externalRef1.ll.po | 31 +++++++++++++++++++++++++++++++ + tests/IT-externalRef1.ll.xml | 10 ++++++++++ + tests/IT-externalRef1.pot | 31 +++++++++++++++++++++++++++++++ + tests/IT-externalRef1.txt | 1 + + tests/IT-externalRef1.xml | 10 ++++++++++ + tests/run_tests.py | 3 +++ + 6 files changed, 86 insertions(+), 0 deletions(-) + +commit ce76559b75fc60ef2a6d69dc99e1781b6adcfe99 +Author: Shaun McCance +Date: Wed Sep 12 10:21:22 2012 -0400 + + Moving some of the tests into subdirectories + + tests/EX-locNote-element-1.pot | 18 ------- + tests/EX-locNote-element-1.xml | 14 ------ + tests/EX-locNote-selector-2.pot | 37 --------------- + tests/EX-locNote-selector-2.xml | 10 ---- + tests/EX-locNotePointer-attribute-1.pot | 23 --------- + tests/EX-locNotePointer-attribute-1.xml | 19 -------- + tests/EX-locNoteRef-attribute-1.pot | 17 ------- + tests/EX-locNoteRef-attribute-1.xml | 12 ----- + tests/EX-locNoteRefPointer-attribute-1.pot | 23 --------- + tests/EX-locNoteRefPointer-attribute-1.xml | 16 ------- + tests/LocNote/EX-locNote-element-1.pot | 18 +++++++ + tests/LocNote/EX-locNote-element-1.xml | 14 ++++++ + tests/LocNote/EX-locNote-selector-2.pot | 37 +++++++++++++++ + tests/LocNote/EX-locNote-selector-2.xml | 10 ++++ + tests/LocNote/EX-locNotePointer-attribute-1.pot | 23 +++++++++ + tests/LocNote/EX-locNotePointer-attribute-1.xml | 19 ++++++++ + tests/LocNote/EX-locNoteRef-attribute-1.pot | 17 +++++++ + tests/LocNote/EX-locNoteRef-attribute-1.xml | 12 +++++ + tests/LocNote/EX-locNoteRefPointer-attribute-1.pot | 23 +++++++++ + tests/LocNote/EX-locNoteRefPointer-attribute-1.xml | 16 +++++++ + tests/LocNote/LocNote1.pot | 48 ++++++++++++++++++++ + tests/LocNote/LocNote1.xml | 21 +++++++++ + tests/LocNote/LocNote2.pot | 33 +++++++++++++ + tests/LocNote/LocNote2.xml | 15 ++++++ + tests/LocNote/LocNote2_LinkedRules.xml | 9 ++++ + tests/LocNote/LocNote3.pot | 35 ++++++++++++++ + tests/LocNote/LocNote3.xml | 17 +++++++ + tests/LocNote/LocNote4.pot | 35 ++++++++++++++ + tests/LocNote/LocNote4.xml | 8 +++ + tests/LocNote1.pot | 48 -------------------- + tests/LocNote1.xml | 21 --------- + tests/LocNote2.pot | 33 ------------- + tests/LocNote2.xml | 15 ------ + tests/LocNote2_LinkedRules.xml | 9 ---- + tests/LocNote3.pot | 35 -------------- + tests/LocNote3.xml | 17 ------- + tests/LocNote4.pot | 35 -------------- + tests/LocNote4.xml | 8 --- + tests/Translate/Translate1.ll.po | 47 +++++++++++++++++++ + tests/Translate/Translate1.ll.xml | 39 ++++++++++++++++ + tests/Translate/Translate1.pot | 46 +++++++++++++++++++ + tests/Translate/Translate1.xml | 39 ++++++++++++++++ + tests/Translate/Translate2.ll.po | 21 +++++++++ + tests/Translate/Translate2.ll.xml | 10 ++++ + tests/Translate/Translate2.pot | 21 +++++++++ + tests/Translate/Translate2.xml | 10 ++++ + tests/Translate/Translate2_LinkedRules.xml | 5 ++ + tests/Translate/Translate3.ll.po | 25 ++++++++++ + tests/Translate/Translate3.ll.wrong.po | 25 ++++++++++ + tests/Translate/Translate3.ll.wrong.xml | 12 +++++ + tests/Translate/Translate3.ll.xml | 10 ++++ + tests/Translate/Translate3.pot | 21 +++++++++ + tests/Translate/Translate3.xml | 13 +++++ + tests/Translate/Translate4.ll.po | 25 ++++++++++ + tests/Translate/Translate4.ll.xml | 10 ++++ + tests/Translate/Translate4.pot | 21 +++++++++ + tests/Translate/Translate4.xml | 10 ++++ + tests/Translate/Translate5.ll.po | 25 ++++++++++ + tests/Translate/Translate5.ll.xml | 19 ++++++++ + tests/Translate/Translate5.pot | 21 +++++++++ + tests/Translate/Translate5.xml | 19 ++++++++ + tests/Translate/Translate6.ll.po | 40 ++++++++++++++++ + tests/Translate/Translate6.ll.xml | 19 ++++++++ + tests/Translate/Translate6.pot | 36 +++++++++++++++ + tests/Translate/Translate6.xml | 19 ++++++++ + tests/Translate/Translate7.ll.po | 21 +++++++++ + tests/Translate/Translate7.ll.xml | 29 ++++++++++++ + tests/Translate/Translate7.pot | 21 +++++++++ + tests/Translate/Translate7.xml | 28 +++++++++++ + tests/Translate/TranslateGlobal.ll.po | 26 +++++++++++ + tests/Translate/TranslateGlobal.ll.xml | 13 +++++ + tests/Translate/TranslateGlobal.pot | 26 +++++++++++ + tests/Translate/TranslateGlobal.xml | 13 +++++ + tests/Translate/TranslateGlobal_LinkedRules.xml | 6 +++ + tests/Translate1.ll.po | 47 ------------------- + tests/Translate1.ll.xml | 39 ---------------- + tests/Translate1.pot | 46 ------------------- + tests/Translate1.xml | 39 ---------------- + tests/Translate2.ll.po | 21 --------- + tests/Translate2.ll.xml | 10 ---- + tests/Translate2.pot | 21 --------- + tests/Translate2.xml | 10 ---- + tests/Translate2_LinkedRules.xml | 5 -- + tests/Translate3.ll.po | 25 ---------- + tests/Translate3.ll.wrong.po | 25 ---------- + tests/Translate3.ll.wrong.xml | 12 ----- + tests/Translate3.ll.xml | 10 ---- + tests/Translate3.pot | 21 --------- + tests/Translate3.xml | 13 ----- + tests/Translate4.ll.po | 25 ---------- + tests/Translate4.ll.xml | 10 ---- + tests/Translate4.pot | 21 --------- + tests/Translate4.xml | 10 ---- + tests/Translate5.ll.po | 25 ---------- + tests/Translate5.ll.xml | 19 -------- + tests/Translate5.pot | 21 --------- + tests/Translate5.xml | 19 -------- + tests/Translate6.ll.po | 40 ---------------- + tests/Translate6.ll.xml | 19 -------- + tests/Translate6.pot | 36 --------------- + tests/Translate6.xml | 19 -------- + tests/Translate7.ll.po | 21 --------- + tests/Translate7.ll.xml | 29 ------------ + tests/Translate7.pot | 21 --------- + tests/Translate7.xml | 28 ----------- + tests/TranslateGlobal.ll.po | 26 ----------- + tests/TranslateGlobal.ll.xml | 13 ----- + tests/TranslateGlobal.pot | 26 ----------- + tests/TranslateGlobal.xml | 13 ----- + tests/TranslateGlobal_LinkedRules.xml | 6 --- + tests/run_tests.py | 44 +++++++++--------- + 111 files changed, 1223 insertions(+), 1223 deletions(-) + +commit 30c0c2c8f86119dc861f5ff8036b79e52eed0292 +Author: Shaun McCance +Date: Wed Sep 12 09:07:23 2012 -0400 + + Adding more Locale Filter tests + + tests/LocaleFilter/Locale2Xml.fr_CA.po | 23 +++++++++++ + tests/LocaleFilter/Locale2Xml.fr_CA.xml | 8 ++++ + tests/LocaleFilter/Locale2Xml.fr_CH.po | 23 +++++++++++ + tests/LocaleFilter/Locale2Xml.fr_CH.xml | 8 ++++ + tests/LocaleFilter/Locale2Xml.fr_FR.po | 23 +++++++++++ + tests/LocaleFilter/Locale2Xml.fr_FR.xml | 5 ++ + tests/LocaleFilter/Locale2Xml.joined.xml | 16 +++++++ + tests/LocaleFilter/Locale2Xml.pot | 23 +++++++++++ + tests/LocaleFilter/Locale2Xml.xml | 14 ++++++ + tests/LocaleFilter/Locale3Rule.xml | 5 ++ + tests/LocaleFilter/Locale3Xml.fr_CA.po | 23 +++++++++++ + tests/LocaleFilter/Locale3Xml.fr_CA.xml | 9 ++++ + tests/LocaleFilter/Locale3Xml.fr_CH.po | 23 +++++++++++ + tests/LocaleFilter/Locale3Xml.fr_CH.xml | 9 ++++ + tests/LocaleFilter/Locale3Xml.fr_FR.po | 23 +++++++++++ + tests/LocaleFilter/Locale3Xml.fr_FR.xml | 6 +++ + tests/LocaleFilter/Locale3Xml.joined.xml | 17 ++++++++ + tests/LocaleFilter/Locale3Xml.pot | 23 +++++++++++ + tests/LocaleFilter/Locale3Xml.xml | 16 +++++++ + tests/LocaleFilter/Locale4Rule.xml | 3 + + tests/LocaleFilter/Locale4Xml.fr_CA.po | 23 +++++++++++ + tests/LocaleFilter/Locale4Xml.fr_CA.xml | 12 ++++++ + tests/LocaleFilter/Locale4Xml.fr_CH.po | 23 +++++++++++ + tests/LocaleFilter/Locale4Xml.fr_CH.xml | 12 ++++++ + tests/LocaleFilter/Locale4Xml.fr_FR.po | 23 +++++++++++ + tests/LocaleFilter/Locale4Xml.fr_FR.xml | 9 ++++ + tests/LocaleFilter/Locale4Xml.joined.xml | 17 ++++++++ + tests/LocaleFilter/Locale4Xml.pot | 23 +++++++++++ + tests/LocaleFilter/Locale4Xml.xml | 16 +++++++ + tests/LocaleFilter/Locale5Xml.fr_CA.po | 23 +++++++++++ + tests/LocaleFilter/Locale5Xml.fr_CA.xml | 8 ++++ + tests/LocaleFilter/Locale5Xml.fr_CH.po | 23 +++++++++++ + tests/LocaleFilter/Locale5Xml.fr_CH.xml | 8 ++++ + tests/LocaleFilter/Locale5Xml.fr_FR.po | 23 +++++++++++ + tests/LocaleFilter/Locale5Xml.fr_FR.xml | 5 ++ + tests/LocaleFilter/Locale5Xml.joined.xml | 16 +++++++ + tests/LocaleFilter/Locale5Xml.pot | 23 +++++++++++ + tests/LocaleFilter/Locale5Xml.xml | 13 ++++++ + tests/run_tests.py | 64 ++++++++++++++++++++++++++++++ + 39 files changed, 664 insertions(+), 0 deletions(-) + +commit 429131c2ffe3fd69d3f29a66ae7eb9ea127b4e72 +Author: Shaun McCance +Date: Tue Sep 11 22:57:18 2012 -0400 + + Sort attribute names in test output + + itstool.in | 4 ++-- + 1 files changed, 2 insertions(+), 2 deletions(-) + +commit 13e8e0c2ab1fb8c793e86e35fa12cbd7694f7f63 +Author: Shaun McCance +Date: Tue Sep 11 16:48:44 2012 -0400 + + Renaming IT-join-1 test output file + + I changed the naming convention to make it easier + to reuse other test input files to also test joins. + + tests/IT-join-1.joined.xml | 36 ++++++++++++++++++++++++++++++++++++ + tests/IT-join-1.ll.xml | 36 ------------------------------------ + 2 files changed, 36 insertions(+), 36 deletions(-) + +commit 2d37295f5ccdafc3372ea0e576400524fc688e45 +Author: Shaun McCance +Date: Tue Sep 11 16:47:19 2012 -0400 + + Updating test files for --no-builtins + + tests/IT-locNote-inline.pot | 5 +++++ + tests/IT-locNote-multiples.pot | 5 +++++ + tests/IT-placeholder-1.ll.xml | 2 +- + tests/LocNote1.pot | 20 ++++++++++++++++++++ + tests/LocNote2.pot | 5 +++++ + tests/Translate3.ll.wrong.xml | 2 +- + tests/Translate3.ll.xml | 2 +- + tests/Translate4.ll.xml | 2 +- + 8 files changed, 39 insertions(+), 4 deletions(-) + +commit 0e6a42ffd5fe7cdac8197f22b89e57fe55926687 +Author: Shaun McCance +Date: Tue Sep 11 16:45:55 2012 -0400 + + First pass at implementing Locale Filter + + itstool.in | 161 +++++++++++++++++++++++------- + tests/LocaleFilter/Locale1Xml.fr_CA.po | 23 ++++ + tests/LocaleFilter/Locale1Xml.fr_CA.xml | 13 +++ + tests/LocaleFilter/Locale1Xml.fr_CH.po | 23 ++++ + tests/LocaleFilter/Locale1Xml.fr_CH.xml | 13 +++ + tests/LocaleFilter/Locale1Xml.fr_FR.po | 23 ++++ + tests/LocaleFilter/Locale1Xml.fr_FR.xml | 10 ++ + tests/LocaleFilter/Locale1Xml.joined.xml | 21 ++++ + tests/LocaleFilter/Locale1Xml.pot | 23 ++++ + tests/LocaleFilter/Locale1Xml.xml | 20 ++++ + tests/run_tests.py | 75 ++++++++------ + 11 files changed, 338 insertions(+), 67 deletions(-) + +commit cc8084a1596e61363585dea0c13fbe94a4266a1f +Author: Shaun McCance +Date: Mon Sep 10 12:26:25 2012 -0400 + + Implemented test suite output for withinText + + itstool.in | 27 +++++++++++++++------------ + 1 files changed, 15 insertions(+), 12 deletions(-) + +commit 7c8434af35f907f5c97741d1d22d8157d51a6a5f +Author: Shaun McCance +Date: Mon Sep 10 11:38:00 2012 -0400 + + Adding test suite output for its:translate + + itstool.in | 110 ++++++++++++++++++++++++++++++++++++++++++------------------ + 1 files changed, 77 insertions(+), 33 deletions(-) + +commit 231548b2539e281a3e68b08a052cb4b03c5368a0 +Author: Shaun McCance +Date: Sat Sep 8 17:36:43 2012 -0400 + + Implemented ITS 2.0 Preserve Space data category + + itstool.in | 20 +++++++++++++------- + tests/preservespace1xml.pot | 23 +++++++++++++++++++++++ + tests/preservespace1xml.xml | 15 +++++++++++++++ + tests/preservespace2xml.pot | 23 +++++++++++++++++++++++ + tests/preservespace2xml.xml | 10 ++++++++++ + tests/preservespace3XmlRule.xml | 3 +++ + tests/preservespace3xml.pot | 23 +++++++++++++++++++++++ + tests/preservespace3xml.xml | 12 ++++++++++++ + tests/preservespace4xml.pot | 23 +++++++++++++++++++++++ + tests/preservespace4xml.xml | 10 ++++++++++ + tests/run_tests.py | 12 ++++++++++++ + 11 files changed, 167 insertions(+), 7 deletions(-) + +commit 34cc26b03424cd3ac72c041c8b576000ce2011d2 +Author: Galen Charlton +Date: Wed Aug 29 10:21:37 2012 -0400 + + add --load-dtd option + + This option tells itstool to load external DTDs when + parsing the document to be translated. This prevents + errors when the document includes entity references defined + in those DTDs. + + Note that externally-defined entity refs still cannot be used + in translated strings in the PO files. + + Also note that this adds test cases that require either + network access or updating the local XML catalog to + including the DocBook DTDs. + + Signed-off-by: Galen Charlton + + itstool.in | 13 ++++++++++--- + tests/IT-uses-external-dtds.ll.po | 21 +++++++++++++++++++++ + tests/IT-uses-external-dtds.ll.xml | 7 +++++++ + tests/IT-uses-external-dtds.pot | 21 +++++++++++++++++++++ + tests/IT-uses-external-dtds.xml | 7 +++++++ + tests/run_tests.py | 19 ++++++++++++++++--- + 6 files changed, 82 insertions(+), 6 deletions(-) + +commit 7ee29a46d229bde60c86703730b24d5ac49a1e75 +Author: Shaun McCance +Date: Sun Jun 24 10:30:01 2012 -0400 + + Version 1.2.0 + + NEWS | 9 +++++++++ + configure.ac | 2 +- + 2 files changed, 10 insertions(+), 1 deletions(-) + commit 6fce21cd7b4417d8ea8cff3a8f2c79063d5353a3 Merge: f951a0f 470b52a Author: Shaun McCance diff --git a/NEWS b/NEWS index 73530c0..6cd7b59 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,22 @@ +2.0.0 +===== +* Support for ITS 2.0 Preserve Space data category +* Support for ITS 2.0 Locale Filter data category +* Support for ITS 2.0 External Resource data category +* Support for ITS 2.0 ID Value data category +* Support for ITS 2.0 parameters, including user overrides +* Support for ITS 2.0 local withinText attribute +* Fixed handling of localization note inheritance +* Fixed handling od namespace prefixes on elements +* Added option to retain entity references in PO files +* Added option to load external DTDs (Galen Charlton) +* Added built-in rules for DocBook 5 +* Updated built-in rules to use ITS 2.0 Preserve Space and External + Resource instead of 1.x custom extensions +* Excluded editor remarks and comments in built-in DocBook and Mallard + rules with Locale Filter +* Made all DocBook *info children not within text in built-in rules + 1.2.0 ===== * Added new "join mode" for multilingual XML formats diff --git a/configure b/configure index ed64cb3..51ccaf7 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.66 for itstool 1.2.0. +# Generated by GNU Autoconf 2.66 for itstool 2.0.0. # # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -548,8 +548,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='itstool' PACKAGE_TARNAME='itstool' -PACKAGE_VERSION='1.2.0' -PACKAGE_STRING='itstool 1.2.0' +PACKAGE_VERSION='2.0.0' +PACKAGE_STRING='itstool 2.0.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1165,7 +1165,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures itstool 1.2.0 to adapt to many kinds of systems. +\`configure' configures itstool 2.0.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1231,7 +1231,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of itstool 1.2.0:";; + short | recursive ) echo "Configuration of itstool 2.0.0:";; esac cat <<\_ACEOF @@ -1298,7 +1298,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -itstool configure 1.2.0 +itstool configure 2.0.0 generated by GNU Autoconf 2.66 Copyright (C) 2010 Free Software Foundation, Inc. @@ -1315,7 +1315,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by itstool $as_me 1.2.0, which was +It was created by itstool $as_me 2.0.0, which was generated by GNU Autoconf 2.66. Invocation command line was $ $0 $@ @@ -2130,7 +2130,7 @@ fi # Define the identity of the package. PACKAGE='itstool' - VERSION='1.2.0' + VERSION='2.0.0' cat >>confdefs.h <<_ACEOF @@ -2180,6 +2180,19 @@ DATADIR=`( )` +py_module=libxml2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for python module $py_module" >&5 +$as_echo_n "checking for python module $py_module... " >&6; } +echo "import $py_module" | python - &>/dev/null +if test $? -ne 0; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: not found" >&5 +$as_echo "not found" >&6; } + as_fn_error $? "Python module $py_module is needed to run this package" "$LINENO" 5 +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: found" >&5 +$as_echo "found" >&6; } +fi + ac_config_files="$ac_config_files Makefile itstool itstool.1 its/Makefile" @@ -2726,7 +2739,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by itstool $as_me 1.2.0, which was +This file was extended by itstool $as_me 2.0.0, which was generated by GNU Autoconf 2.66. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2779,7 +2792,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -itstool config.status 1.2.0 +itstool config.status 2.0.0 configured by $0, generated by GNU Autoconf 2.66, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index e2e86f4..82d1ea8 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([itstool], [1.2.0], []) +AC_INIT([itstool], [2.0.0], []) AM_INIT_AUTOMAKE([1.9 no-dist-gzip dist-bzip2]) DATADIR=`( @@ -10,6 +10,16 @@ DATADIR=`( )` AC_SUBST([DATADIR]) +py_module=libxml2 +AC_MSG_CHECKING(for python module $py_module) +echo "import $py_module" | python - &>/dev/null +if test $? -ne 0; then + AC_MSG_RESULT(not found) + AC_MSG_ERROR(Python module $py_module is needed to run this package) +else + AC_MSG_RESULT(found) +fi + AC_CONFIG_FILES([ Makefile itstool diff --git a/its/Makefile.am b/its/Makefile.am index fb45ba8..5a6ebf7 100644 --- a/its/Makefile.am +++ b/its/Makefile.am @@ -1,5 +1,5 @@ itsdir = $(datadir)/itstool/its -its_DATA = docbook.its its.its mallard.its ttml.its xhtml.its +its_DATA = docbook.its docbook5.its its.its mallard.its ttml.its xhtml.its EXTRA_DIST = $(its_DATA) diff --git a/its/Makefile.in b/its/Makefile.in index 3bfab15..ab33803 100644 --- a/its/Makefile.in +++ b/its/Makefile.in @@ -141,7 +141,7 @@ top_build_prefix = @top_build_prefix@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ itsdir = $(datadir)/itstool/its -its_DATA = docbook.its its.its mallard.its ttml.its xhtml.its +its_DATA = docbook.its docbook5.its its.its mallard.its ttml.its xhtml.its EXTRA_DIST = $(its_DATA) all: all-am diff --git a/its/docbook.its b/its/docbook.its index 3c22c17..ed223d3 100644 --- a/its/docbook.its +++ b/its/docbook.its @@ -1,7 +1,7 @@ + version="2.0"> @@ -33,6 +33,8 @@ + + @@ -156,13 +158,14 @@ - + + @@ -189,23 +192,25 @@ space-preserving translation unit. --> - + + + + - - - - - - + + + + + + - - + + - - - + diff --git a/its/docbook5.its b/its/docbook5.its new file mode 100644 index 0000000..a2f0b1d --- /dev/null +++ b/its/docbook5.its @@ -0,0 +1,204 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/its/its.its b/its/its.its index b832add..8f17c36 100644 --- a/its/its.its +++ b/its/its.its @@ -1,5 +1,6 @@ + version="2.0"> + diff --git a/its/mallard.its b/its/mallard.its index ab5e725..8ac8878 100644 --- a/its/mallard.its +++ b/its/mallard.its @@ -2,7 +2,7 @@ xmlns:its="http://www.w3.org/2005/11/its" xmlns:itst="http://itstool.org/extensions/" xmlns:mal="http://projectmallard.org/1.0/" - version="1.0"> + version="2.0"> @@ -29,13 +29,14 @@ - - + + + + - - + diff --git a/its/ttml.its b/its/ttml.its index 72d544f..31a8300 100644 --- a/its/ttml.its +++ b/its/ttml.its @@ -1,6 +1,6 @@ + version="2.0"> diff --git a/its/xhtml.its b/its/xhtml.its index a3b192c..0720049 100644 --- a/its/xhtml.its +++ b/its/xhtml.its @@ -2,14 +2,14 @@ xmlns:its="http://www.w3.org/2005/11/its" xmlns:itst="http://itstool.org/extensions/" xmlns:html="http://www.w3.org/1999/xhtml" - version="1.0"> + version="2.0"> - + @@ -43,4 +43,7 @@ + + diff --git a/itstool b/itstool index 4e0adea..5be95f7 100644 --- a/itstool +++ b/itstool @@ -17,7 +17,7 @@ # Place, Suite 330, Boston, MA 0211-1307 USA. # -VERSION="1.2.0" +VERSION="2.0.0" DATADIR="/usr/local/share" import gettext @@ -34,6 +34,7 @@ NS_ITS = 'http://www.w3.org/2005/11/its' NS_ITST = 'http://itstool.org/extensions/' NS_BLANK = 'http://itstool.org/extensions/blank/' NS_XLINK = 'http://www.w3.org/1999/xlink' +NS_XML = 'http://www.w3.org/XML/1998/namespace' class NoneTranslations: def gettext(self, message): @@ -94,8 +95,17 @@ class MessageList (object): msgdict[key].add_marker(marker) for comment in msg.get_comments(): msgdict[key].add_comment(comment) + for idvalue in msg.get_id_values(): + msgdict[key].add_id_value(idvalue) if msg.get_preserve_space(): msgdict[key].set_preserve_space() + if msg.get_locale_filter() is not None: + locale = msgdict[key].get_locale_filter() + if locale is not None: + msgdict[key].set_locale_filter('%s, %s' % (locale, msg.get_locale_filter())) + else: + msgdict[key].set_locale_filter(msg.get_locale_filter()) + else: msgs.append(msg) msgdict[key] = msg @@ -117,7 +127,7 @@ class MessageList (object): class Comment (object): def __init__ (self, text): - self._text = text + self._text = str(text) assert(text is not None) self._markers = [] @@ -167,6 +177,8 @@ class Message (object): self._placeholders = [] self._sources = [] self._markers = [] + self._id_values = [] + self._locale_filter = None self._comments = [] self._preserve = False @@ -192,6 +204,10 @@ class Message (object): if re.sub('\s+', ' ', text).strip() != '': self._empty = False + def add_entity_ref (self, name): + self._message.append('&' + name + ';') + self._empty = False + def add_placeholder (self, node): holder = Message.Placeholder(node) self._placeholders.append(holder) @@ -208,18 +224,19 @@ class Message (object): def add_start_tag (self, node): if len(self._message) == 0 or not(isinstance(self._message[-1], basestring)): self._message.append('') - self._message[-1] += ('<%s' % node.name) - if node.properties is not None: - for prop in node.properties: - if prop.type == 'attribute': - name = prop.name - if prop.ns() is not None: - name = prop.ns().name + ':' + name - atval = prop.content - if not isinstance(atval, unicode): - atval = unicode(atval, 'utf-8') - atval = atval.replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"') - self._message += " %s=\"%s\"" % (name, atval) + if node.ns() is not None and node.ns().name is not None: + self._message[-1] += (u'<%s:%s' % (unicode(node.ns().name, 'utf-8'), unicode(node.name, 'utf-8'))) + else: + self._message[-1] += (u'<%s' % unicode(node.name, 'utf-8')) + for prop in xml_attr_iter(node): + name = prop.name + if prop.ns() is not None: + name = prop.ns().name + ':' + name + atval = prop.content + if not isinstance(atval, unicode): + atval = unicode(atval, 'utf-8') + atval = atval.replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"') + self._message += " %s=\"%s\"" % (name, atval) if node.children is not None: self._message[-1] += '>' else: @@ -229,7 +246,10 @@ class Message (object): if node.children is not None: if len(self._message) == 0 or not(isinstance(self._message[-1], basestring)): self._message.append('') - self._message[-1] += (u'' % unicode(node.name, 'utf-8')) + if node.ns() is not None and node.ns().name is not None: + self._message[-1] += (u'' % (unicode(node.ns().name, 'utf-8'), unicode(node.name, 'utf-8'))) + else: + self._message[-1] += (u'' % unicode(node.name, 'utf-8')) def is_empty (self): return self._empty @@ -256,6 +276,12 @@ class Message (object): def get_markers (self): return self._markers + def add_id_value(self, id_value): + self._id_values.append(id_value) + + def get_id_values(self): + return self._id_values + def add_comment (self, comment): if comment is not None: self._comments.append(comment) @@ -282,6 +308,12 @@ class Message (object): def set_preserve_space (self, preserve=True): self._preserve = preserve + def get_locale_filter(self): + return self._locale_filter + + def set_locale_filter(self, locale): + self._locale_filter = locale + def format (self): ret = u'' markers = {} @@ -289,6 +321,10 @@ class Message (object): if not markers.has_key(marker): ret += '#. (itstool) path: ' + marker + '\n' markers[marker] = marker + for idvalue in self._id_values: + ret += '#. (itstool) id: ' + idvalue + '\n' + if self._locale_filter is not None: + ret += '#. (itstool) ' + self._locale_filter[1] + ' locale: ' + self._locale_filter[0] + '\n' comments = [] commentsdict = {} for comment in self._comments: @@ -341,6 +377,34 @@ def xml_is_ns_name (node, ns, name): return False return node.name == name and node.ns() is not None and node.ns().content == ns +def xml_get_node_path(node): + # The built-in nodePath() method only does numeric indexes + # when necessary for disambiguation. For various reasons, + # we prefer always using indexes. + name = node.name + if node.ns() is not None and node.ns().name is not None: + name = node.ns().name + ':' + name + if node.type == 'attribute': + name = '@' + name + name = '/' + name + if node.type == 'element' and node.parent.type == 'element': + count = 1 + prev = node.previousElementSibling() + while prev is not None: + if prev.name == node.name: + if prev.ns() is None: + if node.ns() is None: + count += 1 + else: + if node.ns() is not None: + if prev.ns().name == node.ns().name: + count += 1 + prev = prev.previousElementSibling() + name = '%s[%i]' % (name, count) + if node.parent.type == 'element': + name = xml_get_node_path(node.parent) + name + return name + def xml_error_catcher(doc, error): doc._xml_err += " %s" % error @@ -359,8 +423,28 @@ def fix_node_ns (node, nsdefs): fix_node_ns(child, childnsdefs) +class LocNote (object): + def __init__(self, locnote=None, locnoteref=None, locnotetype=None, space=False): + self.locnote = locnote + self.locnoteref = locnoteref + self.locnotetype = locnotetype + if self.locnotetype != 'alert': + self.locnotetype = 'description' + self._preserve_space=space + + def __repr__(self): + if self.locnote is not None: + if self._preserve_space: + return self.locnote + else: + return re.sub('\s+', ' ', self.locnote).strip() + elif self.locnoteref is not None: + return '(itstool) link: ' + re.sub('\s+', ' ', self.locnoteref).strip() + return '' + + class Document (object): - def __init__ (self, filename, messages): + def __init__ (self, filename, messages, load_dtd=False, keep_entities=False): self._xml_err = '' libxml2.registerErrorHandler(xml_error_catcher, self) try: @@ -369,7 +453,14 @@ class Document (object): sys.stderr.write('Error: cannot open XML file %s\n' % filename) sys.exit(1) ctxt.lineNumbers(1) - ctxt.replaceEntities(1) + self._load_dtd = load_dtd + self._keep_entities = keep_entities + if load_dtd: + ctxt.loadSubset(1) + if keep_entities: + ctxt.replaceEntities(0) + else: + ctxt.replaceEntities(1) ctxt.parseDocument() self._filename = filename self._doc = ctxt.doc() @@ -393,7 +484,7 @@ class Document (object): else: sys.stderr.write('Warning: ITS file %s missing version attribute\n' % os.path.basename(href)) - if version is not None and version != '1.0': + if version is not None and version not in ('1.0', '2.0'): sys.stderr.write('Warning: Skipping ITS file %s with unknown version %s\n' % (os.path.basename(href), root.nsProp('version', None))) else: @@ -407,7 +498,7 @@ class Document (object): version = root.nsProp('version', NS_ITS) else: sys.stderr.write('Warning: Local ITS rules missing version attribute\n') - if version is not None and version != '1.0': + if version is not None and version not in ('1.0', '2.0'): sys.stderr.write('Warning: Skipping local ITS rules with unknown version %s\n' % version) else: @@ -422,19 +513,31 @@ class Document (object): self._msgs = messages self._its_translate_nodes = {} self._its_within_text_nodes = {} + self._its_locale_filters = {} + self._its_id_values = {} self._its_loc_notes = {} - self._itst_preserve_space_nodes = {} + self._its_preserve_space_nodes = {} self._itst_drop_nodes = {} self._itst_contexts = {} self._its_lang = {} self._itst_lang_attr = {} self._itst_credits = None - self._itst_externals = [] + self._its_externals = {} def _check_errors(self): if self._xml_err: raise libxml2.parserError(self._xml_err) + def register_its_params(self, xpath, rules, params={}): + for child in xml_child_iter(rules): + if xml_is_ns_name(child, NS_ITS, 'param'): + name = child.nsProp('name', None) + if params.has_key(name): + value = params[name] + else: + value = child.getContent() + xpath.xpathRegisterVariable(name, None, value) + def apply_its_rule(self, rule, xpath): if rule.type != 'element': return @@ -449,11 +552,48 @@ class Document (object): elif xml_is_ns_name(rule, NS_ITST, 'preserveSpaceRule'): if rule.nsProp('selector', None) is not None: for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)): - self._itst_preserve_space_nodes[node] = rule.nsProp('preserveSpace', None) + val = rule.nsProp('preserveSpace', None) + if val == 'yes': + self._its_preserve_space_nodes[node] = 'preserve' + elif xml_is_ns_name(rule, NS_ITS, 'preserveSpaceRule'): + if rule.nsProp('selector', None) is not None: + for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)): + self._its_preserve_space_nodes[node] = rule.nsProp('space', None) + elif xml_is_ns_name(rule, NS_ITS, 'localeFilterRule'): + if rule.nsProp('selector', None) is not None: + if rule.hasNsProp('localeFilterList', None): + lst = rule.nsProp('localeFilterList', None) + else: + lst = '*' + if rule.hasNsProp('localeFilterType', None): + typ = rule.nsProp('localeFilterType', None) + else: + typ = 'include' + for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)): + self._its_locale_filters[node] = (lst, typ) elif xml_is_ns_name(rule, NS_ITST, 'dropRule'): if rule.nsProp('selector', None) is not None: for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)): self._itst_drop_nodes[node] = rule.nsProp('drop', None) + elif xml_is_ns_name(rule, NS_ITS, 'idValueRule'): + sel = rule.nsProp('selector', None) + idv = rule.nsProp('idValue', None) + if sel is not None and idv is not None: + for node in self._try_xpath_eval(xpath, sel): + try: + oldnode = xpath.contextNode() + except: + oldnode = None + xpath.setContextNode(node) + idvalue = self._try_xpath_eval(xpath, idv) + if isinstance(idvalue, basestring): + self._its_id_values[node] = idvalue + else: + for val in idvalue: + self._its_id_values[node] = val.content + break + xpath.setContextNode(oldnode) + pass elif xml_is_ns_name(rule, NS_ITST, 'contextRule'): if rule.nsProp('selector', None) is not None: for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)): @@ -475,13 +615,14 @@ class Document (object): xpath.setContextNode(oldnode) elif xml_is_ns_name(rule, NS_ITS, 'locNoteRule'): locnote = None + notetype = rule.nsProp('locNoteType', None) for child in xml_child_iter(rule): if xml_is_ns_name(child, NS_ITS, 'locNote'): - locnote = re.sub('\s+', ' ', child.content).strip() + locnote = LocNote(locnote=child.content, locnotetype=notetype) break if locnote is None: if rule.hasNsProp('locNoteRef', None): - locnote = '(itstool) link: ' + re.sub('\s+', ' ', rule.nsProp('locNoteRef', None)).strip() + locnote = LocNote(locnoteref=rule.nsProp('locNoteRef', None), locnotetype=notetype) if rule.nsProp('selector', None) is not None: for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)): if locnote is not None: @@ -502,16 +643,19 @@ class Document (object): xpath.setContextNode(node) note = self._try_xpath_eval(xpath, sel) if isinstance(note, basestring): - self._its_loc_notes.setdefault(node, []).append(note) + if ref: + nodenote = LocNote(locnoteref=note, locnotetype=notetype) + else: + nodenote = LocNote(locnote=note, locnotetype=notetype) + self._its_loc_notes.setdefault(node, []).append(nodenote) else: for note in note: - if self.get_preserve_space(note): - cont = note.content - else: - cont = re.sub('\s+', ' ', note.content).strip() if ref: - cont = '(itstool) link: ' + cont - self._its_loc_notes.setdefault(node, []).append(cont) + nodenote = LocNote(locnoteref=note.content, locnotetype=notetype) + else: + nodenote = LocNote(locnote=note.content, locnotetype=notetype, + space=self.get_preserve_space(note)) + self._its_loc_notes.setdefault(node, []).append(nodenote) break xpath.setContextNode(oldnode) elif xml_is_ns_name(rule, NS_ITS, 'langRule'): @@ -537,44 +681,51 @@ class Document (object): for node in self._try_xpath_eval(xpath, rule.nsProp('appendTo', None)): self._itst_credits = (node, rule) break - elif xml_is_ns_name(rule, NS_ITST, 'externalRefRule'): - if rule.nsProp('selector', None) is not None and rule.nsProp('refPointer', None) is not None: - for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)): + elif (xml_is_ns_name(rule, NS_ITS, 'externalResourceRefRule') or + xml_is_ns_name(rule, NS_ITST, 'externalRefRule')): + sel = rule.nsProp('selector', None) + if xml_is_ns_name(rule, NS_ITS, 'externalResourceRefRule'): + ptr = rule.nsProp('externalResourceRefPointer', None) + else: + ptr = rule.nsProp('refPointer', None) + if sel is not None and ptr is not None: + for node in self._try_xpath_eval(xpath, sel): try: oldnode = xpath.contextNode() except: oldnode = None xpath.setContextNode(node) - res = self._try_xpath_eval(xpath, rule.nsProp('refPointer', None)) + res = self._try_xpath_eval(xpath, ptr) if len(res) > 0: - self._itst_externals.append((node, res[0].content)) + self._its_externals[node] = res[0].content xpath.setContextNode(oldnode) - def apply_its_rules (self): - dirs = [] - ddir = os.getenv('XDG_DATA_HOME', '') - if ddir == '': - ddir = os.path.join(os.path.expanduser('~'), '.local', 'share') - dirs.append(ddir) - ddir = os.getenv('XDG_DATA_DIRS', '') - if ddir == '': - if DATADIR not in ('/usr/local/share', '/usr/share'): - ddir += DATADIR + ':' - ddir += '/usr/local/share:/usr/share' - dirs.extend(ddir.split(':')) - ddone = {} - for ddir in dirs: - itsdir = os.path.join(ddir, 'itstool', 'its') - if not os.path.exists(itsdir): - continue - for dfile in os.listdir(itsdir): - if dfile.endswith('.its'): - if not ddone.get(dfile, False): - self.apply_its_file(os.path.join(itsdir, dfile)) - ddone[dfile] = True - self.apply_local_its_rules() - - def apply_its_file (self, filename): + def apply_its_rules(self, builtins, params={}): + if builtins: + dirs = [] + ddir = os.getenv('XDG_DATA_HOME', '') + if ddir == '': + ddir = os.path.join(os.path.expanduser('~'), '.local', 'share') + dirs.append(ddir) + ddir = os.getenv('XDG_DATA_DIRS', '') + if ddir == '': + if DATADIR not in ('/usr/local/share', '/usr/share'): + ddir += DATADIR + ':' + ddir += '/usr/local/share:/usr/share' + dirs.extend(ddir.split(':')) + ddone = {} + for ddir in dirs: + itsdir = os.path.join(ddir, 'itstool', 'its') + if not os.path.exists(itsdir): + continue + for dfile in os.listdir(itsdir): + if dfile.endswith('.its'): + if not ddone.get(dfile, False): + self.apply_its_file(os.path.join(itsdir, dfile), params=params) + ddone[dfile] = True + self.apply_local_its_rules(params=params) + + def apply_its_file(self, filename, params={}): doc = libxml2.parseFile(filename) root = doc.getRootElement() if not xml_is_ns_name(root, NS_ITS, 'rules'): @@ -585,7 +736,7 @@ class Document (object): else: sys.stderr.write('Warning: ITS file %s missing version attribute\n' % os.path.basename(filename)) - if version is not None and version != '1.0': + if version is not None and version not in ('1.0', '2.0'): sys.stderr.write('Warning: Skipping ITS file %s with unknown version %s\n' % (os.path.basename(filename), root.nsProp('version', None))) return @@ -624,9 +775,10 @@ class Document (object): xpath.xpathRegisterNs(nsdef.name, nsdef.content) nsdef = nsdef.next par = par.parent + self.register_its_params(xpath, root, params=params) self.apply_its_rule(rule, xpath) - def apply_local_its_rules (self): + def apply_local_its_rules(self, params={}): for rules in self._localrules: def reg_ns(xpath, node): if node.parent is not None: @@ -638,12 +790,14 @@ class Document (object): nsdef = nsdef.next xpath = self._doc.xpathNewContext() reg_ns(xpath, rules) + self.register_its_params(xpath, rules, params=params) for rule in xml_child_iter(rules): if rule.type != 'element': continue if rule.nsDefs() is not None: - rule_xpath = self._doc.xpathNewContent() + rule_xpath = self._doc.xpathNewContext() reg_ns(rule_xpath, rule) + self.register_its_params(rule_xpath, rules, params=params) else: rule_xpath = xpath self.apply_its_rule(rule, rule_xpath) @@ -701,8 +855,7 @@ class Document (object): node = self._doc.getRootElement() if node is None or node.type != 'element': return - if ((node.hasNsProp('drop', NS_ITST) and node.nsProp('drop', NS_ITST) == 'yes') or - self._itst_drop_nodes.get(node, 'no') == 'yes'): + if self.get_itst_drop(node) == 'yes': prev = node.prev node.unlinkNode() node.freeNode() @@ -723,6 +876,10 @@ class Document (object): if re.sub('\s+', '', prevtext) == '': prevnode = node.prev for lang in sorted(translations.keys(), reverse=True): + locale = self.get_its_locale_filter(node) + lmatch = match_locale_list(locale[0], lang) + if (locale[1] == 'include' and not lmatch) or (locale[1] == 'exclude' and lmatch): + continue newnode = self.get_translated(node, translations[lang], strict=strict, lang=lang) if newnode != node: newnode.setProp('xml:lang', lang) @@ -744,8 +901,16 @@ class Document (object): node = self._doc.getRootElement() if node is None or node.type != 'element': return - if ((node.hasNsProp('drop', NS_ITST) and node.nsProp('drop', NS_ITST) == 'yes') or - self._itst_drop_nodes.get(node, 'no') == 'yes'): + drop = False + locale = self.get_its_locale_filter(node) + if locale[1] == 'include': + if locale[0] != '*': + if not match_locale_list(locale[0], language): + drop = True + elif locale[1] == 'exclude': + if match_locale_list(locale[0], language): + drop = True + if self.get_itst_drop(node) == 'yes' or drop: prev = node.prev node.unlinkNode() node.freeNode() @@ -826,7 +991,11 @@ class Document (object): nsdef = nsdef.next reg_ns(node, nss) nss['_'] = NS_BLANK - blurb = '<' + node.name + try: + blurb = node.doc.intSubset().serialize('utf-8') + except: + blurb = '' + blurb += '<' + node.name for nsname in nss.keys(): if nsname is None: blurb += ' xmlns="%s"' % nss[nsname] @@ -834,6 +1003,8 @@ class Document (object): blurb += ' xmlns:%s="%s"' % (nsname, nss[nsname]) blurb += '>%s' % (trans.encode('utf-8'), node.name) ctxt = libxml2.createDocParserCtxt(blurb) + if self._load_dtd: + ctxt.loadSubset(1) ctxt.replaceEntities(0) ctxt.parseDocument() trnode = ctxt.doc().getRootElement() @@ -871,36 +1042,6 @@ class Document (object): def generate_messages(self, comments=True): if self._itst_credits is not None: self._msgs.add_credits() - for ext in self._itst_externals: - translate = None - node = ext[0] - while node != None: - translate = self.get_its_translate(node) - if translate is not None: - break - node = node.parent - if translate == 'no': - continue - msg = Message() - try: - fullfile = os.path.join(os.path.dirname(self._filename), ext[1]) - filefp = open(fullfile) - filemd5 = hashlib.md5(filefp.read()).hexdigest() - filefp.close() - except: - filemd5 = '__failed__' - txt = "external ref='%s' md5='%s'" % (ext[1], filemd5) - msg.set_context('_') - msg.add_text(txt) - msg.add_source('%s:%i' % (self._doc.name, ext[0].lineNo())) - msg.add_marker(ext[0].name) - msg.add_comment(Comment('This is a reference to an external file such as an image or' - ' video. When the file changes, the md5 hash will change to' - ' let you know you need to update your localized copy. The' - ' msgstr is not used at all. Set it to whatever you like' - ' once you have updated your copy of the file.')) - self._msgs.add_message(msg, None) - self._in_translatable = True for child in xml_child_iter(self._doc): if child.type == 'element': self.generate_message(child, None, comments=comments) @@ -910,20 +1051,20 @@ class Document (object): if node.type in ('text', 'cdata') and msg is not None: msg.add_text(node.content) return + if node.type == 'entity_ref': + msg.add_entity_ref(node.name); if node.type != 'element': return if node.hasNsProp('drop', NS_ITST) and node.nsProp('drop', NS_ITST) == 'yes': return if self._itst_drop_nodes.get(node, 'no') == 'yes': return + locfil = self.get_its_locale_filter(node) + if locfil == ('', 'include') or locfil == ('*', 'exclude'): + return if path is None: path = '' translate = self.get_its_translate(node) - if translate is None: - if self._in_translatable: - translate = 'yes' - else: - translate = 'no' withinText = False if translate == 'no': if msg is not None: @@ -943,8 +1084,14 @@ class Document (object): ctxt = self._itst_contexts.get(node) if ctxt is not None: msg.set_context(ctxt) + idvalue = self.get_its_id_value(node) + if idvalue is not None: + basename = os.path.basename(self._filename) + msg.add_id_value(basename + '#' + idvalue) if self.get_preserve_space(node): msg.set_preserve_space() + if self.get_its_locale_filter(node) != ('*', 'include'): + msg.set_locale_filter(self.get_its_locale_filter(node)) msg.add_source('%s:%i' % (self._doc.name, node.lineNo())) msg.add_marker('%s/%s' % (node.parent.name, node.name)) else: @@ -971,7 +1118,7 @@ class Document (object): cnode = node while cnode is not None: hasnote = False - for locnote in self.get_its_loc_notes(cnode): + for locnote in self.get_its_loc_notes(cnode, inherit=(not withinText)): comment = Comment(locnote) if withinText: comment.add_marker('.%s/%s' % (path, cnode.name)) @@ -981,13 +1128,18 @@ class Document (object): break cnode = cnode.parent - in_translatable = self._in_translatable - self._in_translatable = (translate == 'yes') + self.generate_external_resource_message(node) + for attr in xml_attr_iter(node): + self.generate_external_resource_message(attr) + idvalue = self.get_its_id_value(attr) + if idvalue is not None: + basename = os.path.basename(self._filename) + msg.add_id_value(basename + '#' + idvalue) + if withinText: path = path + '/' + node.name for child in xml_child_iter(node): self.generate_message(child, msg, comments=comments, path=path) - self._in_translatable = in_translatable if translate: if is_unit and not msg.is_empty(): @@ -995,6 +1147,38 @@ class Document (object): elif msg is not None: msg.add_end_tag(node) + def generate_external_resource_message(self, node): + if not self._its_externals.has_key(node): + return + resref = self._its_externals[node] + if node.type == 'element': + translate = self.get_its_translate(node) + marker = '%s/%s' % (node.parent.name, node.name) + else: + translate = self.get_its_translate(node.parent) + marker = '%s/%s/@%s' % (node.parent.parent.name, node.parent.name, node.name) + if translate == 'no': + return + msg = Message() + try: + fullfile = os.path.join(os.path.dirname(self._filename), resref) + filefp = open(fullfile) + filemd5 = hashlib.md5(filefp.read()).hexdigest() + filefp.close() + except: + filemd5 = '__failed__' + txt = "external ref='%s' md5='%s'" % (resref, filemd5) + msg.set_context('_') + msg.add_text(txt) + msg.add_source('%s:%i' % (self._doc.name, node.lineNo())) + msg.add_marker(marker) + msg.add_comment(Comment('This is a reference to an external file such as an image or' + ' video. When the file changes, the md5 hash will change to' + ' let you know you need to update your localized copy. The' + ' msgstr is not used at all. Set it to whatever you like' + ' once you have updated your copy of the file.')) + self._msgs.add_message(msg, None) + def is_translation_unit (self, node): return self.get_its_within_text(node) != 'yes' @@ -1002,43 +1186,147 @@ class Document (object): return len([child for child in xml_child_iter(node) if child.type=='element']) def get_preserve_space (self, node): - if node.getSpacePreserve() == 1: - return True - else: - while node.type == 'element': - if self._itst_preserve_space_nodes.has_key(node): - return (self._itst_preserve_space_nodes[node] == 'yes') - node = node.parent + while node.type in ('attribute', 'element'): + if node.getSpacePreserve() == 1: + return True + if self._its_preserve_space_nodes.has_key(node): + return (self._its_preserve_space_nodes[node] == 'preserve') + node = node.parent return False - def get_its_translate (self, node): + def get_its_translate(self, node): + val = None if node.hasNsProp('translate', NS_ITS): - return node.nsProp('translate', NS_ITS) - if xml_is_ns_name(node, NS_ITS, 'span'): - if node.hasNsProp('translate', None): - return node.nsProp('translate', None) - if self._its_translate_nodes.has_key(node): - return self._its_translate_nodes[node] - return None + val = node.nsProp('translate', NS_ITS) + elif xml_is_ns_name(node, NS_ITS, 'span') and node.hasNsProp('translate', None): + val = node.nsProp('translate', None) + elif self._its_translate_nodes.has_key(node): + val = self._its_translate_nodes[node] + if val is not None: + return val + if node.type == 'attribute': + return 'no' + if node.parent.type == 'element': + return self.get_its_translate(node.parent) + return 'yes' + + def get_its_within_text(self, node): + if node.hasNsProp('withinText', NS_ITS): + val = node.nsProp('withinText', NS_ITS) + elif xml_is_ns_name(node, NS_ITS, 'span') and node.hasNsProp('withinText', None): + val = node.nsProp('withinText', None) + else: + return self._its_within_text_nodes.get(node, 'no') + if val in ('yes', 'nested'): + return val + return 'no' + + def get_its_locale_filter(self, node): + if node.hasNsProp('localeFilterList', NS_ITS) or node.hasNsProp('localeFilterType', NS_ITS): + if node.hasNsProp('localeFilterList', NS_ITS): + lst = node.nsProp('localeFilterList', NS_ITS) + else: + lst = '*' + if node.hasNsProp('localeFilterType', NS_ITS): + typ = node.nsProp('localeFilterType', NS_ITS) + else: + typ = 'include' + return (lst, typ) + if (xml_is_ns_name(node, NS_ITS, 'span') and + (node.hasNsProp('localeFilterList', None) or node.hasNsProp('localeFilterType', None))): + if node.hasNsProp('localeFilterList', None): + lst = node.nsProp('localeFilterList', None) + else: + lst = '*' + if node.hasNsProp('localeFilterType', None): + typ = node.nsProp('localeFilterType', None) + else: + typ = 'include' + return (lst, typ) + if self._its_locale_filters.has_key(node): + return self._its_locale_filters[node] + if node.parent.type == 'element': + return self.get_its_locale_filter(node.parent) + return ('*', 'include') + + def get_itst_drop(self, node): + if node.hasNsProp('drop', NS_ITST) and node.nsProp('drop', NS_ITST) == 'yes': + return 'yes' + if self._itst_drop_nodes.get(node, 'no') == 'yes': + return 'yes' + return 'no' - def get_its_within_text (self, node): - return self._its_within_text_nodes.get(node, 'no') + def get_its_id_value(self, node): + if node.hasNsProp('id', NS_XML): + return node.nsProp('id', NS_XML) + return self._its_id_values.get(node, None) - def get_its_loc_notes (self, node): + def get_its_loc_notes(self, node, inherit=True): ret = [] - if node.hasNsProp('locNote', NS_ITS): - ret.append(re.sub('\s+', ' ', node.nsProp('locNote', NS_ITS)).strip()) - if node.hasNsProp('locNoteRef', NS_ITS): - ret.append('(itstool) link: ' + re.sub('\s+', ' ', node.nsProp('locNoteRef', NS_ITS)).strip()) - if xml_is_ns_name(node, NS_ITS, 'span'): - if node.hasNsProp('locNote', None): - ret.append(re.sub('\s+', ' ', node.nsProp('locNote', None)).strip()) - if node.hasNsProp('locNoteRef', None): - ret.append('(itstool) link: ' + re.sub('\s+', ' ', node.nsProp('locNoteRef', None)).strip()) - for locnote in self._its_loc_notes.get(node, []): + if node.hasNsProp('locNote', NS_ITS) or node.hasNsProp('locNoteRef', NS_ITS) or node.hasNsProp('locNoteType', NS_ITS): + notetype = node.nsProp('locNoteType', NS_ITS) + if node.hasNsProp('locNote', NS_ITS): + ret.append(LocNote(locnote=node.nsProp('locNote', NS_ITS), locnotetype=notetype)) + elif node.hasNsProp('locNoteRef', NS_ITS): + ret.append(LocNote(locnoteref=node.nsProp('locNoteRef', NS_ITS), locnotetype=notetype)) + elif xml_is_ns_name(node, NS_ITS, 'span'): + if node.hasNsProp('locNote', None) or node.hasNsProp('locNoteRef', None) or node.hasNsProp('locNoteType', None): + notetype = node.nsProp('locNoteType', None) + if node.hasNsProp('locNote', None): + ret.append(LocNote(locnote=node.nsProp('locNote', None), locnotetype=notetype)) + elif node.hasNsProp('locNoteRef', None): + ret.append(LocNote(locnoteref=node.nsProp('locNoteRef', None), locnotetype=notetype)) + for locnote in reversed(self._its_loc_notes.get(node, [])): ret.append(locnote) + if (len(ret) == 0 and inherit and + node.type != 'attribute' and node.parent is not None and node.parent.type == 'element'): + return self.get_its_loc_notes(node.parent) return ret + def output_test_data(self, category, out, node=None): + if node is None: + node = self._doc.getRootElement() + compval = '' + if category == 'translate': + compval = 'translate="%s"' % self.get_its_translate(node) + elif category == 'withinText': + if node.type != 'attribute': + compval = 'withinText="%s"' % self.get_its_within_text(node) + elif category == 'localeFilter': + compval = 'localeFilterList="%s"\tlocaleFilterType="%s"' % self.get_its_locale_filter(node) + elif category == 'locNote': + val = self.get_its_loc_notes(node) + if len(val) > 0: + if val[0].locnote is not None: + compval = 'locNote="%s"\tlocNoteType="%s"' % (str(val[0]), val[0].locnotetype) + elif val[0].locnoteref is not None: + compval = 'locNoteRef="%s"\tlocNoteType="%s"' % (val[0].locnoteref, val[0].locnotetype) + elif category == 'externalResourceRef': + val = self._its_externals.get(node, '') + if val != '': + compval = 'externalResourceRef="%s"' % val + elif category == 'idValue': + val = self.get_its_id_value(node) + if val is not None: + compval = 'idValue="%s"' % val + elif category == 'preserveSpace': + if self.get_preserve_space(node): + compval = 'space="preserve"' + else: + compval = 'space="default"' + else: + sys.stderr.write('Error: Unrecognized category %s\n' % category) + sys.exit(1) + if compval != '': + out.write('%s\t%s\r\n' % (xml_get_node_path(node), compval)) + else: + out.write('%s\r\n' % (xml_get_node_path(node))) + for attr in sorted(xml_attr_iter(node), lambda x, y: cmp(str(x), str(y))): + self.output_test_data(category, out, attr) + for child in xml_child_iter(node): + if child.type == 'element': + self.output_test_data(category, out, child) + @staticmethod def _try_xpath_eval (xpath, expr): try: @@ -1047,6 +1335,37 @@ class Document (object): sys.stderr.write('Warning: Invalid XPath: %s\n' % expr) return [] +def match_locale_list(extranges, locale): + if extranges.strip() == '': + return False + for extrange in [extrange.strip() for extrange in extranges.split(',')]: + if match_locale(extrange, locale): + return True + return False + +def match_locale(extrange, locale): + # Extended filtering for extended language ranges as + # defined by RFC4647, part of BCP47. + # http://tools.ietf.org/html/rfc4647#section-3.3.2 + rangelist = [x.lower() for x in extrange.split('-')] + localelist = [x.lower() for x in locale.split('-')] + if rangelist[0] not in ('*', localelist[0]): + return False + rangei = localei = 0 + while rangei < len(rangelist): + if rangelist[rangei] == '*': + rangei += 1 + continue + if localei >= len(localelist): + return False + if rangelist[rangei] in ('*', localelist[localei]): + rangei += 1 + localei += 1 + continue + if len(localelist[localei]) == 1: + return False + localei += 1 + return True _locale_pattern = re.compile('([a-zA-Z0-9-]+)(_[A-Za-z0-9]+)?(@[A-Za-z0-9]+)?(\.[A-Za-z0-9]+)?') def convert_locale (locale): @@ -1082,50 +1401,82 @@ if __name__ == '__main__': action='append', dest='itsfile', metavar='ITS', - help='load the ITS rules in the file ITS (can specify multiple times)') + help='Load the ITS rules in the file ITS (can specify multiple times)') options.add_option('-l', '--lang', dest='lang', default=None, metavar='LANGUAGE', - help='explicitly set the language code for output file') + help='Explicitly set the language code for output file') options.add_option('-j', '--join', dest='join', metavar='FILE', - help='join multiple MO files with the XML file FILE and output XML file') + help='Join multiple MO files with the XML file FILE and output XML file') options.add_option('-m', '--merge', dest='merge', metavar='FILE', - help='merge from a PO or MO file FILE and output XML files') + help='Merge from a PO or MO file FILE and output XML files') + options.add_option('-n', '--no-builtins', + action='store_true', + dest='nobuiltins', + default=False, + help='Do not apply the built-in ITS rules') options.add_option('-o', '--output', dest='output', default=None, metavar='OUT', - help='output PO files to file OUT or XML files in directory OUT') + help='Output PO files to file OUT or XML files in directory OUT') options.add_option('-s', '--strict', action='store_true', dest='strict', default=False, help='Exit with error when PO files contain broken XML') + options.add_option('-d', '--load-dtd', + action='store_true', + dest='load_dtd', + default=False, + help='Load external DTDs used by input XML') + options.add_option('-k', '--keep-entities', + action='store_true', + dest='keep_entities', + default=False, + help='Keep entity reference unexpanded') + options.add_option('-p', '--param', + action='append', + dest='params', + default=[], + nargs=2, + metavar='NAME VALUE', + help='Define the ITS parameter NAME to the value VALUE (can specify multiple times)') + options.add_option('-t', '--test', + dest='test', + default=None, + metavar='CATEGORY', + help='Generate conformance test output for CATEGORY') options.add_option('-v', '--version', action='store_true', dest='version', default=False, - help='print itstool version and exit') + help='Print itstool version and exit') (opts, args) = options.parse_args(sys.argv) if opts.version: print('itstool %s' % VERSION) sys.exit(0) + params = {} + for name, value in opts.params: + params[name] = value + if opts.merge is None and opts.join is None: messages = MessageList() for filename in args[1:]: - doc = Document(filename, messages) - doc.apply_its_rules() + doc = Document(filename, messages, load_dtd=opts.load_dtd, keep_entities=opts.keep_entities) + doc.apply_its_rules(not(opts.nobuiltins), params=params) if opts.itsfile is not None: for itsfile in opts.itsfile: - doc.apply_its_file(itsfile) - doc.generate_messages() + doc.apply_its_file(itsfile, params=params) + if opts.test is None: + doc.generate_messages() if opts.output is None or opts.output == '-': out = sys.stdout else: @@ -1134,7 +1485,10 @@ if __name__ == '__main__': except: sys.stderr.write('Error: Cannot write to file %s\n' % opts.output) sys.exit(1) - messages.output(out) + if opts.test is not None: + doc.output_test_data(opts.test, out) + else: + messages.output(out) elif opts.merge is not None: try: translations = gettext.GNUTranslations(open(opts.merge, 'rb')) @@ -1158,11 +1512,11 @@ if __name__ == '__main__': sys.exit(1) for filename in args[1:]: messages = MessageList() - doc = Document(filename, messages) - doc.apply_its_rules() + doc = Document(filename, messages, load_dtd=opts.load_dtd, keep_entities=opts.keep_entities) + doc.apply_its_rules(not(opts.nobuiltins), params=params) if opts.itsfile is not None: for itsfile in opts.itsfile: - doc.apply_its_file(itsfile) + doc.apply_its_file(itsfile, params=params) try: doc.merge_translations(translations, opts.lang, strict=opts.strict) except Exception as e: @@ -1191,13 +1545,13 @@ if __name__ == '__main__': out = file(opts.output, 'w') messages = MessageList() doc = Document(opts.join, messages) - doc.apply_its_rules() + doc.apply_its_rules(not(opts.nobuiltins), params=params) doc.join_translations(translations, strict=opts.strict) out.write(doc._doc.serialize('utf-8')) if False: if opts.itsfile is not None: for itsfile in opts.itsfile: - doc.apply_its_file(itsfile) + doc.apply_its_file(itsfile, params=params) try: doc.merge_translations(translations, opts.lang, strict=opts.strict) except Exception as e: diff --git a/itstool.1 b/itstool.1 index 472346c..238ca78 100644 --- a/itstool.1 +++ b/itstool.1 @@ -1,4 +1,4 @@ -.TH ITSTOOL "1" "May 2011" "itstool 1.2.0" +.TH ITSTOOL "1" "May 2011" "itstool 2.0.0" .SH NAME itstool \- convert between XML and PO using ITS diff --git a/itstool.in b/itstool.in index 3a21aaf..150b395 100755 --- a/itstool.in +++ b/itstool.in @@ -34,6 +34,7 @@ NS_ITS = 'http://www.w3.org/2005/11/its' NS_ITST = 'http://itstool.org/extensions/' NS_BLANK = 'http://itstool.org/extensions/blank/' NS_XLINK = 'http://www.w3.org/1999/xlink' +NS_XML = 'http://www.w3.org/XML/1998/namespace' class NoneTranslations: def gettext(self, message): @@ -94,8 +95,17 @@ class MessageList (object): msgdict[key].add_marker(marker) for comment in msg.get_comments(): msgdict[key].add_comment(comment) + for idvalue in msg.get_id_values(): + msgdict[key].add_id_value(idvalue) if msg.get_preserve_space(): msgdict[key].set_preserve_space() + if msg.get_locale_filter() is not None: + locale = msgdict[key].get_locale_filter() + if locale is not None: + msgdict[key].set_locale_filter('%s, %s' % (locale, msg.get_locale_filter())) + else: + msgdict[key].set_locale_filter(msg.get_locale_filter()) + else: msgs.append(msg) msgdict[key] = msg @@ -117,7 +127,7 @@ class MessageList (object): class Comment (object): def __init__ (self, text): - self._text = text + self._text = str(text) assert(text is not None) self._markers = [] @@ -167,6 +177,8 @@ class Message (object): self._placeholders = [] self._sources = [] self._markers = [] + self._id_values = [] + self._locale_filter = None self._comments = [] self._preserve = False @@ -192,6 +204,10 @@ class Message (object): if re.sub('\s+', ' ', text).strip() != '': self._empty = False + def add_entity_ref (self, name): + self._message.append('&' + name + ';') + self._empty = False + def add_placeholder (self, node): holder = Message.Placeholder(node) self._placeholders.append(holder) @@ -208,18 +224,19 @@ class Message (object): def add_start_tag (self, node): if len(self._message) == 0 or not(isinstance(self._message[-1], basestring)): self._message.append('') - self._message[-1] += ('<%s' % node.name) - if node.properties is not None: - for prop in node.properties: - if prop.type == 'attribute': - name = prop.name - if prop.ns() is not None: - name = prop.ns().name + ':' + name - atval = prop.content - if not isinstance(atval, unicode): - atval = unicode(atval, 'utf-8') - atval = atval.replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"') - self._message += " %s=\"%s\"" % (name, atval) + if node.ns() is not None and node.ns().name is not None: + self._message[-1] += (u'<%s:%s' % (unicode(node.ns().name, 'utf-8'), unicode(node.name, 'utf-8'))) + else: + self._message[-1] += (u'<%s' % unicode(node.name, 'utf-8')) + for prop in xml_attr_iter(node): + name = prop.name + if prop.ns() is not None: + name = prop.ns().name + ':' + name + atval = prop.content + if not isinstance(atval, unicode): + atval = unicode(atval, 'utf-8') + atval = atval.replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"') + self._message += " %s=\"%s\"" % (name, atval) if node.children is not None: self._message[-1] += '>' else: @@ -229,7 +246,10 @@ class Message (object): if node.children is not None: if len(self._message) == 0 or not(isinstance(self._message[-1], basestring)): self._message.append('') - self._message[-1] += (u'' % unicode(node.name, 'utf-8')) + if node.ns() is not None and node.ns().name is not None: + self._message[-1] += (u'' % (unicode(node.ns().name, 'utf-8'), unicode(node.name, 'utf-8'))) + else: + self._message[-1] += (u'' % unicode(node.name, 'utf-8')) def is_empty (self): return self._empty @@ -256,6 +276,12 @@ class Message (object): def get_markers (self): return self._markers + def add_id_value(self, id_value): + self._id_values.append(id_value) + + def get_id_values(self): + return self._id_values + def add_comment (self, comment): if comment is not None: self._comments.append(comment) @@ -282,6 +308,12 @@ class Message (object): def set_preserve_space (self, preserve=True): self._preserve = preserve + def get_locale_filter(self): + return self._locale_filter + + def set_locale_filter(self, locale): + self._locale_filter = locale + def format (self): ret = u'' markers = {} @@ -289,6 +321,10 @@ class Message (object): if not markers.has_key(marker): ret += '#. (itstool) path: ' + marker + '\n' markers[marker] = marker + for idvalue in self._id_values: + ret += '#. (itstool) id: ' + idvalue + '\n' + if self._locale_filter is not None: + ret += '#. (itstool) ' + self._locale_filter[1] + ' locale: ' + self._locale_filter[0] + '\n' comments = [] commentsdict = {} for comment in self._comments: @@ -341,6 +377,34 @@ def xml_is_ns_name (node, ns, name): return False return node.name == name and node.ns() is not None and node.ns().content == ns +def xml_get_node_path(node): + # The built-in nodePath() method only does numeric indexes + # when necessary for disambiguation. For various reasons, + # we prefer always using indexes. + name = node.name + if node.ns() is not None and node.ns().name is not None: + name = node.ns().name + ':' + name + if node.type == 'attribute': + name = '@' + name + name = '/' + name + if node.type == 'element' and node.parent.type == 'element': + count = 1 + prev = node.previousElementSibling() + while prev is not None: + if prev.name == node.name: + if prev.ns() is None: + if node.ns() is None: + count += 1 + else: + if node.ns() is not None: + if prev.ns().name == node.ns().name: + count += 1 + prev = prev.previousElementSibling() + name = '%s[%i]' % (name, count) + if node.parent.type == 'element': + name = xml_get_node_path(node.parent) + name + return name + def xml_error_catcher(doc, error): doc._xml_err += " %s" % error @@ -359,8 +423,28 @@ def fix_node_ns (node, nsdefs): fix_node_ns(child, childnsdefs) +class LocNote (object): + def __init__(self, locnote=None, locnoteref=None, locnotetype=None, space=False): + self.locnote = locnote + self.locnoteref = locnoteref + self.locnotetype = locnotetype + if self.locnotetype != 'alert': + self.locnotetype = 'description' + self._preserve_space=space + + def __repr__(self): + if self.locnote is not None: + if self._preserve_space: + return self.locnote + else: + return re.sub('\s+', ' ', self.locnote).strip() + elif self.locnoteref is not None: + return '(itstool) link: ' + re.sub('\s+', ' ', self.locnoteref).strip() + return '' + + class Document (object): - def __init__ (self, filename, messages): + def __init__ (self, filename, messages, load_dtd=False, keep_entities=False): self._xml_err = '' libxml2.registerErrorHandler(xml_error_catcher, self) try: @@ -369,7 +453,14 @@ class Document (object): sys.stderr.write('Error: cannot open XML file %s\n' % filename) sys.exit(1) ctxt.lineNumbers(1) - ctxt.replaceEntities(1) + self._load_dtd = load_dtd + self._keep_entities = keep_entities + if load_dtd: + ctxt.loadSubset(1) + if keep_entities: + ctxt.replaceEntities(0) + else: + ctxt.replaceEntities(1) ctxt.parseDocument() self._filename = filename self._doc = ctxt.doc() @@ -393,7 +484,7 @@ class Document (object): else: sys.stderr.write('Warning: ITS file %s missing version attribute\n' % os.path.basename(href)) - if version is not None and version != '1.0': + if version is not None and version not in ('1.0', '2.0'): sys.stderr.write('Warning: Skipping ITS file %s with unknown version %s\n' % (os.path.basename(href), root.nsProp('version', None))) else: @@ -407,7 +498,7 @@ class Document (object): version = root.nsProp('version', NS_ITS) else: sys.stderr.write('Warning: Local ITS rules missing version attribute\n') - if version is not None and version != '1.0': + if version is not None and version not in ('1.0', '2.0'): sys.stderr.write('Warning: Skipping local ITS rules with unknown version %s\n' % version) else: @@ -422,19 +513,31 @@ class Document (object): self._msgs = messages self._its_translate_nodes = {} self._its_within_text_nodes = {} + self._its_locale_filters = {} + self._its_id_values = {} self._its_loc_notes = {} - self._itst_preserve_space_nodes = {} + self._its_preserve_space_nodes = {} self._itst_drop_nodes = {} self._itst_contexts = {} self._its_lang = {} self._itst_lang_attr = {} self._itst_credits = None - self._itst_externals = [] + self._its_externals = {} def _check_errors(self): if self._xml_err: raise libxml2.parserError(self._xml_err) + def register_its_params(self, xpath, rules, params={}): + for child in xml_child_iter(rules): + if xml_is_ns_name(child, NS_ITS, 'param'): + name = child.nsProp('name', None) + if params.has_key(name): + value = params[name] + else: + value = child.getContent() + xpath.xpathRegisterVariable(name, None, value) + def apply_its_rule(self, rule, xpath): if rule.type != 'element': return @@ -449,11 +552,48 @@ class Document (object): elif xml_is_ns_name(rule, NS_ITST, 'preserveSpaceRule'): if rule.nsProp('selector', None) is not None: for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)): - self._itst_preserve_space_nodes[node] = rule.nsProp('preserveSpace', None) + val = rule.nsProp('preserveSpace', None) + if val == 'yes': + self._its_preserve_space_nodes[node] = 'preserve' + elif xml_is_ns_name(rule, NS_ITS, 'preserveSpaceRule'): + if rule.nsProp('selector', None) is not None: + for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)): + self._its_preserve_space_nodes[node] = rule.nsProp('space', None) + elif xml_is_ns_name(rule, NS_ITS, 'localeFilterRule'): + if rule.nsProp('selector', None) is not None: + if rule.hasNsProp('localeFilterList', None): + lst = rule.nsProp('localeFilterList', None) + else: + lst = '*' + if rule.hasNsProp('localeFilterType', None): + typ = rule.nsProp('localeFilterType', None) + else: + typ = 'include' + for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)): + self._its_locale_filters[node] = (lst, typ) elif xml_is_ns_name(rule, NS_ITST, 'dropRule'): if rule.nsProp('selector', None) is not None: for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)): self._itst_drop_nodes[node] = rule.nsProp('drop', None) + elif xml_is_ns_name(rule, NS_ITS, 'idValueRule'): + sel = rule.nsProp('selector', None) + idv = rule.nsProp('idValue', None) + if sel is not None and idv is not None: + for node in self._try_xpath_eval(xpath, sel): + try: + oldnode = xpath.contextNode() + except: + oldnode = None + xpath.setContextNode(node) + idvalue = self._try_xpath_eval(xpath, idv) + if isinstance(idvalue, basestring): + self._its_id_values[node] = idvalue + else: + for val in idvalue: + self._its_id_values[node] = val.content + break + xpath.setContextNode(oldnode) + pass elif xml_is_ns_name(rule, NS_ITST, 'contextRule'): if rule.nsProp('selector', None) is not None: for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)): @@ -475,13 +615,14 @@ class Document (object): xpath.setContextNode(oldnode) elif xml_is_ns_name(rule, NS_ITS, 'locNoteRule'): locnote = None + notetype = rule.nsProp('locNoteType', None) for child in xml_child_iter(rule): if xml_is_ns_name(child, NS_ITS, 'locNote'): - locnote = re.sub('\s+', ' ', child.content).strip() + locnote = LocNote(locnote=child.content, locnotetype=notetype) break if locnote is None: if rule.hasNsProp('locNoteRef', None): - locnote = '(itstool) link: ' + re.sub('\s+', ' ', rule.nsProp('locNoteRef', None)).strip() + locnote = LocNote(locnoteref=rule.nsProp('locNoteRef', None), locnotetype=notetype) if rule.nsProp('selector', None) is not None: for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)): if locnote is not None: @@ -502,16 +643,19 @@ class Document (object): xpath.setContextNode(node) note = self._try_xpath_eval(xpath, sel) if isinstance(note, basestring): - self._its_loc_notes.setdefault(node, []).append(note) + if ref: + nodenote = LocNote(locnoteref=note, locnotetype=notetype) + else: + nodenote = LocNote(locnote=note, locnotetype=notetype) + self._its_loc_notes.setdefault(node, []).append(nodenote) else: for note in note: - if self.get_preserve_space(note): - cont = note.content - else: - cont = re.sub('\s+', ' ', note.content).strip() if ref: - cont = '(itstool) link: ' + cont - self._its_loc_notes.setdefault(node, []).append(cont) + nodenote = LocNote(locnoteref=note.content, locnotetype=notetype) + else: + nodenote = LocNote(locnote=note.content, locnotetype=notetype, + space=self.get_preserve_space(note)) + self._its_loc_notes.setdefault(node, []).append(nodenote) break xpath.setContextNode(oldnode) elif xml_is_ns_name(rule, NS_ITS, 'langRule'): @@ -537,44 +681,51 @@ class Document (object): for node in self._try_xpath_eval(xpath, rule.nsProp('appendTo', None)): self._itst_credits = (node, rule) break - elif xml_is_ns_name(rule, NS_ITST, 'externalRefRule'): - if rule.nsProp('selector', None) is not None and rule.nsProp('refPointer', None) is not None: - for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)): + elif (xml_is_ns_name(rule, NS_ITS, 'externalResourceRefRule') or + xml_is_ns_name(rule, NS_ITST, 'externalRefRule')): + sel = rule.nsProp('selector', None) + if xml_is_ns_name(rule, NS_ITS, 'externalResourceRefRule'): + ptr = rule.nsProp('externalResourceRefPointer', None) + else: + ptr = rule.nsProp('refPointer', None) + if sel is not None and ptr is not None: + for node in self._try_xpath_eval(xpath, sel): try: oldnode = xpath.contextNode() except: oldnode = None xpath.setContextNode(node) - res = self._try_xpath_eval(xpath, rule.nsProp('refPointer', None)) + res = self._try_xpath_eval(xpath, ptr) if len(res) > 0: - self._itst_externals.append((node, res[0].content)) + self._its_externals[node] = res[0].content xpath.setContextNode(oldnode) - def apply_its_rules (self): - dirs = [] - ddir = os.getenv('XDG_DATA_HOME', '') - if ddir == '': - ddir = os.path.join(os.path.expanduser('~'), '.local', 'share') - dirs.append(ddir) - ddir = os.getenv('XDG_DATA_DIRS', '') - if ddir == '': - if DATADIR not in ('/usr/local/share', '/usr/share'): - ddir += DATADIR + ':' - ddir += '/usr/local/share:/usr/share' - dirs.extend(ddir.split(':')) - ddone = {} - for ddir in dirs: - itsdir = os.path.join(ddir, 'itstool', 'its') - if not os.path.exists(itsdir): - continue - for dfile in os.listdir(itsdir): - if dfile.endswith('.its'): - if not ddone.get(dfile, False): - self.apply_its_file(os.path.join(itsdir, dfile)) - ddone[dfile] = True - self.apply_local_its_rules() - - def apply_its_file (self, filename): + def apply_its_rules(self, builtins, params={}): + if builtins: + dirs = [] + ddir = os.getenv('XDG_DATA_HOME', '') + if ddir == '': + ddir = os.path.join(os.path.expanduser('~'), '.local', 'share') + dirs.append(ddir) + ddir = os.getenv('XDG_DATA_DIRS', '') + if ddir == '': + if DATADIR not in ('/usr/local/share', '/usr/share'): + ddir += DATADIR + ':' + ddir += '/usr/local/share:/usr/share' + dirs.extend(ddir.split(':')) + ddone = {} + for ddir in dirs: + itsdir = os.path.join(ddir, 'itstool', 'its') + if not os.path.exists(itsdir): + continue + for dfile in os.listdir(itsdir): + if dfile.endswith('.its'): + if not ddone.get(dfile, False): + self.apply_its_file(os.path.join(itsdir, dfile), params=params) + ddone[dfile] = True + self.apply_local_its_rules(params=params) + + def apply_its_file(self, filename, params={}): doc = libxml2.parseFile(filename) root = doc.getRootElement() if not xml_is_ns_name(root, NS_ITS, 'rules'): @@ -585,7 +736,7 @@ class Document (object): else: sys.stderr.write('Warning: ITS file %s missing version attribute\n' % os.path.basename(filename)) - if version is not None and version != '1.0': + if version is not None and version not in ('1.0', '2.0'): sys.stderr.write('Warning: Skipping ITS file %s with unknown version %s\n' % (os.path.basename(filename), root.nsProp('version', None))) return @@ -624,9 +775,10 @@ class Document (object): xpath.xpathRegisterNs(nsdef.name, nsdef.content) nsdef = nsdef.next par = par.parent + self.register_its_params(xpath, root, params=params) self.apply_its_rule(rule, xpath) - def apply_local_its_rules (self): + def apply_local_its_rules(self, params={}): for rules in self._localrules: def reg_ns(xpath, node): if node.parent is not None: @@ -638,12 +790,14 @@ class Document (object): nsdef = nsdef.next xpath = self._doc.xpathNewContext() reg_ns(xpath, rules) + self.register_its_params(xpath, rules, params=params) for rule in xml_child_iter(rules): if rule.type != 'element': continue if rule.nsDefs() is not None: - rule_xpath = self._doc.xpathNewContent() + rule_xpath = self._doc.xpathNewContext() reg_ns(rule_xpath, rule) + self.register_its_params(rule_xpath, rules, params=params) else: rule_xpath = xpath self.apply_its_rule(rule, rule_xpath) @@ -701,8 +855,7 @@ class Document (object): node = self._doc.getRootElement() if node is None or node.type != 'element': return - if ((node.hasNsProp('drop', NS_ITST) and node.nsProp('drop', NS_ITST) == 'yes') or - self._itst_drop_nodes.get(node, 'no') == 'yes'): + if self.get_itst_drop(node) == 'yes': prev = node.prev node.unlinkNode() node.freeNode() @@ -723,6 +876,10 @@ class Document (object): if re.sub('\s+', '', prevtext) == '': prevnode = node.prev for lang in sorted(translations.keys(), reverse=True): + locale = self.get_its_locale_filter(node) + lmatch = match_locale_list(locale[0], lang) + if (locale[1] == 'include' and not lmatch) or (locale[1] == 'exclude' and lmatch): + continue newnode = self.get_translated(node, translations[lang], strict=strict, lang=lang) if newnode != node: newnode.setProp('xml:lang', lang) @@ -744,8 +901,16 @@ class Document (object): node = self._doc.getRootElement() if node is None or node.type != 'element': return - if ((node.hasNsProp('drop', NS_ITST) and node.nsProp('drop', NS_ITST) == 'yes') or - self._itst_drop_nodes.get(node, 'no') == 'yes'): + drop = False + locale = self.get_its_locale_filter(node) + if locale[1] == 'include': + if locale[0] != '*': + if not match_locale_list(locale[0], language): + drop = True + elif locale[1] == 'exclude': + if match_locale_list(locale[0], language): + drop = True + if self.get_itst_drop(node) == 'yes' or drop: prev = node.prev node.unlinkNode() node.freeNode() @@ -826,7 +991,11 @@ class Document (object): nsdef = nsdef.next reg_ns(node, nss) nss['_'] = NS_BLANK - blurb = '<' + node.name + try: + blurb = node.doc.intSubset().serialize('utf-8') + except: + blurb = '' + blurb += '<' + node.name for nsname in nss.keys(): if nsname is None: blurb += ' xmlns="%s"' % nss[nsname] @@ -834,6 +1003,8 @@ class Document (object): blurb += ' xmlns:%s="%s"' % (nsname, nss[nsname]) blurb += '>%s' % (trans.encode('utf-8'), node.name) ctxt = libxml2.createDocParserCtxt(blurb) + if self._load_dtd: + ctxt.loadSubset(1) ctxt.replaceEntities(0) ctxt.parseDocument() trnode = ctxt.doc().getRootElement() @@ -871,36 +1042,6 @@ class Document (object): def generate_messages(self, comments=True): if self._itst_credits is not None: self._msgs.add_credits() - for ext in self._itst_externals: - translate = None - node = ext[0] - while node != None: - translate = self.get_its_translate(node) - if translate is not None: - break - node = node.parent - if translate == 'no': - continue - msg = Message() - try: - fullfile = os.path.join(os.path.dirname(self._filename), ext[1]) - filefp = open(fullfile) - filemd5 = hashlib.md5(filefp.read()).hexdigest() - filefp.close() - except: - filemd5 = '__failed__' - txt = "external ref='%s' md5='%s'" % (ext[1], filemd5) - msg.set_context('_') - msg.add_text(txt) - msg.add_source('%s:%i' % (self._doc.name, ext[0].lineNo())) - msg.add_marker(ext[0].name) - msg.add_comment(Comment('This is a reference to an external file such as an image or' - ' video. When the file changes, the md5 hash will change to' - ' let you know you need to update your localized copy. The' - ' msgstr is not used at all. Set it to whatever you like' - ' once you have updated your copy of the file.')) - self._msgs.add_message(msg, None) - self._in_translatable = True for child in xml_child_iter(self._doc): if child.type == 'element': self.generate_message(child, None, comments=comments) @@ -910,20 +1051,20 @@ class Document (object): if node.type in ('text', 'cdata') and msg is not None: msg.add_text(node.content) return + if node.type == 'entity_ref': + msg.add_entity_ref(node.name); if node.type != 'element': return if node.hasNsProp('drop', NS_ITST) and node.nsProp('drop', NS_ITST) == 'yes': return if self._itst_drop_nodes.get(node, 'no') == 'yes': return + locfil = self.get_its_locale_filter(node) + if locfil == ('', 'include') or locfil == ('*', 'exclude'): + return if path is None: path = '' translate = self.get_its_translate(node) - if translate is None: - if self._in_translatable: - translate = 'yes' - else: - translate = 'no' withinText = False if translate == 'no': if msg is not None: @@ -943,8 +1084,14 @@ class Document (object): ctxt = self._itst_contexts.get(node) if ctxt is not None: msg.set_context(ctxt) + idvalue = self.get_its_id_value(node) + if idvalue is not None: + basename = os.path.basename(self._filename) + msg.add_id_value(basename + '#' + idvalue) if self.get_preserve_space(node): msg.set_preserve_space() + if self.get_its_locale_filter(node) != ('*', 'include'): + msg.set_locale_filter(self.get_its_locale_filter(node)) msg.add_source('%s:%i' % (self._doc.name, node.lineNo())) msg.add_marker('%s/%s' % (node.parent.name, node.name)) else: @@ -971,7 +1118,7 @@ class Document (object): cnode = node while cnode is not None: hasnote = False - for locnote in self.get_its_loc_notes(cnode): + for locnote in self.get_its_loc_notes(cnode, inherit=(not withinText)): comment = Comment(locnote) if withinText: comment.add_marker('.%s/%s' % (path, cnode.name)) @@ -981,13 +1128,18 @@ class Document (object): break cnode = cnode.parent - in_translatable = self._in_translatable - self._in_translatable = (translate == 'yes') + self.generate_external_resource_message(node) + for attr in xml_attr_iter(node): + self.generate_external_resource_message(attr) + idvalue = self.get_its_id_value(attr) + if idvalue is not None: + basename = os.path.basename(self._filename) + msg.add_id_value(basename + '#' + idvalue) + if withinText: path = path + '/' + node.name for child in xml_child_iter(node): self.generate_message(child, msg, comments=comments, path=path) - self._in_translatable = in_translatable if translate: if is_unit and not msg.is_empty(): @@ -995,6 +1147,38 @@ class Document (object): elif msg is not None: msg.add_end_tag(node) + def generate_external_resource_message(self, node): + if not self._its_externals.has_key(node): + return + resref = self._its_externals[node] + if node.type == 'element': + translate = self.get_its_translate(node) + marker = '%s/%s' % (node.parent.name, node.name) + else: + translate = self.get_its_translate(node.parent) + marker = '%s/%s/@%s' % (node.parent.parent.name, node.parent.name, node.name) + if translate == 'no': + return + msg = Message() + try: + fullfile = os.path.join(os.path.dirname(self._filename), resref) + filefp = open(fullfile) + filemd5 = hashlib.md5(filefp.read()).hexdigest() + filefp.close() + except: + filemd5 = '__failed__' + txt = "external ref='%s' md5='%s'" % (resref, filemd5) + msg.set_context('_') + msg.add_text(txt) + msg.add_source('%s:%i' % (self._doc.name, node.lineNo())) + msg.add_marker(marker) + msg.add_comment(Comment('This is a reference to an external file such as an image or' + ' video. When the file changes, the md5 hash will change to' + ' let you know you need to update your localized copy. The' + ' msgstr is not used at all. Set it to whatever you like' + ' once you have updated your copy of the file.')) + self._msgs.add_message(msg, None) + def is_translation_unit (self, node): return self.get_its_within_text(node) != 'yes' @@ -1002,43 +1186,147 @@ class Document (object): return len([child for child in xml_child_iter(node) if child.type=='element']) def get_preserve_space (self, node): - if node.getSpacePreserve() == 1: - return True - else: - while node.type == 'element': - if self._itst_preserve_space_nodes.has_key(node): - return (self._itst_preserve_space_nodes[node] == 'yes') - node = node.parent + while node.type in ('attribute', 'element'): + if node.getSpacePreserve() == 1: + return True + if self._its_preserve_space_nodes.has_key(node): + return (self._its_preserve_space_nodes[node] == 'preserve') + node = node.parent return False - def get_its_translate (self, node): + def get_its_translate(self, node): + val = None if node.hasNsProp('translate', NS_ITS): - return node.nsProp('translate', NS_ITS) - if xml_is_ns_name(node, NS_ITS, 'span'): - if node.hasNsProp('translate', None): - return node.nsProp('translate', None) - if self._its_translate_nodes.has_key(node): - return self._its_translate_nodes[node] - return None + val = node.nsProp('translate', NS_ITS) + elif xml_is_ns_name(node, NS_ITS, 'span') and node.hasNsProp('translate', None): + val = node.nsProp('translate', None) + elif self._its_translate_nodes.has_key(node): + val = self._its_translate_nodes[node] + if val is not None: + return val + if node.type == 'attribute': + return 'no' + if node.parent.type == 'element': + return self.get_its_translate(node.parent) + return 'yes' + + def get_its_within_text(self, node): + if node.hasNsProp('withinText', NS_ITS): + val = node.nsProp('withinText', NS_ITS) + elif xml_is_ns_name(node, NS_ITS, 'span') and node.hasNsProp('withinText', None): + val = node.nsProp('withinText', None) + else: + return self._its_within_text_nodes.get(node, 'no') + if val in ('yes', 'nested'): + return val + return 'no' + + def get_its_locale_filter(self, node): + if node.hasNsProp('localeFilterList', NS_ITS) or node.hasNsProp('localeFilterType', NS_ITS): + if node.hasNsProp('localeFilterList', NS_ITS): + lst = node.nsProp('localeFilterList', NS_ITS) + else: + lst = '*' + if node.hasNsProp('localeFilterType', NS_ITS): + typ = node.nsProp('localeFilterType', NS_ITS) + else: + typ = 'include' + return (lst, typ) + if (xml_is_ns_name(node, NS_ITS, 'span') and + (node.hasNsProp('localeFilterList', None) or node.hasNsProp('localeFilterType', None))): + if node.hasNsProp('localeFilterList', None): + lst = node.nsProp('localeFilterList', None) + else: + lst = '*' + if node.hasNsProp('localeFilterType', None): + typ = node.nsProp('localeFilterType', None) + else: + typ = 'include' + return (lst, typ) + if self._its_locale_filters.has_key(node): + return self._its_locale_filters[node] + if node.parent.type == 'element': + return self.get_its_locale_filter(node.parent) + return ('*', 'include') + + def get_itst_drop(self, node): + if node.hasNsProp('drop', NS_ITST) and node.nsProp('drop', NS_ITST) == 'yes': + return 'yes' + if self._itst_drop_nodes.get(node, 'no') == 'yes': + return 'yes' + return 'no' - def get_its_within_text (self, node): - return self._its_within_text_nodes.get(node, 'no') + def get_its_id_value(self, node): + if node.hasNsProp('id', NS_XML): + return node.nsProp('id', NS_XML) + return self._its_id_values.get(node, None) - def get_its_loc_notes (self, node): + def get_its_loc_notes(self, node, inherit=True): ret = [] - if node.hasNsProp('locNote', NS_ITS): - ret.append(re.sub('\s+', ' ', node.nsProp('locNote', NS_ITS)).strip()) - if node.hasNsProp('locNoteRef', NS_ITS): - ret.append('(itstool) link: ' + re.sub('\s+', ' ', node.nsProp('locNoteRef', NS_ITS)).strip()) - if xml_is_ns_name(node, NS_ITS, 'span'): - if node.hasNsProp('locNote', None): - ret.append(re.sub('\s+', ' ', node.nsProp('locNote', None)).strip()) - if node.hasNsProp('locNoteRef', None): - ret.append('(itstool) link: ' + re.sub('\s+', ' ', node.nsProp('locNoteRef', None)).strip()) - for locnote in self._its_loc_notes.get(node, []): + if node.hasNsProp('locNote', NS_ITS) or node.hasNsProp('locNoteRef', NS_ITS) or node.hasNsProp('locNoteType', NS_ITS): + notetype = node.nsProp('locNoteType', NS_ITS) + if node.hasNsProp('locNote', NS_ITS): + ret.append(LocNote(locnote=node.nsProp('locNote', NS_ITS), locnotetype=notetype)) + elif node.hasNsProp('locNoteRef', NS_ITS): + ret.append(LocNote(locnoteref=node.nsProp('locNoteRef', NS_ITS), locnotetype=notetype)) + elif xml_is_ns_name(node, NS_ITS, 'span'): + if node.hasNsProp('locNote', None) or node.hasNsProp('locNoteRef', None) or node.hasNsProp('locNoteType', None): + notetype = node.nsProp('locNoteType', None) + if node.hasNsProp('locNote', None): + ret.append(LocNote(locnote=node.nsProp('locNote', None), locnotetype=notetype)) + elif node.hasNsProp('locNoteRef', None): + ret.append(LocNote(locnoteref=node.nsProp('locNoteRef', None), locnotetype=notetype)) + for locnote in reversed(self._its_loc_notes.get(node, [])): ret.append(locnote) + if (len(ret) == 0 and inherit and + node.type != 'attribute' and node.parent is not None and node.parent.type == 'element'): + return self.get_its_loc_notes(node.parent) return ret + def output_test_data(self, category, out, node=None): + if node is None: + node = self._doc.getRootElement() + compval = '' + if category == 'translate': + compval = 'translate="%s"' % self.get_its_translate(node) + elif category == 'withinText': + if node.type != 'attribute': + compval = 'withinText="%s"' % self.get_its_within_text(node) + elif category == 'localeFilter': + compval = 'localeFilterList="%s"\tlocaleFilterType="%s"' % self.get_its_locale_filter(node) + elif category == 'locNote': + val = self.get_its_loc_notes(node) + if len(val) > 0: + if val[0].locnote is not None: + compval = 'locNote="%s"\tlocNoteType="%s"' % (str(val[0]), val[0].locnotetype) + elif val[0].locnoteref is not None: + compval = 'locNoteRef="%s"\tlocNoteType="%s"' % (val[0].locnoteref, val[0].locnotetype) + elif category == 'externalResourceRef': + val = self._its_externals.get(node, '') + if val != '': + compval = 'externalResourceRef="%s"' % val + elif category == 'idValue': + val = self.get_its_id_value(node) + if val is not None: + compval = 'idValue="%s"' % val + elif category == 'preserveSpace': + if self.get_preserve_space(node): + compval = 'space="preserve"' + else: + compval = 'space="default"' + else: + sys.stderr.write('Error: Unrecognized category %s\n' % category) + sys.exit(1) + if compval != '': + out.write('%s\t%s\r\n' % (xml_get_node_path(node), compval)) + else: + out.write('%s\r\n' % (xml_get_node_path(node))) + for attr in sorted(xml_attr_iter(node), lambda x, y: cmp(str(x), str(y))): + self.output_test_data(category, out, attr) + for child in xml_child_iter(node): + if child.type == 'element': + self.output_test_data(category, out, child) + @staticmethod def _try_xpath_eval (xpath, expr): try: @@ -1047,6 +1335,37 @@ class Document (object): sys.stderr.write('Warning: Invalid XPath: %s\n' % expr) return [] +def match_locale_list(extranges, locale): + if extranges.strip() == '': + return False + for extrange in [extrange.strip() for extrange in extranges.split(',')]: + if match_locale(extrange, locale): + return True + return False + +def match_locale(extrange, locale): + # Extended filtering for extended language ranges as + # defined by RFC4647, part of BCP47. + # http://tools.ietf.org/html/rfc4647#section-3.3.2 + rangelist = [x.lower() for x in extrange.split('-')] + localelist = [x.lower() for x in locale.split('-')] + if rangelist[0] not in ('*', localelist[0]): + return False + rangei = localei = 0 + while rangei < len(rangelist): + if rangelist[rangei] == '*': + rangei += 1 + continue + if localei >= len(localelist): + return False + if rangelist[rangei] in ('*', localelist[localei]): + rangei += 1 + localei += 1 + continue + if len(localelist[localei]) == 1: + return False + localei += 1 + return True _locale_pattern = re.compile('([a-zA-Z0-9-]+)(_[A-Za-z0-9]+)?(@[A-Za-z0-9]+)?(\.[A-Za-z0-9]+)?') def convert_locale (locale): @@ -1082,50 +1401,82 @@ if __name__ == '__main__': action='append', dest='itsfile', metavar='ITS', - help='load the ITS rules in the file ITS (can specify multiple times)') + help='Load the ITS rules in the file ITS (can specify multiple times)') options.add_option('-l', '--lang', dest='lang', default=None, metavar='LANGUAGE', - help='explicitly set the language code for output file') + help='Explicitly set the language code for output file') options.add_option('-j', '--join', dest='join', metavar='FILE', - help='join multiple MO files with the XML file FILE and output XML file') + help='Join multiple MO files with the XML file FILE and output XML file') options.add_option('-m', '--merge', dest='merge', metavar='FILE', - help='merge from a PO or MO file FILE and output XML files') + help='Merge from a PO or MO file FILE and output XML files') + options.add_option('-n', '--no-builtins', + action='store_true', + dest='nobuiltins', + default=False, + help='Do not apply the built-in ITS rules') options.add_option('-o', '--output', dest='output', default=None, metavar='OUT', - help='output PO files to file OUT or XML files in directory OUT') + help='Output PO files to file OUT or XML files in directory OUT') options.add_option('-s', '--strict', action='store_true', dest='strict', default=False, help='Exit with error when PO files contain broken XML') + options.add_option('-d', '--load-dtd', + action='store_true', + dest='load_dtd', + default=False, + help='Load external DTDs used by input XML') + options.add_option('-k', '--keep-entities', + action='store_true', + dest='keep_entities', + default=False, + help='Keep entity reference unexpanded') + options.add_option('-p', '--param', + action='append', + dest='params', + default=[], + nargs=2, + metavar='NAME VALUE', + help='Define the ITS parameter NAME to the value VALUE (can specify multiple times)') + options.add_option('-t', '--test', + dest='test', + default=None, + metavar='CATEGORY', + help='Generate conformance test output for CATEGORY') options.add_option('-v', '--version', action='store_true', dest='version', default=False, - help='print itstool version and exit') + help='Print itstool version and exit') (opts, args) = options.parse_args(sys.argv) if opts.version: print('itstool %s' % VERSION) sys.exit(0) + params = {} + for name, value in opts.params: + params[name] = value + if opts.merge is None and opts.join is None: messages = MessageList() for filename in args[1:]: - doc = Document(filename, messages) - doc.apply_its_rules() + doc = Document(filename, messages, load_dtd=opts.load_dtd, keep_entities=opts.keep_entities) + doc.apply_its_rules(not(opts.nobuiltins), params=params) if opts.itsfile is not None: for itsfile in opts.itsfile: - doc.apply_its_file(itsfile) - doc.generate_messages() + doc.apply_its_file(itsfile, params=params) + if opts.test is None: + doc.generate_messages() if opts.output is None or opts.output == '-': out = sys.stdout else: @@ -1134,7 +1485,10 @@ if __name__ == '__main__': except: sys.stderr.write('Error: Cannot write to file %s\n' % opts.output) sys.exit(1) - messages.output(out) + if opts.test is not None: + doc.output_test_data(opts.test, out) + else: + messages.output(out) elif opts.merge is not None: try: translations = gettext.GNUTranslations(open(opts.merge, 'rb')) @@ -1158,11 +1512,11 @@ if __name__ == '__main__': sys.exit(1) for filename in args[1:]: messages = MessageList() - doc = Document(filename, messages) - doc.apply_its_rules() + doc = Document(filename, messages, load_dtd=opts.load_dtd, keep_entities=opts.keep_entities) + doc.apply_its_rules(not(opts.nobuiltins), params=params) if opts.itsfile is not None: for itsfile in opts.itsfile: - doc.apply_its_file(itsfile) + doc.apply_its_file(itsfile, params=params) try: doc.merge_translations(translations, opts.lang, strict=opts.strict) except Exception as e: @@ -1191,13 +1545,13 @@ if __name__ == '__main__': out = file(opts.output, 'w') messages = MessageList() doc = Document(opts.join, messages) - doc.apply_its_rules() + doc.apply_its_rules(not(opts.nobuiltins), params=params) doc.join_translations(translations, strict=opts.strict) out.write(doc._doc.serialize('utf-8')) if False: if opts.itsfile is not None: for itsfile in opts.itsfile: - doc.apply_its_file(itsfile) + doc.apply_its_file(itsfile, params=params) try: doc.merge_translations(translations, opts.lang, strict=opts.strict) except Exception as e: