src/xmlpatterns/parser/qxslttokenizer.cpp

   1 /****************************************************************************
   2 **
   3 ** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies).
   4 ** All rights reserved.
   5 ** Contact: Nokia Corporation (qt-info@nokia.com)
   6 **
   7 ** This file is part of the QtXmlPatterns module of the Qt Toolkit.
   8 **
   9 ** $QT_BEGIN_LICENSE:LGPL$
  10 ** GNU Lesser General Public License Usage
  11 ** This file may be used under the terms of the GNU Lesser General Public
  12 ** License version 2.1 as published by the Free Software Foundation and
  13 ** appearing in the file LICENSE.LGPL included in the packaging of this
  14 ** file. Please review the following information to ensure the GNU Lesser
  15 ** General Public License version 2.1 requirements will be met:
  16 ** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
  17 **
  18 ** In addition, as a special exception, Nokia gives you certain additional
  19 ** rights. These rights are described in the Nokia Qt LGPL Exception
  20 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
  21 **
  22 ** GNU General Public License Usage
  23 ** Alternatively, this file may be used under the terms of the GNU General
  24 ** Public License version 3.0 as published by the Free Software Foundation
  25 ** and appearing in the file LICENSE.GPL included in the packaging of this
  26 ** file. Please review the following information to ensure the GNU General
  27 ** Public License version 3.0 requirements will be met:
  28 ** http://www.gnu.org/copyleft/gpl.html.
  29 **
  30 ** Other Usage
  31 ** Alternatively, this file may be used in accordance with the terms and
  32 ** conditions contained in a signed written agreement between you and Nokia.
  33 **
  34 **
  35 **
  36 **
  37 **
  38 ** $QT_END_LICENSE$
  39 **
  40 ****************************************************************************/
  41
  42 #include <QStringList>
  43
  44 #include "qbuiltintypes_p.h"
  45 #include "qcommonnamespaces_p.h"
  46 #include "qquerytransformparser_p.h"
  47 #include "qxquerytokenizer_p.h"
  48 #include "qpatternistlocale_p.h"
  49
  50 #include "qxslttokenizer_p.h"
  51
  52 QT_BEGIN_NAMESPACE
  53
  54 using namespace QPatternist;
  55
  56 Tokenizer::Token SingleTokenContainer::nextToken(YYLTYPE *const location)
  57 {
  58     if(m_hasDelivered)
  59         return Tokenizer::Token(END_OF_FILE);
  60     else
  61     {
  62         *location = m_location;
  63         m_hasDelivered = true;
  64         return m_token;
  65     }
  66 }
  67
  68 XSLTTokenizer::XSLTTokenizer(QIODevice *const queryDevice,
  69                              const QUrl &location,
  70                              const ReportContext::Ptr &context,
  71                              const NamePool::Ptr &np) : Tokenizer(location)
  72                                                       , MaintainingReader<XSLTTokenLookup>(createElementDescriptions(), createStandardAttributes(), context, queryDevice)
  73                                                       , m_location(location)
  74                                                       , m_namePool(np)
  75                                                       /* We initialize after all name constants. */
  76                                                       , m_validationAlternatives(createValidationAlternatives())
  77                                                       , m_parseInfo(0)
  78 {
  79     Q_ASSERT(m_namePool);
  80
  81     pushState(OutsideDocumentElement);
  82 }
  83
  84 bool XSLTTokenizer::isAnyAttributeAllowed() const
  85 {
  86     return m_processingMode.top() == ForwardCompatible;
  87 }
  88
  89 void XSLTTokenizer::setParserContext(const ParserContext::Ptr &parseInfo)
  90 {
  91     m_parseInfo = parseInfo;
  92 }
  93
  94 void XSLTTokenizer::validateElement() const
  95 {
  96     MaintainingReader<XSLTTokenLookup>::validateElement(currentElementName());
  97 }
  98
  99 QSet<XSLTTokenizer::NodeName> XSLTTokenizer::createStandardAttributes()
 100 {
 101     QSet<NodeName> retval;
 102     enum
 103     {
 104         ReservedForAttributes = 6
 105     };
 106
 107     retval.reserve(6);
 108
 109     retval.insert(DefaultCollation);
 110     retval.insert(ExcludeResultPrefixes);
 111     retval.insert(ExtensionElementPrefixes);
 112     retval.insert(UseWhen);
 113     retval.insert(Version);
 114     retval.insert(XpathDefaultNamespace);
 115
 116     Q_ASSERT(retval.count() == ReservedForAttributes);
 117
 118     return retval;
 119 }
 120
 121 ElementDescription<XSLTTokenLookup>::Hash XSLTTokenizer::createElementDescriptions()
 122 {
 123     ElementDescription<XSLTTokenLookup>::Hash result;
 124     enum
 125     {
 126         ReservedForElements = 40
 127     };
 128     result.reserve(ReservedForElements);
 129
 130     /* xsl:apply-templates */
 131     {
 132         ElementDescription<XSLTTokenLookup> &e = result[ApplyTemplates];
 133         e.optionalAttributes.insert(Select);
 134         e.optionalAttributes.insert(Mode);
 135     }
 136
 137     /* xsl:template */
 138     {
 139         ElementDescription<XSLTTokenLookup> &e = result[Template];
 140         e.optionalAttributes.insert(Match);
 141         e.optionalAttributes.insert(Name);
 142         e.optionalAttributes.insert(Mode);
 143         e.optionalAttributes.insert(Priority);
 144         e.optionalAttributes.insert(As);
 145     }
 146
 147     /* xsl:text, xsl:choose and xsl:otherwise */
 148     {
 149         ElementDescription<XSLTTokenLookup> &e = result[Text];
 150         result.insert(Choose, e);
 151         result.insert(Otherwise, e);
 152     }
 153
 154     /* xsl:stylesheet */
 155     {
 156         ElementDescription<XSLTTokenLookup> &e = result[Stylesheet];
 157
 158         e.requiredAttributes.insert(Version);
 159
 160         e.optionalAttributes.insert(Id);
 161         e.optionalAttributes.insert(ExtensionElementPrefixes);
 162         e.optionalAttributes.insert(ExcludeResultPrefixes);
 163         e.optionalAttributes.insert(XpathDefaultNamespace);
 164         e.optionalAttributes.insert(DefaultValidation);
 165         e.optionalAttributes.insert(DefaultCollation);
 166         e.optionalAttributes.insert(InputTypeAnnotations);
 167     }
 168
 169     /* xsl:transform */
 170     {
 171         result[Transform] = result[Stylesheet];
 172     }
 173
 174     /* xsl:value-of */
 175     {
 176         ElementDescription<XSLTTokenLookup> &e = result[ValueOf];
 177         e.optionalAttributes.insert(Separator);
 178         e.optionalAttributes.insert(Select);
 179     }
 180
 181     /* xsl:variable */
 182     {
 183         ElementDescription<XSLTTokenLookup> &e = result[Variable];
 184
 185         e.requiredAttributes.insert(Name);
 186
 187         e.optionalAttributes.insert(Select);
 188         e.optionalAttributes.insert(As);
 189     }
 190
 191     /* xsl:when & xsl:if */
 192     {
 193         ElementDescription<XSLTTokenLookup> &e = result[When];
 194
 195         e.requiredAttributes.insert(Test);
 196
 197         result.insert(If, e);
 198     }
 199
 200     /* xsl:sequence, xsl:for-each */
 201     {
 202         ElementDescription<XSLTTokenLookup> &e = result[Sequence];
 203
 204         e.requiredAttributes.insert(Select);
 205
 206         result.insert(ForEach, e);
 207     }
 208
 209     /* xsl:comment */
 210     {
 211         ElementDescription<XSLTTokenLookup> &e = result[XSLTTokenLookup::Comment];
 212
 213         e.optionalAttributes.insert(Select);
 214     }
 215
 216     /* xsl:processing-instruction */
 217     {
 218         ElementDescription<XSLTTokenLookup> &e = result[XSLTTokenLookup::ProcessingInstruction];
 219
 220         e.requiredAttributes.insert(Name);
 221         e.optionalAttributes.insert(Select);
 222     }
 223
 224     /* xsl:document */
 225     {
 226         ElementDescription<XSLTTokenLookup> &e = result[Document];
 227
 228         e.optionalAttributes.insert(Validation);
 229         e.optionalAttributes.insert(Type);
 230     }
 231
 232     /* xsl:element */
 233     {
 234         ElementDescription<XSLTTokenLookup> &e = result[Element];
 235
 236         e.requiredAttributes.insert(Name);
 237
 238         e.optionalAttributes.insert(Namespace);
 239         e.optionalAttributes.insert(InheritNamespaces);
 240         e.optionalAttributes.insert(UseAttributeSets);
 241         e.optionalAttributes.insert(Validation);
 242         e.optionalAttributes.insert(Type);
 243     }
 244
 245     /* xsl:attribute */
 246     {
 247         ElementDescription<XSLTTokenLookup> &e = result[Attribute];
 248
 249         e.requiredAttributes.insert(Name);
 250
 251         e.optionalAttributes.insert(Namespace);
 252         e.optionalAttributes.insert(Select);
 253         e.optionalAttributes.insert(Separator);
 254         e.optionalAttributes.insert(Validation);
 255         e.optionalAttributes.insert(Type);
 256     }
 257
 258     /* xsl:function */
 259     {
 260         ElementDescription<XSLTTokenLookup> &e = result[Function];
 261
 262         e.requiredAttributes.insert(Name);
 263
 264         e.optionalAttributes.insert(As);
 265         e.optionalAttributes.insert(Override);
 266     }
 267
 268     /* xsl:param */
 269     {
 270         ElementDescription<XSLTTokenLookup> &e = result[Param];
 271
 272         e.requiredAttributes.insert(Name);
 273
 274         e.optionalAttributes.insert(Select);
 275         e.optionalAttributes.insert(As);
 276         e.optionalAttributes.insert(Required);
 277         e.optionalAttributes.insert(Tunnel);
 278     }
 279
 280     /* xsl:namespace */
 281     {
 282         ElementDescription<XSLTTokenLookup> &e = result[Namespace];
 283
 284         e.requiredAttributes.insert(Name);
 285         e.optionalAttributes.insert(Select);
 286     }
 287
 288     /* xsl:call-template */
 289     {
 290         ElementDescription<XSLTTokenLookup> &e = result[CallTemplate];
 291         e.requiredAttributes.insert(Name);
 292     }
 293
 294     /* xsl:perform-sort */
 295     {
 296         ElementDescription<XSLTTokenLookup> &e = result[PerformSort];
 297         e.requiredAttributes.insert(Select);
 298     }
 299
 300     /* xsl:sort */
 301     {
 302         ElementDescription<XSLTTokenLookup> &e = result[Sort];
 303
 304         e.optionalAttributes.reserve(7);
 305         e.optionalAttributes.insert(Select);
 306         e.optionalAttributes.insert(Lang);
 307         e.optionalAttributes.insert(Order);
 308         e.optionalAttributes.insert(Collation);
 309         e.optionalAttributes.insert(Stable);
 310         e.optionalAttributes.insert(CaseOrder);
 311         e.optionalAttributes.insert(DataType);
 312     }
 313
 314     /* xsl:import-schema */
 315     {
 316         ElementDescription<XSLTTokenLookup> &e = result[ImportSchema];
 317
 318         e.optionalAttributes.reserve(2);
 319         e.optionalAttributes.insert(Namespace);
 320         e.optionalAttributes.insert(SchemaLocation);
 321     }
 322
 323     /* xsl:message */
 324     {
 325         ElementDescription<XSLTTokenLookup> &e = result[Message];
 326
 327         e.optionalAttributes.reserve(2);
 328         e.optionalAttributes.insert(Select);
 329         e.optionalAttributes.insert(Terminate);
 330     }
 331
 332     /* xsl:copy-of */
 333     {
 334         ElementDescription<XSLTTokenLookup> &e = result[CopyOf];
 335
 336         e.requiredAttributes.insert(Select);
 337
 338         e.optionalAttributes.reserve(2);
 339         e.optionalAttributes.insert(CopyNamespaces);
 340         e.optionalAttributes.insert(Type);
 341         e.optionalAttributes.insert(Validation);
 342     }
 343
 344     /* xsl:copy */
 345     {
 346         ElementDescription<XSLTTokenLookup> &e = result[Copy];
 347
 348         e.optionalAttributes.reserve(5);
 349         e.optionalAttributes.insert(CopyNamespaces);
 350         e.optionalAttributes.insert(InheritNamespaces);
 351         e.optionalAttributes.insert(UseAttributeSets);
 352         e.optionalAttributes.insert(Type);
 353         e.optionalAttributes.insert(Validation);
 354     }
 355
 356     /* xsl:output */
 357     {
 358         ElementDescription<XSLTTokenLookup> &e = result[Output];
 359
 360         e.optionalAttributes.reserve(17);
 361         e.optionalAttributes.insert(Name);
 362         e.optionalAttributes.insert(Method);
 363         e.optionalAttributes.insert(ByteOrderMark);
 364         e.optionalAttributes.insert(CdataSectionElements);
 365         e.optionalAttributes.insert(DoctypePublic);
 366         e.optionalAttributes.insert(DoctypeSystem);
 367         e.optionalAttributes.insert(Encoding);
 368         e.optionalAttributes.insert(EscapeUriAttributes);
 369         e.optionalAttributes.insert(IncludeContentType);
 370         e.optionalAttributes.insert(Indent);
 371         e.optionalAttributes.insert(MediaType);
 372         e.optionalAttributes.insert(NormalizationForm);
 373         e.optionalAttributes.insert(OmitXmlDeclaration);
 374         e.optionalAttributes.insert(Standalone);
 375         e.optionalAttributes.insert(UndeclarePrefixes);
 376         e.optionalAttributes.insert(UseCharacterMaps);
 377         e.optionalAttributes.insert(Version);
 378     }
 379
 380     /* xsl:attribute-set */
 381     {
 382         ElementDescription<XSLTTokenLookup> &e = result[AttributeSet];
 383
 384         e.requiredAttributes.insert(Name);
 385         e.optionalAttributes.insert(UseAttributeSets);
 386     }
 387
 388     /* xsl:include and xsl:import. */
 389     {
 390         ElementDescription<XSLTTokenLookup> &e = result[Include];
 391         e.requiredAttributes.insert(Href);
 392         result[Import] = e;
 393     }
 394
 395     /* xsl:with-param */
 396     {
 397         ElementDescription<XSLTTokenLookup> &e = result[WithParam];
 398         e.requiredAttributes.insert(Name);
 399
 400         e.optionalAttributes.insert(Select);
 401         e.optionalAttributes.insert(As);
 402         e.optionalAttributes.insert(Tunnel);
 403     }
 404
 405     /* xsl:strip-space */
 406     {
 407         ElementDescription<XSLTTokenLookup> &e = result[StripSpace];
 408         e.requiredAttributes.insert(Elements);
 409
 410         result.insert(PreserveSpace, e);
 411     }
 412
 413     /* xsl:result-document */
 414     {
 415         ElementDescription<XSLTTokenLookup> &e = result[ResultDocument];
 416
 417         e.optionalAttributes.insert(ByteOrderMark);
 418         e.optionalAttributes.insert(CdataSectionElements);
 419         e.optionalAttributes.insert(DoctypePublic);
 420         e.optionalAttributes.insert(DoctypeSystem);
 421         e.optionalAttributes.insert(Encoding);
 422         e.optionalAttributes.insert(EscapeUriAttributes);
 423         e.optionalAttributes.insert(Format);
 424         e.optionalAttributes.insert(Href);
 425         e.optionalAttributes.insert(IncludeContentType);
 426         e.optionalAttributes.insert(Indent);
 427         e.optionalAttributes.insert(MediaType);
 428         e.optionalAttributes.insert(Method);
 429         e.optionalAttributes.insert(NormalizationForm);
 430         e.optionalAttributes.insert(OmitXmlDeclaration);
 431         e.optionalAttributes.insert(OutputVersion);
 432         e.optionalAttributes.insert(Standalone);
 433         e.optionalAttributes.insert(Type);
 434         e.optionalAttributes.insert(UndeclarePrefixes);
 435         e.optionalAttributes.insert(UseCharacterMaps);
 436         e.optionalAttributes.insert(Validation);
 437     }
 438
 439     /* xsl:key */
 440     {
 441         ElementDescription<XSLTTokenLookup> &e = result[Key];
 442
 443         e.requiredAttributes.insert(Name);
 444         e.requiredAttributes.insert(Match);
 445
 446         e.optionalAttributes.insert(Use);
 447         e.optionalAttributes.insert(Collation);
 448     }
 449
 450     /* xsl:analyze-string */
 451     {
 452         ElementDescription<XSLTTokenLookup> &e = result[AnalyzeString];
 453
 454         e.requiredAttributes.insert(Select);
 455         e.requiredAttributes.insert(Regex);
 456
 457         e.optionalAttributes.insert(Flags);
 458     }
 459
 460     /* xsl:matching-substring */
 461     {
 462         /* We insert a default constructed value. */
 463         result[MatchingSubstring];
 464     }
 465
 466     /* xsl:non-matching-substring */
 467     {
 468         /* We insert a default constructed value. */
 469         result[NonMatchingSubstring];
 470     }
 471
 472     Q_ASSERT(result.count() == ReservedForElements);
 473
 474     return result;
 475 }
 476
 477 QHash<QString, int> XSLTTokenizer::createValidationAlternatives()
 478 {
 479     QHash<QString, int> retval;
 480
 481     retval.insert(QLatin1String("preserve"), 0);
 482     retval.insert(QLatin1String("strip"), 1);
 483     retval.insert(QLatin1String("strict"), 2);
 484     retval.insert(QLatin1String("lax"), 3);
 485
 486     return retval;
 487 }
 488
 489 bool XSLTTokenizer::whitespaceToSkip() const
 490 {
 491     return m_stripWhitespace.top() && isWhitespace();
 492 }
 493
 494 void XSLTTokenizer::unexpectedContent(const ReportContext::ErrorCode code) const
 495 {
 496     QString message;
 497
 498     ReportContext::ErrorCode effectiveCode = code;
 499
 500     switch(tokenType())
 501     {
 502         case QXmlStreamReader::StartElement:
 503         {
 504             if(isXSLT())
 505             {
 506                 switch(currentElementName())
 507                 {
 508                     case Include:
 509                         effectiveCode = ReportContext::XTSE0170;
 510                         break;
 511                     case Import:
 512                         effectiveCode = ReportContext::XTSE0190;
 513                         break;
 514                     default:
 515                         ;
 516                 }
 517             }
 518
 519             message = QtXmlPatterns::tr("Element %1 is not allowed at this location.")
 520                                        .arg(formatKeyword(name()));
 521             break;
 522         }
 523         case QXmlStreamReader::Characters:
 524         {
 525             if(whitespaceToSkip())
 526                 return;
 527
 528             message = QtXmlPatterns::tr("Text nodes are not allowed at this location.");
 529             break;
 530         }
 531         case QXmlStreamReader::Invalid:
 532         {
 533             /* It's an issue with well-formedness. */
 534             message = escape(errorString());
 535             break;
 536         }
 537         default:
 538             Q_ASSERT(false);
 539     }
 540
 541     error(message, effectiveCode);
 542 }
 543
 544 void XSLTTokenizer::checkForParseError() const
 545 {
 546     if(hasError())
 547     {
 548         error(QtXmlPatterns::tr("Parse error: %1").arg(escape(errorString())), ReportContext::XTSE0010);
 549     }
 550 }
 551
 552 QString XSLTTokenizer::readElementText()
 553 {
 554     QString result;
 555
 556     while(!atEnd())
 557     {
 558         switch(readNext())
 559         {
 560             case QXmlStreamReader::Characters:
 561             {
 562                 result += text().toString();
 563                 continue;
 564             }
 565             case QXmlStreamReader::Comment:
 566             /* Fallthrough. */
 567             case QXmlStreamReader::ProcessingInstruction:
 568                 continue;
 569             case QXmlStreamReader::EndElement:
 570                 return result;
 571             default:
 572                 unexpectedContent();
 573         }
 574     }
 575
 576     checkForParseError();
 577     return result;
 578 }
 579
 580 int XSLTTokenizer::commenceScanOnly()
 581 {
 582     /* Do nothing, return a dummy value. */
 583     return 0;
 584 }
 585
 586 void XSLTTokenizer::resumeTokenizationFrom(const int position)
 587 {
 588     /* Do nothing. */
 589     Q_UNUSED(position);
 590 }
 591
 592 void XSLTTokenizer::handleXSLTVersion(TokenSource::Queue *const to,
 593                                       QStack<Token> *const queueOnExit,
 594                                       const bool isXSLTElement,
 595                                       const QXmlStreamAttributes *atts,
 596                                       const bool generateCode,
 597                                       const bool setGlobalVersion)
 598 {
 599     const QString ns(isXSLTElement ? QString() : CommonNamespaces::XSLT);
 600     const QXmlStreamAttributes effectiveAtts(atts ? *atts : attributes());
 601
 602     if(!effectiveAtts.hasAttribute(ns, QLatin1String("version")))
 603         return;
 604
 605     const QString attribute(effectiveAtts.value(ns, QLatin1String("version")).toString());
 606     const AtomicValue::Ptr number(Decimal::fromLexical(attribute));
 607
 608     if(number->hasError())
 609     {
 610         error(QtXmlPatterns::tr("The value of the XSL-T version attribute "
 611                                            "must be a value of type %1, which %2 isn't.").arg(formatType(m_namePool, BuiltinTypes::xsDecimal),
 612                                                                                               formatData(attribute)),
 613               ReportContext::XTSE0110);
 614     }
 615     else
 616     {
 617
 618         if(generateCode)
 619         {
 620             queueToken(Token(XSLT_VERSION, attribute), to);
 621             queueToken(CURLY_LBRACE, to);
 622         }
 623
 624         const xsDecimal version = number->as<Numeric>()->toDecimal();
 625         if(version == 2.0)
 626             m_processingMode.push(NormalProcessing);
 627         else if(version == 1.0)
 628         {
 629             /* See section 3.6 Stylesheet Element discussing this. */
 630             warning(QtXmlPatterns::tr("Running an XSL-T 1.0 stylesheet with a 2.0 processor."));
 631             m_processingMode.push(BackwardsCompatible);
 632
 633             if(setGlobalVersion)
 634             {
 635                 m_parseInfo->staticContext->setCompatModeEnabled(true);
 636                 m_parseInfo->isBackwardsCompat.push(true);
 637             }
 638         }
 639         else if(version > 2.0)
 640             m_processingMode.push(ForwardCompatible);
 641         else if(version < 2.0)
 642             m_processingMode.push(BackwardsCompatible);
 643     }
 644
 645     if(generateCode)
 646         queueOnExit->push(CURLY_RBRACE);
 647 }
 648
 649 void XSLTTokenizer::handleXMLBase(TokenSource::Queue *const to,
 650                                   QStack<Token> *const queueOnExit,
 651                                   const bool isInstruction,
 652                                   const QXmlStreamAttributes *atts)
 653 {
 654     const QXmlStreamAttributes effectiveAtts(atts ? *atts : m_currentAttributes);
 655
 656     if(effectiveAtts.hasAttribute(QLatin1String("xml:base")))
 657     {
 658         const QStringRef val(effectiveAtts.value(QLatin1String("xml:base")));
 659
 660         if(!val.isEmpty())
 661         {
 662             if(isInstruction)
 663             {
 664                 queueToken(BASEURI, to);
 665                 queueToken(Token(STRING_LITERAL, val.toString()), to);
 666                 queueToken(CURLY_LBRACE, to);
 667                 queueOnExit->push(CURLY_RBRACE);
 668             }
 669             else
 670             {
 671                 queueToken(DECLARE, to);
 672                 queueToken(BASEURI, to);
 673                 queueToken(INTERNAL, to);
 674                 queueToken(Token(STRING_LITERAL, val.toString()), to);
 675                 queueToken(SEMI_COLON, to);
 676             }
 677         }
 678     }
 679 }
 680
 681 void XSLTTokenizer::handleStandardAttributes(const bool isXSLTElement)
 682 {
 683     /* We're not necessarily StartElement, that's why we have atts passed in. */
 684     Q_ASSERT(tokenType() == QXmlStreamReader::StartElement);
 685
 686     if(m_hasHandledStandardAttributes)
 687         return;
 688
 689     m_hasHandledStandardAttributes = true;
 690
 691     const QString ns(isXSLTElement ? QString() : CommonNamespaces::XSLT);
 692     const int len = m_currentAttributes.count();
 693
 694     for(int i = 0; i < len; ++i)
 695     {
 696         const QXmlStreamAttribute &att = m_currentAttributes.at(i);
 697
 698         if(att.qualifiedName() == QLatin1String("xml:space"))
 699         {
 700             const QStringRef val(m_currentAttributes.value(CommonNamespaces::XML, QLatin1String("space")));
 701
 702             /* We raise an error if the value is not recognized.
 703              *
 704              * Extensible Markup Language (XML) 1.0 (Fourth Edition), 2.10
 705              * White Space Handling:
 706              *
 707              * 'This specification does not give meaning to any value of
 708              * xml:space other than "default" and "preserve". It is an error
 709              * for other values to be specified; the XML processor may report
 710              * the error or may recover by ignoring the attribute specification
 711              * or by reporting the (erroneous) value to the application.' */
 712             m_stripWhitespace.push(readToggleAttribute(QLatin1String("xml:space"),
 713                                                        QLatin1String("default"),
 714                                                        QLatin1String("preserve"),
 715                                                        &m_currentAttributes));
 716         }
 717
 718         if(att.namespaceUri() != ns)
 719             continue;
 720
 721         switch(toToken(att.name()))
 722         {
 723             case Type:
 724             /* Fallthrough. */
 725             case Validation:
 726             /* Fallthrough. */
 727             case UseAttributeSets:
 728             /* Fallthrough. */
 729             case Version:
 730                 /* These are handled by other function such as
 731                  * handleValidationAttributes() and handleXSLTVersion(). */
 732                 continue;
 733             default:
 734             {
 735                 if(!isXSLTElement) /* validateElement() will take care of it, and we
 736                                     * don't want to flag non-standard XSL-T attributes. */
 737                 {
 738                     error(QtXmlPatterns::tr("Unknown XSL-T attribute %1.")
 739                                                       .arg(formatKeyword(att.name())),
 740                           ReportContext::XTSE0805);
 741                 }
 742             }
 743         }
 744     }
 745 }
 746
 747 void XSLTTokenizer::handleValidationAttributes(const bool isLRE) const
 748 {
 749     Q_ASSERT(tokenType() == QXmlStreamReader::StartElement);
 750
 751     const QString ns(isLRE ? QString() : CommonNamespaces::XSLT);
 752
 753     const bool hasValidation = hasAttribute(ns, QLatin1String("validation"));
 754     const bool hasType = hasAttribute(ns, QLatin1String("type"));
 755
 756     if(!hasType && !hasValidation)
 757         return;
 758
 759     if(hasType && hasValidation)
 760     {
 761         error(QtXmlPatterns::tr("Attribute %1 and %2 are mutually exclusive.")
 762                                           .arg(formatKeyword(QLatin1String("validation")),
 763                                                formatKeyword(QLatin1String("type"))),
 764               ReportContext::XTSE1505);
 765     }
 766
 767     /* QXmlStreamReader surely doesn't make this easy. */
 768     QXmlStreamAttribute validationAttribute;
 769     int len = m_currentAttributes.count();
 770
 771     for(int i = 0; i < len; ++i)
 772     {
 773         const QXmlStreamAttribute &at = m_currentAttributes.at(i);
 774         if(at.name() == QLatin1String("validation") && at.namespaceUri() == ns)
 775             validationAttribute = at;
 776     }
 777
 778     Q_ASSERT_X(!validationAttribute.name().isNull(), Q_FUNC_INFO,
 779                "We should always find the attribute.");
 780
 781     /* We don't care about the return value, we just want to check it's a valid
 782      * one. */
 783     readAlternativeAttribute(m_validationAlternatives,
 784                              validationAttribute);
 785 }
 786
 787 Tokenizer::Token XSLTTokenizer::nextToken(YYLTYPE *const sourceLocator)
 788 {
 789     Q_UNUSED(sourceLocator);
 790
 791     if(m_tokenSource.isEmpty())
 792     {
 793         switch(m_state.top())
 794         {
 795             case OutsideDocumentElement:
 796                 outsideDocumentElement();
 797                 break;
 798             case InsideStylesheetModule:
 799                 insideStylesheetModule();
 800                 break;
 801             case InsideSequenceConstructor:
 802                 insideSequenceConstructor(&m_tokenSource);
 803                 break;
 804         }
 805
 806         if(m_tokenSource.isEmpty())
 807         {
 808             *sourceLocator = currentSourceLocator();
 809             return Token(END_OF_FILE);
 810         }
 811         else
 812             return m_tokenSource.head()->nextToken(sourceLocator);
 813     }
 814     else
 815     {
 816         do
 817         {
 818             const Token candidate(m_tokenSource.head()->nextToken(sourceLocator));
 819             if(candidate.type == END_OF_FILE)
 820                 m_tokenSource.dequeue();
 821             else
 822                 return candidate;
 823         }
 824         while(!m_tokenSource.isEmpty());
 825
 826         /* Now we will resume parsing inside the regular XSL-T(XML) file. */
 827         return nextToken(sourceLocator);
 828     }
 829 }
 830
 831 bool XSLTTokenizer::isElement(const XSLTTokenLookup::NodeName &name) const
 832 {
 833     Q_ASSERT(isXSLT());
 834     Q_ASSERT(tokenType() == QXmlStreamReader::StartElement ||
 835              tokenType() == QXmlStreamReader::EndElement);
 836
 837     return currentElementName() == name;
 838 }
 839
 840 inline bool XSLTTokenizer::isXSLT() const
 841 {
 842     Q_ASSERT_X(tokenType() == QXmlStreamReader::StartElement ||
 843                tokenType() == QXmlStreamReader::EndElement,
 844                Q_FUNC_INFO, "The current token state must be StartElement or EndElement.");
 845     /* Possible optimization: let MaintainingReader set an m_isXSLT which we
 846      * read. */
 847     return namespaceUri() == CommonNamespaces::XSLT;
 848 }
 849
 850 void XSLTTokenizer::queueOnExit(QStack<Token> &source,
 851                                 TokenSource::Queue *const destination)
 852 {
 853     while(!source.isEmpty())
 854         queueToken(source.pop(), destination);
 855 }
 856
 857 void XSLTTokenizer::outsideDocumentElement()
 858 {
 859     while(!atEnd())
 860     {
 861         switch(readNext())
 862         {
 863             case QXmlStreamReader::StartElement:
 864             {
 865                 /* First, we synthesize one of the built-in templates,
 866                  * see section 6.6 Built-in Template Rules.
 867                  *
 868                  * Note that insideStylesheetModule() can be called multiple
 869                  * times so we can't do it there.  */
 870                 {
 871                     /* Start with the one for text nodes and attributes.
 872                      * declare template matches (text() | @*) mode #all
 873                      * {
 874                      *      text{.}
 875                      * };
 876                      */
 877
 878                     /* declare template matches (text() | @*) */
 879                     queueToken(DECLARE, &m_tokenSource);
 880                     queueToken(TEMPLATE, &m_tokenSource);
 881                     queueToken(MATCHES, &m_tokenSource);
 882                     queueToken(LPAREN, &m_tokenSource);
 883                     queueToken(TEXT, &m_tokenSource);
 884                     queueToken(LPAREN, &m_tokenSource);
 885                     queueToken(RPAREN, &m_tokenSource);
 886                     queueToken(BAR, &m_tokenSource);
 887                     queueToken(AT_SIGN, &m_tokenSource);
 888                     queueToken(STAR, &m_tokenSource);
 889                     queueToken(RPAREN, &m_tokenSource);
 890
 891                     /* mode #all */
 892                     queueToken(MODE, &m_tokenSource);
 893                     queueToken(Token(NCNAME, QLatin1String("#all")), &m_tokenSource);
 894                     queueToken(CURLY_LBRACE, &m_tokenSource);
 895
 896                     /* text{.} { */
 897                     queueToken(TEXT, &m_tokenSource);
 898                     queueToken(CURLY_LBRACE, &m_tokenSource);
 899                     queueToken(DOT, &m_tokenSource);
 900                     queueToken(CURLY_RBRACE, &m_tokenSource);
 901
 902                     /* }; */
 903                     queueToken(CURLY_RBRACE, &m_tokenSource);
 904                     queueToken(SEMI_COLON, &m_tokenSource);
 905                 }
 906
 907                 if(isXSLT() && isStylesheetElement())
 908                 {
 909                     handleStandardAttributes(true);
 910                     QStack<Token> onExitTokens;
 911                     handleXMLBase(&m_tokenSource, &onExitTokens, false);
 912                     handleXSLTVersion(&m_tokenSource, &onExitTokens, true, 0, false, true);
 913                     validateElement();
 914                     queueNamespaceDeclarations(&m_tokenSource, 0, true);
 915
 916                     /* We're a regular stylesheet. */
 917
 918                     pushState(InsideStylesheetModule);
 919                     insideStylesheetModule();
 920                 }
 921                 else
 922                 {
 923                     /* We're a simplified stylesheet. */
 924
 925                     if(!hasAttribute(CommonNamespaces::XSLT, QLatin1String("version")))
 926                     {
 927                         error(QtXmlPatterns::tr("In a simplified stylesheet module, attribute %1 must be present.")
 928                                                           .arg(formatKeyword(QLatin1String("version"))),
 929                               ReportContext::XTSE0010);
 930                     }
 931
 932                     QStack<Token> onExitTokens;
 933
 934                     /* We synthesize this as exemplified in
 935                      * 3.7 Simplified Stylesheet Modules. */
 936                     queueToken(DECLARE, &m_tokenSource);
 937                     queueToken(TEMPLATE, &m_tokenSource);
 938                     queueToken(MATCHES, &m_tokenSource);
 939                     queueToken(LPAREN, &m_tokenSource);
 940                     queueToken(SLASH, &m_tokenSource);
 941                     queueToken(RPAREN, &m_tokenSource);
 942                     queueToken(CURLY_LBRACE, &m_tokenSource);
 943                     pushState(InsideSequenceConstructor);
 944
 945                     handleXSLTVersion(&m_tokenSource, &onExitTokens, false, 0, true);
 946                     handleStandardAttributes(false);
 947
 948                     insideSequenceConstructor(&m_tokenSource, false);
 949
 950                     queueOnExit(onExitTokens, &m_tokenSource);
 951                     queueToken(CURLY_RBRACE, &m_tokenSource);
 952                     queueToken(CURLY_RBRACE, &m_tokenSource);
 953                     queueToken(SEMI_COLON, &m_tokenSource);
 954                 }
 955
 956                 queueToken(APPLY_TEMPLATE, &m_tokenSource);
 957                 queueToken(LPAREN, &m_tokenSource);
 958                 queueToken(RPAREN, &m_tokenSource);
 959
 960                 break;
 961             }
 962             default:
 963                 /* Do nothing. */;
 964         }
 965     }
 966     checkForParseError();
 967 }
 968
 969 void XSLTTokenizer::queueToken(const Token &token,
 970                                TokenSource::Queue *const to)
 971 {
 972     TokenSource::Queue *const effective = to ? to : &m_tokenSource;
 973
 974     effective->enqueue(TokenSource::Ptr(new SingleTokenContainer(token, currentSourceLocator())));
 975 }
 976
 977 void XSLTTokenizer::pushState(const State nextState)
 978 {
 979     m_state.push(nextState);
 980 }
 981
 982 void XSLTTokenizer::leaveState()
 983 {
 984     m_state.pop();
 985 }
 986
 987 void XSLTTokenizer::insideTemplate()
 988 {
 989     const bool hasPriority  = hasAttribute(QLatin1String("priority"));
 990     const bool hasMatch     = hasAttribute(QLatin1String("match"));
 991     const bool hasName      = hasAttribute(QLatin1String("name"));
 992     const bool hasMode      = hasAttribute(QLatin1String("mode"));
 993     const bool hasAs        = hasAttribute(QLatin1String("as"));
 994
 995     if(!hasMatch &&
 996        (hasMode ||
 997         hasPriority))
 998     {
 999         error(QtXmlPatterns::tr("If element %1 has no attribute %2, it cannot have attribute %3 or %4.")
1000                          .arg(formatKeyword(QLatin1String("template")),
1001                               formatKeyword(QLatin1String("match")),
1002                               formatKeyword(QLatin1String("mode")),
1003                               formatKeyword(QLatin1String("priority"))),
1004               ReportContext::XTSE0500);
1005     }
1006     else if(!hasMatch && !hasName)
1007     {
1008         error(QtXmlPatterns::tr("Element %1 must have at least one of the attributes %2 or %3.")
1009                          .arg(formatKeyword(QLatin1String("template")),
1010                               formatKeyword(QLatin1String("name")),
1011                               formatKeyword(QLatin1String("match"))),
1012               ReportContext::XTSE0500);
1013     }
1014
1015     queueToken(DECLARE, &m_tokenSource);
1016     queueToken(TEMPLATE, &m_tokenSource);
1017
1018     if(hasName)
1019     {
1020         queueToken(NAME, &m_tokenSource);
1021         queueToken(Token(QNAME, readAttribute(QLatin1String("name"))), &m_tokenSource);
1022     }
1023
1024     if(hasMatch)
1025     {
1026         queueToken(MATCHES, &m_tokenSource);
1027         queueExpression(readAttribute(QLatin1String("match")), &m_tokenSource);
1028     }
1029
1030     if(hasMode)
1031     {
1032         const QString modeString(readAttribute(QLatin1String("mode")).simplified());
1033
1034         if(modeString.isEmpty())
1035         {
1036             error(QtXmlPatterns::tr("At least one mode must be specified in the %1-attribute on element %2.")
1037                              .arg(formatKeyword(QLatin1String("mode")),
1038                                   formatKeyword(QLatin1String("template"))),
1039                   ReportContext::XTSE0500);
1040         }
1041
1042         queueToken(MODE, &m_tokenSource);
1043
1044         const QStringList modeList(modeString.split(QLatin1Char(' ')));
1045
1046         for(int i = 0; i < modeList.count(); ++i)
1047         {
1048             const QString &mode = modeList.at(i);
1049
1050             queueToken(Token(mode.contains(QLatin1Char(':')) ? QNAME : NCNAME, mode), &m_tokenSource);
1051
1052             if(i < modeList.count() - 1)
1053                 queueToken(COMMA, &m_tokenSource);
1054         }
1055     }
1056
1057     if(hasPriority)
1058     {
1059         queueToken(PRIORITY, &m_tokenSource);
1060         queueToken(Token(STRING_LITERAL, readAttribute(QLatin1String("priority"))), &m_tokenSource);
1061     }
1062
1063     QStack<Token> onExitTokens;
1064     Q_ASSERT(tokenType() == QXmlStreamReader::StartElement);
1065
1066     /* queueParams moves the reader so we need to freeze the attributes. */
1067     const QXmlStreamAttributes atts(m_currentAttributes);
1068     handleStandardAttributes(true);
1069     queueToken(LPAREN, &m_tokenSource);
1070     queueParams(Template, &m_tokenSource);
1071     queueToken(RPAREN, &m_tokenSource);
1072
1073     if(hasAs)
1074     {
1075         queueToken(AS, &m_tokenSource);
1076         queueSequenceType(atts.value(QLatin1String("as")).toString());
1077     }
1078
1079     queueToken(CURLY_LBRACE, &m_tokenSource);
1080
1081     handleXMLBase(&m_tokenSource, &onExitTokens, true, &atts);
1082     handleXSLTVersion(&m_tokenSource, &onExitTokens, true, &atts);
1083     pushState(InsideSequenceConstructor);
1084     startStorageOfCurrent(&m_tokenSource);
1085     insideSequenceConstructor(&m_tokenSource, onExitTokens, false);
1086     queueOnExit(onExitTokens, &m_tokenSource);
1087 }
1088
1089 void XSLTTokenizer::queueExpression(const QString &expr,
1090                                     TokenSource::Queue *const to,
1091                                     const bool wrapWithParantheses)
1092 {
1093     TokenSource::Queue *const effectiveTo = to ? to : &m_tokenSource;
1094
1095     if(wrapWithParantheses)
1096         queueToken(LPAREN, effectiveTo);
1097
1098     effectiveTo->enqueue(TokenSource::Ptr(new XQueryTokenizer(expr, queryURI())));
1099
1100     if(wrapWithParantheses)
1101         queueToken(RPAREN, effectiveTo);
1102 }
1103
1104 void XSLTTokenizer::queueAVT(const QString &expr,
1105                              TokenSource::Queue *const to)
1106 {
1107     queueToken(AVT, to);
1108     queueToken(LPAREN, to);
1109     to->enqueue(TokenSource::Ptr(new XQueryTokenizer(expr, queryURI(),
1110                                                                XQueryTokenizer::QuotAttributeContent)));
1111     queueToken(RPAREN, to);
1112 }
1113
1114 void XSLTTokenizer::queueSequenceType(const QString &expr)
1115 {
1116     m_tokenSource.enqueue(TokenSource::Ptr(new XQueryTokenizer(expr, queryURI(),
1117                                                                          XQueryTokenizer::ItemType)));
1118 }
1119
1120 void XSLTTokenizer::commencingExpression(bool &hasWrittenExpression,
1121                                          TokenSource::Queue *const to)
1122 {
1123     if(hasWrittenExpression)
1124         queueToken(COMMA, to);
1125     else
1126         hasWrittenExpression = true;
1127 }
1128
1129 void XSLTTokenizer::queueEmptySequence(TokenSource::Queue *const to)
1130 {
1131     queueToken(LPAREN, to);
1132     queueToken(RPAREN, to);
1133 }
1134
1135 void XSLTTokenizer::insideChoose(TokenSource::Queue *const to)
1136 {
1137     Q_ASSERT(tokenType() == QXmlStreamReader::StartElement);
1138     bool hasHandledOtherwise = false;
1139     bool hasEncounteredAtLeastOneWhen = false;
1140
1141     while(!atEnd())
1142     {
1143         switch(readNext())
1144         {
1145             case QXmlStreamReader::StartElement:
1146             {
1147                 if(isXSLT())
1148                 {
1149                     QStack<Token> onExitTokens;
1150                     handleStandardAttributes(true);
1151                     validateElement();
1152
1153                     switch(currentElementName())
1154                     {
1155                         case When:
1156                         {
1157                             if(hasHandledOtherwise)
1158                             {
1159                                 error(QtXmlPatterns::tr("Element %1 must come last.")
1160                                                                   .arg(formatKeyword(QLatin1String("otherwise"))),
1161                                       ReportContext::XTSE0010);
1162                             }
1163
1164                             queueToken(IF, to);
1165                             queueToken(LPAREN, to);
1166                             queueExpression(readAttribute(QLatin1String("test")), to);
1167                             queueToken(RPAREN, to);
1168                             queueToken(THEN, to);
1169                             queueToken(LPAREN, to);
1170                             pushState(InsideSequenceConstructor);
1171                             insideSequenceConstructor(to);
1172                             queueToken(RPAREN, to);
1173                             Q_ASSERT(tokenType() == QXmlStreamReader::EndElement);
1174                             queueToken(ELSE, to);
1175                             hasEncounteredAtLeastOneWhen = true;
1176                             queueOnExit(onExitTokens, to);
1177                             break;
1178                         }
1179                         case Otherwise:
1180                         {
1181                             if(!hasEncounteredAtLeastOneWhen)
1182                             {
1183                                 error(QtXmlPatterns::tr("At least one %1-element must occur before %2.")
1184                                                                   .arg(formatKeyword(QLatin1String("when")),
1185                                                                        formatKeyword(QLatin1String("otherwise"))),
1186                                       ReportContext::XTSE0010);
1187                             }
1188                             else if(hasHandledOtherwise)
1189                             {
1190                                 error(QtXmlPatterns::tr("Only one %1-element can appear.")
1191                                                                   .arg(formatKeyword(QLatin1String("otherwise"))),
1192                                       ReportContext::XTSE0010);
1193                             }
1194
1195                             pushState(InsideSequenceConstructor);
1196                             queueToken(LPAREN, to);
1197                             insideSequenceConstructor(to, to);
1198                             queueToken(RPAREN, to);
1199                             hasHandledOtherwise = true;
1200                             queueOnExit(onExitTokens, to);
1201                             break;
1202                         }
1203                         default:
1204                             unexpectedContent();
1205                     }
1206                 }
1207                 else
1208                     unexpectedContent();
1209                 break;
1210             }
1211             case QXmlStreamReader::EndElement:
1212             {
1213                 if(isXSLT())
1214                 {
1215                     switch(currentElementName())
1216                     {
1217                         case Choose:
1218                         {
1219                             if(!hasEncounteredAtLeastOneWhen)
1220                             {
1221                                 error(QtXmlPatterns::tr("At least one %1-element must occur inside %2.")
1222                                                                   .arg(formatKeyword(QLatin1String("when")),
1223                                                                        formatKeyword(QLatin1String("choose"))),
1224                                       ReportContext::XTSE0010);
1225                             }
1226
1227                             if(!hasHandledOtherwise)
1228                                 queueEmptySequence(to);
1229                             return;
1230                         }
1231                         case Otherwise:
1232                             continue;
1233                         default:
1234                             unexpectedContent();
1235                     }
1236                 }
1237                 else
1238                     unexpectedContent();
1239                 break;
1240             }
1241             case QXmlStreamReader::Comment:
1242             /* Fallthrough. */
1243             case QXmlStreamReader::ProcessingInstruction:
1244                 continue;
1245             case QXmlStreamReader::Characters:
1246             {
1247                 /* We ignore regardless of what xml:space says, see step 4 in
1248                  * 4.2 Stripping Whitespace from the Stylesheet. */
1249                 if(isWhitespace())
1250                     continue;
1251                 /* Fallthrough. */
1252             }
1253             default:
1254                 /* Fallthrough. */
1255                 unexpectedContent();
1256                 break;
1257         }
1258     }
1259     checkForParseError();
1260 }
1261
1262 bool XSLTTokenizer::queueSelectOrSequenceConstructor(const ReportContext::ErrorCode code,
1263                                                      const bool emptynessAllowed,
1264                                                      TokenSource::Queue *const to,
1265                                                      const QXmlStreamAttributes *const attsP,
1266                                                      const bool queueEmptyOnEmpty)
1267 {
1268     Q_ASSERT(tokenType() == QXmlStreamReader::StartElement || attsP);
1269     const NodeName elementName(currentElementName());
1270     const QXmlStreamAttributes atts(attsP ? *attsP : m_currentAttributes);
1271
1272     if(atts.hasAttribute(QLatin1String("select")))
1273     {
1274         queueExpression(atts.value(QLatin1String("select")).toString(), to);
1275
1276         /* First, verify that we don't have a body. */
1277         if(skipSubTree(true))
1278         {
1279             error(QtXmlPatterns::tr("When attribute %1 is present on %2, a sequence "
1280                                                "constructor cannot be used.").arg(formatKeyword(QLatin1String("select")),
1281                                                                                   formatKeyword(toString(elementName))),
1282                   code);
1283         }
1284
1285         return true;
1286     }
1287     else
1288     {
1289         pushState(InsideSequenceConstructor);
1290         if(!insideSequenceConstructor(to, true, queueEmptyOnEmpty) && !emptynessAllowed)
1291         {
1292             error(QtXmlPatterns::tr("Element %1 must have either a %2-attribute "
1293                                                "or a sequence constructor.").arg(formatKeyword(toString(elementName)),
1294                                                                                  formatKeyword(QLatin1String("select"))),
1295                   code);
1296
1297         }
1298
1299         return false;
1300     }
1301 }
1302
1303 void XSLTTokenizer::queueSimpleContentConstructor(const ReportContext::ErrorCode code,
1304                                                   const bool emptynessAllowed,
1305                                                   TokenSource::Queue *const to,
1306                                                   const bool selectOnlyFirst)
1307 {
1308     queueToken(INTERNAL_NAME, to);
1309     queueToken(Token(NCNAME, QLatin1String("generic-string-join")), to);
1310     queueToken(LPAREN, to);
1311
1312     /* We have to read the attribute before calling
1313      * queueSelectOrSequenceConstructor(), since it advances the reader. */
1314     const bool hasSeparator = m_currentAttributes.hasAttribute(QLatin1String("separator"));
1315     const QString separatorAVT(m_currentAttributes.value(QLatin1String("separator")).toString());
1316
1317     queueToken(LPAREN, to);
1318     const bool viaSelectAttribute = queueSelectOrSequenceConstructor(code, emptynessAllowed, to);
1319     queueToken(RPAREN, to);
1320
1321     if(selectOnlyFirst)
1322     {
1323         queueToken(LBRACKET, to);
1324         queueToken(Token(NUMBER, QChar::fromLatin1('1')), to);
1325         queueToken(RBRACKET, to);
1326     }
1327
1328     queueToken(COMMA, to);
1329
1330     if(hasSeparator)
1331         queueAVT(separatorAVT, to);
1332     else
1333     {
1334         /* The default value depends on whether the value is from @select, or from
1335          * the sequence constructor. */
1336         queueToken(Token(STRING_LITERAL, viaSelectAttribute ? QString(QLatin1Char(' '))
1337                                                             : QString()),
1338                    to);
1339     }
1340
1341     queueToken(RPAREN, to);
1342 }
1343
1344 void XSLTTokenizer::queueTextConstructor(QString &chars,
1345                                          bool &hasWrittenExpression,
1346                                          TokenSource::Queue *const to)
1347 {
1348     if(!chars.isEmpty())
1349     {
1350         commencingExpression(hasWrittenExpression, to);
1351         queueToken(TEXT, to);
1352         queueToken(CURLY_LBRACE, to);
1353         queueToken(Token(STRING_LITERAL, chars), to);
1354         queueToken(CURLY_RBRACE, to);
1355         chars.clear();
1356     }
1357 }
1358
1359 void XSLTTokenizer::queueVariableDeclaration(const VariableType variableType,
1360                                              TokenSource::Queue *const to)
1361 {
1362     Q_ASSERT(tokenType() == QXmlStreamReader::StartElement);
1363
1364     if(variableType == VariableInstruction)
1365     {
1366         queueToken(LET, to);
1367         queueToken(INTERNAL, to);
1368     }
1369     else if(variableType == VariableDeclaration || variableType == GlobalParameter)
1370     {
1371         queueToken(DECLARE, to);
1372         queueToken(VARIABLE, to);
1373         queueToken(INTERNAL, to);
1374     }
1375
1376     queueToken(DOLLAR, to);
1377
1378     queueExpression(readAttribute(QLatin1String("name")), to, false);
1379
1380     const bool hasAs = m_currentAttributes.hasAttribute(QLatin1String("as"));
1381     if(hasAs)
1382     {
1383         queueToken(AS, to);
1384         queueSequenceType(m_currentAttributes.value(QLatin1String("as")).toString());
1385     }
1386
1387     if(variableType == FunctionParameter)
1388     {
1389         skipBodyOfParam(ReportContext::XTSE0760);
1390         return;
1391     }
1392
1393     /* We must do this here, because queueSelectOrSequenceConstructor()
1394      * advances the reader. */
1395     const bool hasSelect = hasAttribute(QLatin1String("select"));
1396     const bool isRequired = hasAttribute(QLatin1String("required")) ? attributeYesNo(QLatin1String("required")) : false;
1397
1398     TokenSource::Queue storage;
1399     queueSelectOrSequenceConstructor(ReportContext::XTSE0620, true, &storage, 0, false);
1400
1401     /* XSL-T has some wicked rules, see
1402      * 9.3 Values of Variables and Parameters. */
1403
1404     const bool hasQueuedContent = !storage.isEmpty();
1405
1406     /* The syntax for global parameters is:
1407      *
1408      * declare variable $var external := 'defaultValue';
1409      */
1410     if(variableType == GlobalParameter)
1411         queueToken(EXTERNAL, to);
1412
1413     if(isRequired)
1414     {
1415         if(hasQueuedContent)
1416         {
1417             error(QtXmlPatterns::tr("When a parameter is required, a default value "
1418                                                "cannot be supplied through a %1-attribute or "
1419                                                "a sequence constructor.").arg(formatKeyword(QLatin1String("select"))),
1420                   ReportContext::XTSE0010);
1421         }
1422     }
1423     else
1424     {
1425         if(hasQueuedContent)
1426         {
1427             queueToken(ASSIGN, to);
1428
1429             if(!hasSelect && !hasAs && !hasQueuedContent)
1430                 queueToken(Token(STRING_LITERAL, QString()), to);
1431             else if(hasAs || hasSelect)
1432                 queueToken(LPAREN, to);
1433             else
1434             {
1435                 queueToken(DOCUMENT, to);
1436                 queueToken(INTERNAL, to);
1437                 queueToken(CURLY_LBRACE, to);
1438             }
1439         }
1440         else
1441         {
1442             if(!hasAs)
1443             {
1444                 queueToken(ASSIGN, to);
1445                 queueToken(Token(STRING_LITERAL, QString()), to);
1446             }
1447             else if(variableType == VariableDeclaration || variableType == VariableInstruction)
1448             {
1449                 queueToken(ASSIGN, to);
1450                 queueEmptySequence(to);
1451             }
1452         }
1453
1454         /* storage has tokens if hasSelect or hasQueuedContent is true. */
1455         if(hasSelect | hasQueuedContent)
1456             *to += storage;
1457
1458         if(hasQueuedContent)
1459         {
1460             if(!hasSelect && !hasAs && !hasQueuedContent)
1461                 queueToken(Token(STRING_LITERAL, QString()), to);
1462             else if(hasAs || hasSelect)
1463                 queueToken(RPAREN, to);
1464             else
1465                 queueToken(CURLY_RBRACE, to);
1466         }
1467     }
1468
1469     if(variableType == VariableInstruction)
1470         queueToken(RETURN, to);
1471     else if(variableType == VariableDeclaration || variableType == GlobalParameter)
1472         queueToken(SEMI_COLON, to);
1473 }
1474
1475 void XSLTTokenizer::startStorageOfCurrent(TokenSource::Queue *const to)
1476 {
1477     queueToken(CURRENT, to);
1478     queueToken(CURLY_LBRACE, to);
1479 }
1480
1481 void XSLTTokenizer::endStorageOfCurrent(TokenSource::Queue *const to)
1482 {
1483     queueToken(CURLY_RBRACE, to);
1484 }
1485
1486 void XSLTTokenizer::queueNamespaceDeclarations(TokenSource::Queue *const to,
1487                                                QStack<Token> *const queueOnExit,
1488                                                const bool isDeclaration)
1489 {
1490     Q_ASSERT(tokenType() == QXmlStreamReader::StartElement);
1491     Q_ASSERT_X(isDeclaration || queueOnExit,
1492                Q_FUNC_INFO,
1493                "If isDeclaration is false, queueOnExit must be passed.");
1494
1495     const QXmlStreamNamespaceDeclarations nss(namespaceDeclarations());
1496
1497     for(int i = 0; i < nss.count(); ++i)
1498     {
1499         const QXmlStreamNamespaceDeclaration &at = nss.at(i);
1500         queueToken(DECLARE, to);
1501         queueToken(NAMESPACE, to);
1502         queueToken(Token(NCNAME, at.prefix().toString()), to);
1503         queueToken(G_EQ, to);
1504         queueToken(Token(STRING_LITERAL, at.namespaceUri().toString()), to);
1505
1506         if(isDeclaration)
1507         {
1508             queueToken(INTERNAL, to);
1509             queueToken(SEMI_COLON, to);
1510         }
1511         else
1512         {
1513             queueToken(CURLY_LBRACE, to);
1514             queueOnExit->push(CURLY_RBRACE);
1515         }
1516     }
1517 }
1518
1519 bool XSLTTokenizer::insideSequenceConstructor(TokenSource::Queue *const to,
1520                                               const bool initialAdvance,
1521                                               const bool queueEmptyOnEmpty)
1522 {
1523     QStack<Token> onExitTokens;
1524     return insideSequenceConstructor(to, onExitTokens, initialAdvance, queueEmptyOnEmpty);
1525 }
1526
1527 bool XSLTTokenizer::insideSequenceConstructor(TokenSource::Queue *const to,
1528                                               QStack<Token> &onExitTokens,
1529                                               const bool initialAdvance,
1530                                               const bool queueEmptyOnEmpty)
1531 {
1532     bool effectiveInitialAdvance = initialAdvance;
1533     bool hasWrittenExpression = false;
1534
1535     /* Buffer which all text nodes, that might be split up by comments,
1536      * processing instructions and CDATA sections, are appended to. */
1537     QString characters;
1538
1539     while(!atEnd())
1540     {
1541         if(effectiveInitialAdvance)
1542             readNext();
1543         else
1544             effectiveInitialAdvance = true;
1545
1546         switch(tokenType())
1547         {
1548             case QXmlStreamReader::StartElement:
1549             {
1550                 queueTextConstructor(characters, hasWrittenExpression, to);
1551                 handleXMLBase(to, &onExitTokens);
1552
1553                 pushState(InsideSequenceConstructor);
1554
1555                 commencingExpression(hasWrittenExpression, to);
1556
1557                 if(isXSLT())
1558                 {
1559                     handleXSLTVersion(&m_tokenSource, &onExitTokens, true);
1560                     handleStandardAttributes(true);
1561                     validateElement();
1562
1563                     queueNamespaceDeclarations(to, &onExitTokens);
1564
1565                     switch(currentElementName())
1566                     {
1567                         case If:
1568                         {
1569                             queueToken(IF, to);
1570                             queueToken(LPAREN, to);
1571
1572                             queueExpression(readAttribute(QLatin1String("test")), to);
1573                             queueToken(RPAREN, to);
1574                             queueToken(THEN, to);
1575
1576                             queueToken(LPAREN, to);
1577                             pushState(InsideSequenceConstructor);
1578                             insideSequenceConstructor(to);
1579
1580                             break;
1581                         }
1582                         case Choose:
1583                         {
1584                             insideChoose(to);
1585                             break;
1586                         }
1587                         case ValueOf:
1588                         {
1589                             /* We generate a computed text node constructor. */
1590                             queueToken(TEXT, to);
1591                             queueToken(CURLY_LBRACE, to);
1592
1593                             queueSimpleContentConstructor(ReportContext::XTSE0870, true, to,
1594                                                           !hasAttribute(QLatin1String("separator")) && m_processingMode.top() == BackwardsCompatible);
1595                             queueToken(CURLY_RBRACE, to);
1596                             break;
1597                         }
1598                         case Sequence:
1599                         {
1600                             queueExpression(readAttribute(QLatin1String("select")), to);
1601                             parseFallbacksOnly();
1602                             break;
1603                         }
1604                         case Text:
1605                         {
1606                             queueToken(TEXT, to);
1607                             queueToken(CURLY_LBRACE, to);
1608
1609                             queueToken(Token(STRING_LITERAL, readElementText()), to);
1610                             queueToken(CURLY_RBRACE, to);
1611                             break;
1612                         }
1613                         case Variable:
1614                         {
1615                             queueVariableDeclaration(VariableInstruction, to);
1616
1617                             /* We wrap the children in parantheses since we may
1618                              * queue several expressions using the comma operator,
1619                              * and in that case the let-binding is only in-scope
1620                              * for the first expression. */
1621                             queueToken(LPAREN, to);
1622
1623                             /* We don't want a comma outputted, we're expecting an
1624                              * expression now. */
1625                             hasWrittenExpression = false;
1626
1627                             onExitTokens.push(RPAREN);
1628
1629                             break;
1630                         }
1631                         case CallTemplate:
1632                         {
1633                             queueToken(CALL_TEMPLATE, to);
1634                             queueToken(Token(QNAME, readAttribute(QLatin1String("name"))), to);
1635                             queueToken(LPAREN, to);
1636                             queueWithParams(CallTemplate, to);
1637                             queueToken(RPAREN, to);
1638                             break;
1639                         }
1640                         case ForEach:
1641                         {
1642                             queueExpression(readAttribute(QLatin1String("select")), to);
1643                             queueToken(MAP, to);
1644                             pushState(InsideSequenceConstructor);
1645
1646                             TokenSource::Queue sorts;
1647                             queueSorting(false, &sorts);
1648
1649
1650                             if(sorts.isEmpty())
1651                             {
1652                                 startStorageOfCurrent(to);
1653                                 insideSequenceConstructor(to, false);
1654                                 endStorageOfCurrent(to);
1655                             }
1656                             else
1657                             {
1658                                 queueToken(SORT, to);
1659                                 *to += sorts;
1660                                 queueToken(RETURN, to);
1661                                 startStorageOfCurrent(to);
1662                                 insideSequenceConstructor(to, false);
1663                                 endStorageOfCurrent(to);
1664                                 queueToken(END_SORT, to);
1665                             }
1666
1667                             break;
1668                         }
1669                         case XSLTTokenLookup::Comment:
1670                         {
1671                             queueToken(COMMENT, to);
1672                             queueToken(INTERNAL, to);
1673                             queueToken(CURLY_LBRACE, to);
1674                             queueSelectOrSequenceConstructor(ReportContext::XTSE0940, true, to);
1675                             queueToken(CURLY_RBRACE, to);
1676                             break;
1677                         }
1678                         case CopyOf:
1679                         {
1680                             queueExpression(readAttribute(QLatin1String("select")), to);
1681                             // TODO
1682
1683                             if(readNext() == QXmlStreamReader::EndElement)
1684                                 break;
1685                             else
1686                             {
1687                                 error(QtXmlPatterns::tr("Element %1 cannot have children.").arg(formatKeyword(QLatin1String("copy-of"))),
1688                                       ReportContext::XTSE0010);
1689                             }
1690                             break;
1691                         }
1692                         case AnalyzeString:
1693                         {
1694                             // TODO
1695                             skipSubTree();
1696                             break;
1697                         }
1698                         case ResultDocument:
1699                         {
1700                             // TODO
1701                             pushState(InsideSequenceConstructor);
1702                             insideSequenceConstructor(to);
1703                             break;
1704                         }
1705                         case Copy:
1706                         {
1707                             /* We translate:
1708                              *      <xsl:copy>expr</xsl:copy>
1709                              * into:
1710                              *
1711                              *  let $body := expr
1712                              *  return
1713                              *      if(self::element()) then
1714                              *          element internal {node-name()} {$body}
1715                              *      else if(self::document-node()) then
1716                              *          document internal {$body}
1717                              *      else (: This includes comments, processing-instructions,
1718                              *              attributes, and comments. :)
1719                              *          .
1720                              *
1721                              * TODO node identity is the same as the old node.
1722                              * TODO namespace bindings are lost when elements are constructed
1723                              */
1724
1725                             /* let $body := expr */
1726                             queueToken(LET, to);
1727                             queueToken(INTERNAL, to);
1728                             queueToken(DOLLAR, to);
1729                             queueToken(Token(NCNAME, QString(QLatin1Char('b'))), to); // TODO we need an internal name
1730                             queueToken(ASSIGN, to);
1731                             queueToken(LPAREN, to);
1732                             pushState(InsideSequenceConstructor);
1733                             /* Don't queue an empty sequence, we want the dot. */
1734                             insideSequenceConstructor(to);
1735                             queueToken(RPAREN, to);
1736                             queueToken(RETURN, to);
1737
1738                             /* if(self::element()) then */
1739                             queueToken(IF, to);
1740                             queueToken(LPAREN, to);
1741                             queueToken(SELF, to);
1742                             queueToken(COLONCOLON, to);
1743                             queueToken(ELEMENT, to);
1744                             queueToken(LPAREN, to);
1745                             queueToken(RPAREN, to);
1746                             queueToken(RPAREN, to);
1747                             queueToken(THEN, to);
1748
1749                             /* element internal {node-name()} {$body} */
1750                             queueToken(ELEMENT, to);
1751                             queueToken(INTERNAL, to);
1752                             queueToken(CURLY_LBRACE, to);
1753                             queueToken(Token(NCNAME, QLatin1String("node-name")), to); // TODO what if the default ns changes?
1754                             queueToken(LPAREN, to);
1755                             queueToken(DOT, to);
1756                             queueToken(RPAREN, to);
1757                             queueToken(CURLY_RBRACE, to);
1758                             queueToken(CURLY_LBRACE, to);
1759                             queueToken(DOLLAR, to);
1760                             queueToken(Token(NCNAME, QString(QLatin1Char('b'))), to); // TODO we need an internal name
1761                             queueToken(CURLY_RBRACE, to);
1762
1763                             /* else if(self::document-node()) then */
1764                             queueToken(ELSE, to);
1765                             queueToken(IF, to);
1766                             queueToken(LPAREN, to);
1767                             queueToken(SELF, to);
1768                             queueToken(COLONCOLON, to);
1769                             queueToken(DOCUMENT_NODE, to);
1770                             queueToken(LPAREN, to);
1771                             queueToken(RPAREN, to);
1772                             queueToken(RPAREN, to);
1773                             queueToken(THEN, to);
1774
1775                             /* document internal {$body} */
1776                             queueToken(DOCUMENT, to);
1777                             queueToken(INTERNAL, to);
1778                             queueToken(CURLY_LBRACE, to);
1779                             queueToken(DOLLAR, to);
1780                             queueToken(Token(NCNAME, QString(QLatin1Char('b'))), to); // TODO we need an internal name
1781                             queueToken(CURLY_RBRACE, to);
1782
1783                             /* else . */
1784                             queueToken(ELSE, to);
1785                             queueToken(DOT, to);
1786
1787                             break;
1788                         }
1789                         case XSLTTokenLookup::ProcessingInstruction:
1790                         {
1791                             queueToken(PROCESSING_INSTRUCTION, to);
1792                             queueToken(CURLY_LBRACE, to);
1793                             queueAVT(readAttribute(QLatin1String("name")), to);
1794                             queueToken(CURLY_RBRACE, to);
1795                             queueToken(CURLY_LBRACE, to);
1796                             queueSelectOrSequenceConstructor(ReportContext::XTSE0880, true, to);
1797                             queueToken(CURLY_RBRACE, to);
1798                             break;
1799                         }
1800                         case Document:
1801                         {
1802                             handleValidationAttributes(false);
1803
1804                             // TODO base-URI
1805                             queueToken(DOCUMENT, to);
1806                             queueToken(INTERNAL, to);
1807                             queueToken(CURLY_LBRACE, to);
1808                             pushState(InsideSequenceConstructor);
1809                             insideSequenceConstructor(to);
1810                             queueToken(CURLY_RBRACE, to);
1811                             break;
1812                         }
1813                         case Element:
1814                         {
1815                             handleValidationAttributes(false);
1816
1817                             // TODO base-URI
1818                             queueToken(ELEMENT, to);
1819                             queueToken(INTERNAL, to);
1820
1821                             /* The name. */
1822                             queueToken(CURLY_LBRACE, to);
1823                             // TODO only strings allowed, not qname values.
1824                             queueAVT(readAttribute(QLatin1String("name")), to);
1825                             queueToken(CURLY_RBRACE, to);
1826
1827                             /* The sequence constructor. */
1828                             queueToken(CURLY_LBRACE, to);
1829                             pushState(InsideSequenceConstructor);
1830                             insideSequenceConstructor(to);
1831                             queueToken(CURLY_RBRACE, to);
1832                             break;
1833                         }
1834                         case Attribute:
1835                         {
1836                             handleValidationAttributes(false);
1837
1838                             // TODO base-URI
1839                             queueToken(ATTRIBUTE, to);
1840                             queueToken(INTERNAL, to);
1841
1842                             /* The name. */
1843                             queueToken(CURLY_LBRACE, to);
1844                             // TODO only strings allowed, not qname values.
1845                             queueAVT(readAttribute(QLatin1String("name")), to);
1846                             queueToken(CURLY_RBRACE, to);
1847
1848                             /* The sequence constructor. */
1849                             queueToken(CURLY_LBRACE, to);
1850                             queueSimpleContentConstructor(ReportContext::XTSE0840,
1851                                                           true, to);
1852                             queueToken(CURLY_RBRACE, to);
1853                             break;
1854                         }
1855                         case Namespace:
1856                         {
1857                             queueToken(NAMESPACE, to);
1858
1859                             /* The name. */
1860                             queueToken(CURLY_LBRACE, to);
1861                             queueAVT(readAttribute(QLatin1String("name")), to);
1862                             queueToken(CURLY_RBRACE, to);
1863
1864                             /* The sequence constructor. */
1865                             queueToken(CURLY_LBRACE, to);
1866                             queueSelectOrSequenceConstructor(ReportContext::XTSE0910,
1867                                                              false, to);
1868                             queueToken(CURLY_RBRACE, to);
1869                             break;
1870                         }
1871                         case PerformSort:
1872                         {
1873                             /* For:
1874                              * <xsl:perform-sort select="$in">
1875                              *      <xsl:sort select="@key"/>
1876                              * </xsl:perform-sort>
1877                              *
1878                              * we generate:
1879                              *
1880                              * $in map sort order by @key
1881                              *         return .
1882                              *         end_sort
1883                              */
1884
1885                             /* In XQuery, the sort keys appear after the expression
1886                              * supplying the initial sequence, while in
1887                              * xsl:perform-sort, if a sequence constructor is used,
1888                              * they appear in the opposite order. Hence, we need to
1889                              * reorder it. */
1890
1891                             /* We store the attributes of xsl:perform-sort, before
1892                              * queueSorting() advances the reader. */
1893                             const QXmlStreamAttributes atts(m_currentAttributes);
1894
1895                             TokenSource::Queue sorts;
1896                             queueSorting(true, &sorts);
1897                             queueSelectOrSequenceConstructor(ReportContext::XTSE1040,
1898                                                              true,
1899                                                              to,
1900                                                              &atts);
1901                             /* queueSelectOrSequenceConstructor() positions us on EndElement. */
1902                             effectiveInitialAdvance = false;
1903                             queueToken(MAP, to);
1904                             queueToken(SORT, to);
1905                             *to += sorts;
1906                             queueToken(RETURN, to);
1907                             queueToken(DOT, to);
1908                             queueToken(END_SORT, to);
1909
1910                             break;
1911                         }
1912                         case Message:
1913                         {
1914                             // TODO
1915                             queueEmptySequence(to);
1916                             skipSubTree();
1917                             break;
1918                         }
1919                         case ApplyTemplates:
1920                         {
1921                             if(hasAttribute(QLatin1String("select")))
1922                                 queueExpression(readAttribute(QLatin1String("select")), to);
1923                             else
1924                             {
1925                                 queueToken(CHILD, to);
1926                                 queueToken(COLONCOLON, to);
1927                                 queueToken(NODE, to);
1928                                 queueToken(LPAREN, to);
1929                                 queueToken(RPAREN, to);
1930                             }
1931
1932                             bool hasMode = hasAttribute(QLatin1String("mode"));
1933                             QString mode;
1934
1935                             if(hasMode)
1936                                 mode = readAttribute(QLatin1String("mode")).trimmed();
1937
1938                             queueToken(FOR_APPLY_TEMPLATE, to);
1939
1940                             TokenSource::Queue sorts;
1941                             queueSorting(false, &sorts, true);
1942
1943                             if(!sorts.isEmpty())
1944                             {
1945                                 queueToken(SORT, to);
1946                                 *to += sorts;
1947                                 queueToken(RETURN, to);
1948                             }
1949
1950                             queueToken(APPLY_TEMPLATE, to);
1951
1952                             if(hasMode)
1953                             {
1954                                 queueToken(MODE, to);
1955                                 queueToken(Token(mode.startsWith(QLatin1Char('#')) ? NCNAME : QNAME, mode), to);
1956                             }
1957
1958                             queueToken(LPAREN, to);
1959                             queueWithParams(ApplyTemplates, to, false);
1960                             queueToken(RPAREN, to);
1961
1962                             if(!sorts.isEmpty())
1963                                 queueToken(END_SORT, to);
1964
1965                             break;
1966                         }
1967                         default:
1968                             unexpectedContent();
1969                     }
1970                     continue;
1971                 }
1972                 else
1973                 {
1974                     handleXSLTVersion(&m_tokenSource, &onExitTokens, true);
1975                     handleStandardAttributes(false);
1976                     handleValidationAttributes(false);
1977
1978                     /* We're generating an element constructor. */
1979                     queueNamespaceDeclarations(to, &onExitTokens); // TODO same in the isXSLT() branch
1980                     queueToken(ELEMENT, to);
1981                     queueToken(INTERNAL, to);
1982                     queueToken(Token(QNAME, qualifiedName().toString()), to);
1983                     queueToken(CURLY_LBRACE, to);
1984                     const int len = m_currentAttributes.count();
1985
1986                     for(int i = 0; i < len; ++i)
1987                     {
1988                         const QXmlStreamAttribute &at = m_currentAttributes.at(i);
1989
1990                         /* We don't want to generate constructors for XSL-T attributes. */
1991                         if(at.namespaceUri() == CommonNamespaces::XSLT)
1992                             continue;
1993
1994                         queueToken(ATTRIBUTE, to);
1995                         queueToken(INTERNAL, to);
1996
1997                         queueToken(Token(at.prefix().isEmpty() ? NCNAME : QNAME, at.qualifiedName().toString()), to);
1998                         queueToken(CURLY_LBRACE, to);
1999                         queueAVT(at.value().toString(), to);
2000                         queueToken(CURLY_RBRACE, to);
2001                         queueToken(COMMA, to);
2002                     }
2003
2004                     pushState(InsideSequenceConstructor);
2005                     insideSequenceConstructor(to);
2006                     Q_ASSERT(tokenType() == QXmlStreamReader::EndElement || hasError());
2007                     continue;
2008                 }
2009
2010                 unexpectedContent();
2011                 break;
2012             }
2013             case QXmlStreamReader::EndElement:
2014             {
2015                 queueTextConstructor(characters, hasWrittenExpression, to);
2016                 leaveState();
2017
2018                 if(!hasWrittenExpression && queueEmptyOnEmpty)
2019                     queueEmptySequence(to);
2020
2021                 queueOnExit(onExitTokens, to);
2022
2023                 if(isXSLT())
2024                 {
2025                     Q_ASSERT(!isElement(Sequence));
2026
2027                     switch(currentElementName())
2028                     {
2029                         /* Fallthrough all these. */
2030                         case When:
2031                         case Choose:
2032                         case ForEach:
2033                         case Otherwise:
2034                         case PerformSort:
2035                         case Message:
2036                         case ResultDocument:
2037                         case Copy:
2038                         case CallTemplate:
2039                         case Text:
2040                         case ValueOf:
2041                         {
2042                             hasWrittenExpression = true;
2043                             break;
2044                         }
2045                         case If:
2046                         {
2047                             queueToken(RPAREN, to);
2048                             queueToken(ELSE, to);
2049                             queueEmptySequence(to);
2050                             break;
2051                         }
2052                         case Function:
2053                         {
2054                             queueToken(CURLY_RBRACE, to);
2055                             queueToken(SEMI_COLON, to);
2056                             break;
2057                         }
2058                         case Template:
2059                         {
2060                             endStorageOfCurrent(&m_tokenSource);
2061                             /* TODO, fallthrough to Function. */
2062                             queueToken(CURLY_RBRACE, to);
2063                             queueToken(SEMI_COLON, to);
2064                             break;
2065                         }
2066                         default:
2067                             ;
2068                     }
2069                 }
2070                 else
2071                 {
2072                     /* We're closing a direct element constructor. */
2073                     hasWrittenExpression = true;
2074                     queueToken(CURLY_RBRACE, to);
2075                 }
2076
2077                 return hasWrittenExpression;
2078             }
2079             case QXmlStreamReader::ProcessingInstruction:
2080             /* Fallthrough. */
2081             case QXmlStreamReader::Comment:
2082                 /* We do nothing, we just ignore them. */
2083                 continue;
2084             case QXmlStreamReader::Characters:
2085             {
2086                 if(whitespaceToSkip())
2087                     continue;
2088                 else
2089                 {
2090                     characters += text().toString();
2091                     continue;
2092                 }
2093             }
2094             default:
2095                 ;
2096         }
2097     }
2098
2099     leaveState();
2100     return hasWrittenExpression;
2101 }
2102
2103 bool XSLTTokenizer::isStylesheetElement() const
2104 {
2105     Q_ASSERT(isXSLT());
2106     Q_ASSERT(tokenType() == QXmlStreamReader::StartElement ||
2107              tokenType() == QXmlStreamReader::EndElement);
2108
2109     const NodeName name = currentElementName();
2110     return name == Stylesheet || name == Transform;
2111 }
2112
2113 void XSLTTokenizer::skipBodyOfParam(const ReportContext::ErrorCode code)
2114 {
2115     Q_ASSERT(isXSLT());
2116     Q_ASSERT(tokenType() == QXmlStreamReader::StartElement);
2117     const NodeName name(currentElementName());
2118
2119     if(skipSubTree())
2120     {
2121         error(QtXmlPatterns::tr("Element %1 cannot have a sequence constructor.")
2122                                           .arg(formatKeyword(toString(name))),
2123               code);
2124     }
2125 }
2126
2127 void XSLTTokenizer::queueWithParams(const XSLTTokenLookup::NodeName parentName,
2128                                     TokenSource::Queue *const to,
2129                                     const bool initialAdvance)
2130 {
2131     Q_ASSERT(parentName == ApplyTemplates || parentName == CallTemplate);
2132
2133     bool effectiveInitialAdvance = initialAdvance;
2134     bool hasQueuedParam = false;
2135
2136     while(!atEnd())
2137     {
2138         if(effectiveInitialAdvance)
2139             readNext();
2140         else
2141             effectiveInitialAdvance = true;
2142
2143         switch(tokenType())
2144         {
2145             case QXmlStreamReader::StartElement:
2146             {
2147                 if(hasQueuedParam)
2148                     queueToken(COMMA, to);
2149
2150                 if(isXSLT() && isElement(WithParam))
2151                 {
2152                     if(hasAttribute(QLatin1String("tunnel")) && attributeYesNo(QLatin1String("tunnel")))
2153                         queueToken(TUNNEL, to);
2154
2155                     queueVariableDeclaration(WithParamVariable, to);
2156                     hasQueuedParam = true;
2157                     continue;
2158                 }
2159                 else
2160                     unexpectedContent();
2161             }
2162             case QXmlStreamReader::EndElement:
2163             {
2164                 if(isElement(parentName))
2165                     return;
2166                 else
2167                     continue;
2168             }
2169             case QXmlStreamReader::ProcessingInstruction:
2170             /* Fallthrough. */
2171             case QXmlStreamReader::Comment:
2172                 continue;
2173             case QXmlStreamReader::Characters:
2174                 if(whitespaceToSkip())
2175                     continue;
2176                 else
2177                     return;
2178             default:
2179                 unexpectedContent();
2180         }
2181     }
2182     unexpectedContent();
2183 }
2184
2185 void XSLTTokenizer::queueParams(const XSLTTokenLookup::NodeName parentName,
2186                                 TokenSource::Queue *const to)
2187 {
2188     bool hasQueuedParam = false;
2189
2190     Q_ASSERT(tokenType() == QXmlStreamReader::StartElement);
2191
2192     while(!atEnd())
2193     {
2194         switch(readNext())
2195         {
2196             case QXmlStreamReader::StartElement:
2197             {
2198                 if(isXSLT() && isElement(Param))
2199                 {
2200                     if(hasQueuedParam)
2201                         queueToken(COMMA, to);
2202
2203                     validateElement();
2204
2205                     if(parentName == Function && m_currentAttributes.hasAttribute(QLatin1String("select")))
2206                     {
2207                         error(QtXmlPatterns::tr("The attribute %1 cannot appear on %2, when it is a child of %3.")
2208                                          .arg(formatKeyword(QLatin1String("select")),
2209                                               formatKeyword(QLatin1String("param")),
2210                                               formatKeyword(QLatin1String("function"))),
2211                               ReportContext::XTSE0760);
2212                     }
2213
2214                     if(parentName == Function && m_currentAttributes.hasAttribute(QLatin1String("required")))
2215                     {
2216                         error(QtXmlPatterns::tr("The attribute %1 cannot appear on %2, when it is a child of %3.")
2217                                          .arg(formatKeyword(QLatin1String("required")),
2218                                               formatKeyword(QLatin1String("param")),
2219                                               formatKeyword(QLatin1String("function"))),
2220                               ReportContext::XTSE0010);
2221                     }
2222
2223                     const bool hasTunnel = m_currentAttributes.hasAttribute(QLatin1String("tunnel"));
2224                     const bool isTunnel = hasTunnel ? attributeYesNo(QLatin1String("tunnel")) : false;
2225
2226                     if(isTunnel)
2227                     {
2228                         if(parentName == Function)
2229                         {
2230                             /* See W3C public report 5650: http://www.w3.org/Bugs/Public/show_bug.cgi?id=5650 */
2231                             error(QtXmlPatterns::tr("A parameter in a function cannot be declared to be a tunnel."),
2232                                   ReportContext::XTSE0010);
2233                         }
2234                         else
2235                             queueToken(TUNNEL, to);
2236                     }
2237
2238                     hasQueuedParam = true;
2239                     queueVariableDeclaration(parentName == Function ? FunctionParameter : TemplateParameter, to);
2240                     continue;
2241                 }
2242                 else
2243                     return;
2244             }
2245             case QXmlStreamReader::Characters:
2246             {
2247                 if(whitespaceToSkip())
2248                     continue;
2249                 /* Fallthrough. */
2250             }
2251             case QXmlStreamReader::EndElement:
2252                 return;
2253             default:
2254                 ;
2255         }
2256     }
2257 }
2258
2259 bool XSLTTokenizer::skipSubTree(const bool exitOnContent)
2260 {
2261     bool hasContent = false;
2262     int depth = 0;
2263
2264     while(!atEnd())
2265     {
2266         switch(readNext())
2267         {
2268             case QXmlStreamReader::Characters:
2269             {
2270                 if(whitespaceToSkip())
2271                     continue;
2272                 else
2273                 {
2274                     hasContent = true;
2275                     if(exitOnContent)
2276                         return true;
2277
2278                     break;
2279                 }
2280             }
2281             case QXmlStreamReader::StartElement:
2282             {
2283                 hasContent = true;
2284                 if(exitOnContent)
2285                     return true;
2286
2287                 ++depth;
2288                 break;
2289             }
2290             case QXmlStreamReader::EndElement:
2291             {
2292                 --depth;
2293                 break;
2294             }
2295             default:
2296                 continue;
2297         }
2298
2299         if(depth == -1)
2300             return hasContent;
2301     }
2302
2303     checkForParseError();
2304     return hasContent;
2305 }
2306
2307 void XSLTTokenizer::parseFallbacksOnly()
2308 {
2309     Q_ASSERT(isXSLT());
2310     Q_ASSERT(tokenType() == QXmlStreamReader::StartElement);
2311
2312     skipSubTree();
2313     Q_ASSERT(tokenType() == QXmlStreamReader::EndElement);
2314 }
2315
2316 void XSLTTokenizer::insideAttributeSet()
2317 {
2318     while(!atEnd())
2319     {
2320         switch(readNext())
2321         {
2322             case QXmlStreamReader::StartElement:
2323             {
2324                 if(isXSLT() && isElement(AttributeSet))
2325                 {
2326                     // TODO
2327                     skipSubTree();
2328                 }
2329                 else
2330                     unexpectedContent();
2331             }
2332             case QXmlStreamReader::EndElement:
2333                 return;
2334             case QXmlStreamReader::ProcessingInstruction:
2335             /* Fallthrough. */
2336             case QXmlStreamReader::Comment:
2337                 continue;
2338             case QXmlStreamReader::Characters:
2339                 if(whitespaceToSkip())
2340                     continue;
2341                 /* Fallthrough. */
2342             default:
2343                 unexpectedContent();
2344         }
2345     }
2346     unexpectedContent();
2347 }
2348
2349 void XSLTTokenizer::insideStylesheetModule()
2350 {
2351     while(!atEnd())
2352     {
2353         switch(readNext())
2354         {
2355             case QXmlStreamReader::StartElement:
2356             {
2357                 if(isXSLT())
2358                 {
2359                     handleStandardAttributes(true);
2360                     handleXSLTVersion(0, 0, true, 0, false);
2361                     validateElement();
2362
2363                     /* Handle the various declarations. */
2364                     switch(currentElementName())
2365                     {
2366                         case Template:
2367                             insideTemplate();
2368                             break;
2369                         case Function:
2370                             insideFunction();
2371                             break;
2372                         case Variable:
2373                             queueVariableDeclaration(VariableDeclaration, &m_tokenSource);
2374                             break;
2375                         case Param:
2376                             queueVariableDeclaration(GlobalParameter, &m_tokenSource);
2377                             break;
2378                         case ImportSchema:
2379                         {
2380                             error(QtXmlPatterns::tr("This processor is not Schema-aware and "
2381                                                                "therefore %1 cannot be used.").arg(formatKeyword(toString(ImportSchema))),
2382                                   ReportContext::XTSE1660);
2383                             break;
2384                         }
2385                         case Output:
2386                         {
2387                             // TODO
2388                             skipSubTree();
2389                             break;
2390                         }
2391                         case StripSpace:
2392                         /* Fallthrough. */
2393                         case PreserveSpace:
2394                         {
2395                             // TODO @elements
2396                             skipSubTree(true);
2397                             readNext();
2398
2399                             if(!isEndElement())
2400                                 unexpectedContent();
2401                             break;
2402                         }
2403                         case Include:
2404                         {
2405                             // TODO
2406                             if(skipSubTree(true))
2407                                 unexpectedContent();
2408                             break;
2409                         }
2410                         case Import:
2411                         {
2412                             // TODO
2413                             if(skipSubTree(true))
2414                                 unexpectedContent();
2415                             break;
2416                         }
2417                         case Key:
2418                         {
2419                             // TODO
2420                             skipSubTree();
2421                             break;
2422                         }
2423                         case AttributeSet:
2424                             insideAttributeSet();
2425                             break;
2426                         default:
2427                             if(m_processingMode.top() != ForwardCompatible)
2428                                 unexpectedContent();
2429                     }
2430                 }
2431                 else
2432                 {
2433                     /* We have a user-defined data element. See section 3.6.2. */
2434
2435                     if(namespaceUri().isEmpty())
2436                     {
2437                         error(QtXmlPatterns::tr("Top level stylesheet elements must be "
2438                                                            "in a non-null namespace, which %1 isn't.").arg(formatKeyword(name())),
2439                               ReportContext::XTSE0130);
2440                     }
2441                     else
2442                         skipSubTree();
2443                 }
2444                 break;
2445             }
2446             case QXmlStreamReader::Characters:
2447             {
2448                 /* Regardless of xml:space, we skip whitespace, see step 4 in
2449                  * 4.2 Stripping Whitespace from the Stylesheet. */
2450                 if(isWhitespace())
2451                     continue;
2452
2453                 unexpectedContent(ReportContext::XTSE0120);
2454                 break;
2455             }
2456             case QXmlStreamReader::EndElement:
2457             {
2458                 if(isXSLT())
2459                     leaveState();
2460
2461                 break;
2462             }
2463             default:
2464                 ;
2465         }
2466     }
2467     checkForParseError();
2468 }
2469
2470 bool XSLTTokenizer::readToggleAttribute(const QString &localName,
2471                                         const QString &isTrue,
2472                                         const QString &isFalse,
2473                                         const QXmlStreamAttributes *const attsP) const
2474 {
2475     const QXmlStreamAttributes atts(attsP ? *attsP : m_currentAttributes);
2476     Q_ASSERT(atts.hasAttribute(localName));
2477     const QString value(atts.value(localName).toString());
2478
2479     if(value == isTrue)
2480         return true;
2481     else if(value == isFalse)
2482         return false;
2483     else
2484     {
2485         error(QtXmlPatterns::tr("The value for attribute %1 on element %2 must either "
2486                                            "be %3 or %4, not %5.").arg(formatKeyword(localName),
2487                                                                        formatKeyword(name()),
2488                                                                        formatData(isTrue),
2489                                                                        formatData(isFalse),
2490                                                                        formatData(value)),
2491               ReportContext::XTSE0020);
2492         /* Silences a compiler warning. */
2493         return false;
2494     }
2495 }
2496
2497 int XSLTTokenizer::readAlternativeAttribute(const QHash<QString, int> &alternatives,
2498                                             const QXmlStreamAttribute &attr) const
2499 {
2500     const QString value(attr.value().toString().trimmed());
2501
2502     if(alternatives.contains(value))
2503         return alternatives[value];
2504
2505     error(QtXmlPatterns::tr("Attribute %1 cannot have the value %2.")
2506                                        .arg(formatKeyword(attr.name().toString()),
2507                                             formatData(attr.value().toString())),
2508           ReportContext::XTSE0020);
2509     return 0; /* Silence compiler warning. */
2510 }
2511
2512 bool XSLTTokenizer::attributeYesNo(const QString &localName) const
2513 {
2514     return readToggleAttribute(localName, QLatin1String("yes"), QLatin1String("no"));
2515 }
2516
2517 void XSLTTokenizer::queueSorting(const bool oneSortRequired,
2518                                  TokenSource::Queue *const to,
2519                                  const bool speciallyTreatWhitespace)
2520 {
2521     Q_ASSERT(tokenType() == QXmlStreamReader::StartElement);
2522
2523     const NodeName elementName(currentElementName());
2524     bool hasQueuedOneSort = false;
2525
2526     while(!atEnd())
2527     {
2528         switch(readNext())
2529         {
2530             case QXmlStreamReader::EndElement:
2531             {
2532                 /* Let's say we have no sequence constructor, but only
2533                  * ignorable space. In that case we will actually loop
2534                  * infinitely if we don't have this check. */
2535                 if(isXSLT())
2536                 {
2537                     switch(currentElementName())
2538                     {
2539                         case PerformSort:
2540                         /* Fallthrough. */
2541                         case ForEach:
2542                         /* Fallthrough. */
2543                         case ApplyTemplates:
2544                             return;
2545                         default:
2546                             ;
2547                     }
2548                 }
2549                 continue;
2550             }
2551             case QXmlStreamReader::StartElement:
2552             {
2553                 if(isXSLT() && isElement(Sort))
2554                 {
2555                     if(hasQueuedOneSort)
2556                         queueToken(COMMA, to);
2557
2558                     /* sorts are by default stable. */
2559                     if(hasAttribute(QLatin1String("stable")))
2560                     {
2561                         if(hasQueuedOneSort)
2562                         {
2563                             error(QtXmlPatterns::tr("The attribute %1 can only appear on "
2564                                                                "the first %2 element.").arg(formatKeyword(QLatin1String("stable")),
2565                                                                                             formatKeyword(QLatin1String("sort"))),
2566                                   ReportContext::XTSE0020);
2567                         }
2568
2569                         if(attributeYesNo(QLatin1String("stable")))
2570                             queueToken(STABLE, to);
2571                     }
2572
2573                     if(!hasQueuedOneSort)
2574                     {
2575                         queueToken(ORDER, to);
2576                         queueToken(BY, to);
2577                     }
2578
2579                     /* We store a copy such that we can use them after
2580                      * queueSelectOrSequenceConstructor() advances the reader. */
2581                     const QXmlStreamAttributes atts(m_currentAttributes);
2582
2583                     const int before = to->count();
2584
2585                     // TODO This doesn't work as is. @data-type can be an AVT.
2586                     if(atts.hasAttribute(QLatin1String("data-type")))
2587                     {
2588                         if(readToggleAttribute(QLatin1String("data-type"),
2589                                                QLatin1String("text"),
2590                                                QLatin1String("number"),
2591                                                &atts))
2592                             queueToken(Token(NCNAME, QLatin1String("string")), to);
2593                         else
2594                             queueToken(Token(NCNAME, QLatin1String("number")), to);
2595                     }
2596                     /* We queue these parantheses for the sake of the function
2597                      * call for attribute data-type. In the case we don't have
2598                      * such an attribute, the parantheses are just redundant. */
2599                     queueToken(LPAREN, to);
2600                     queueSelectOrSequenceConstructor(ReportContext::XTSE1015,
2601                                                      true,
2602                                                      to,
2603                                                      0,
2604                                                      false);
2605                     /* If neither a select attribute or a sequence constructor is supplied,
2606                      * we're supposed to use the context item. */
2607                     queueToken(RPAREN, to);
2608                     if(before == to->count())
2609                         queueToken(DOT, to);
2610
2611                     // TODO case-order
2612                     // TODO lang
2613
2614                     // TODO This doesn't work as is. @order can be an AVT, and so can case-order and lang.
2615                     if(atts.hasAttribute(QLatin1String("order")) && readToggleAttribute(QLatin1String("order"),
2616                                                                                        QLatin1String("descending"),
2617                                                                                        QLatin1String("ascending"),
2618                                                                                        &atts))
2619                     {
2620                         queueToken(DESCENDING, to);
2621                     }
2622                     else
2623                     {
2624                         /* This is the default. */
2625                         queueToken(ASCENDING, to);
2626                     }
2627
2628                     if(atts.hasAttribute(QLatin1String("collation")))
2629                     {
2630                         queueToken(INTERNAL, to);
2631                         queueToken(COLLATION, to);
2632                         queueAVT(atts.value(QLatin1String("collation")).toString(), to);
2633                     }
2634
2635                     hasQueuedOneSort = true;
2636                     continue;
2637                 }
2638                 else
2639                     break;
2640             }
2641             case QXmlStreamReader::Characters:
2642             {
2643                 if(speciallyTreatWhitespace && isWhitespace())
2644                     continue;
2645
2646                 if(QXmlStreamReader::Characters && whitespaceToSkip())
2647                     continue;
2648
2649                 /* We have an instruction which is a text node, we're done. */
2650                 break;
2651             }
2652             case QXmlStreamReader::ProcessingInstruction:
2653             /* Fallthrough. */
2654             case QXmlStreamReader::Comment:
2655                 continue;
2656             default:
2657                 unexpectedContent();
2658
2659         };
2660         if(oneSortRequired && !hasQueuedOneSort)
2661         {
2662             error(QtXmlPatterns::tr("At least one %1 element must appear as child of %2.")
2663                                               .arg(formatKeyword(QLatin1String("sort")), formatKeyword(toString(elementName))),
2664                   ReportContext::XTSE0010);
2665         }
2666         else
2667             return;
2668     }
2669     checkForParseError();
2670 }
2671
2672 void XSLTTokenizer::insideFunction()
2673 {
2674     queueToken(DECLARE, &m_tokenSource);
2675     queueToken(FUNCTION, &m_tokenSource);
2676     queueToken(INTERNAL, &m_tokenSource);
2677     queueToken(Token(QNAME, readAttribute(QLatin1String("name"))), &m_tokenSource);
2678     queueToken(LPAREN, &m_tokenSource);
2679     const QString expectedType(hasAttribute(QLatin1String("as")) ? readAttribute(QLatin1String("as")): QString());
2680
2681     if(hasAttribute(QLatin1String("override")))
2682     {
2683         /* We currently have no external functions, so we don't pass it on currently. */
2684         attributeYesNo(QLatin1String("override"));
2685     }
2686
2687     queueParams(Function, &m_tokenSource);
2688
2689     queueToken(RPAREN, &m_tokenSource);
2690
2691     if(!expectedType.isNull())
2692     {
2693         queueToken(AS, &m_tokenSource);
2694         queueSequenceType(expectedType);
2695     }
2696
2697     QStack<Token> onExitTokens;
2698     handleXMLBase(&m_tokenSource, &onExitTokens, true, &m_currentAttributes);
2699     handleXSLTVersion(&m_tokenSource, &onExitTokens, true);
2700     queueToken(CURLY_LBRACE, &m_tokenSource);
2701
2702     pushState(InsideSequenceConstructor);
2703     insideSequenceConstructor(&m_tokenSource, onExitTokens, false);
2704     /* We don't queue CURLY_RBRACE, because it's done in
2705      * insideSequenceConstructor(). */
2706 }
2707
2708 YYLTYPE XSLTTokenizer::currentSourceLocator() const
2709 {
2710     YYLTYPE retval;
2711     retval.first_line = lineNumber();
2712     retval.first_column = columnNumber();
2713     return retval;
2714 }
2715
2716 QT_END_NAMESPACE
2717