2 * Copyright (C) 2010 Google, Inc. All Rights Reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 #include "core/html/parser/HTMLDocumentParser.h"
29 #include "core/HTMLNames.h"
30 #include "core/css/MediaValuesCached.h"
31 #include "core/dom/DocumentFragment.h"
32 #include "core/dom/Element.h"
33 #include "core/frame/LocalFrame.h"
34 #include "core/html/HTMLDocument.h"
35 #include "core/html/parser/AtomicHTMLToken.h"
36 #include "core/html/parser/BackgroundHTMLParser.h"
37 #include "core/html/parser/HTMLParserScheduler.h"
38 #include "core/html/parser/HTMLParserThread.h"
39 #include "core/html/parser/HTMLScriptRunner.h"
40 #include "core/html/parser/HTMLTreeBuilder.h"
41 #include "core/inspector/InspectorInstrumentation.h"
42 #include "core/inspector/InspectorTraceEvents.h"
43 #include "core/loader/DocumentLoader.h"
44 #include "platform/SharedBuffer.h"
45 #include "platform/TraceEvent.h"
46 #include "public/platform/WebThreadedDataReceiver.h"
47 #include "wtf/Functional.h"
51 using namespace HTMLNames;
53 // This is a direct transcription of step 4 from:
54 // http://www.whatwg.org/specs/web-apps/current-work/multipage/the-end.html#fragment-case
55 static HTMLTokenizer::State tokenizerStateForContextElement(Element* contextElement, bool reportErrors, const HTMLParserOptions& options)
58 return HTMLTokenizer::DataState;
60 const QualifiedName& contextTag = contextElement->tagQName();
62 if (contextTag.matches(titleTag) || contextTag.matches(textareaTag))
63 return HTMLTokenizer::RCDATAState;
64 if (contextTag.matches(styleTag)
65 || contextTag.matches(xmpTag)
66 || contextTag.matches(iframeTag)
67 || (contextTag.matches(noembedTag) && options.pluginsEnabled)
68 || (contextTag.matches(noscriptTag) && options.scriptEnabled)
69 || contextTag.matches(noframesTag))
70 return reportErrors ? HTMLTokenizer::RAWTEXTState : HTMLTokenizer::PLAINTEXTState;
71 if (contextTag.matches(scriptTag))
72 return reportErrors ? HTMLTokenizer::ScriptDataState : HTMLTokenizer::PLAINTEXTState;
73 if (contextTag.matches(plaintextTag))
74 return HTMLTokenizer::PLAINTEXTState;
75 return HTMLTokenizer::DataState;
78 class ParserDataReceiver : public blink::WebThreadedDataReceiver {
80 explicit ParserDataReceiver(WeakPtr<BackgroundHTMLParser> backgroundParser)
81 : m_backgroundParser(backgroundParser)
85 // WebThreadedDataReceiver
86 virtual void acceptData(const char* data, int dataLength) OVERRIDE FINAL
88 ASSERT(backgroundThread() && backgroundThread()->isCurrentThread());
89 if (m_backgroundParser.get())
90 m_backgroundParser.get()->appendRawBytesFromParserThread(data, dataLength);
93 virtual blink::WebThread* backgroundThread() OVERRIDE FINAL
95 if (HTMLParserThread::shared())
96 return &HTMLParserThread::shared()->platformThread();
102 WeakPtr<BackgroundHTMLParser> m_backgroundParser;
105 HTMLDocumentParser::HTMLDocumentParser(HTMLDocument& document, bool reportErrors)
106 : ScriptableDocumentParser(document)
107 , m_options(&document)
108 , m_token(m_options.useThreading ? nullptr : adoptPtr(new HTMLToken))
109 , m_tokenizer(m_options.useThreading ? nullptr : HTMLTokenizer::create(m_options))
110 , m_scriptRunner(HTMLScriptRunner::create(&document, this))
111 , m_treeBuilder(HTMLTreeBuilder::create(this, &document, parserContentPolicy(), reportErrors, m_options))
112 , m_parserScheduler(HTMLParserScheduler::create(this))
113 , m_xssAuditorDelegate(&document)
114 , m_weakFactory(this)
115 , m_preloader(HTMLResourcePreloader::create(document))
116 , m_isPinnedToMainThread(false)
117 , m_endWasDelayed(false)
118 , m_haveBackgroundParser(false)
119 , m_pumpSessionNestingLevel(0)
121 ASSERT(shouldUseThreading() || (m_token && m_tokenizer));
124 // FIXME: Member variables should be grouped into self-initializing structs to
125 // minimize code duplication between these constructors.
126 HTMLDocumentParser::HTMLDocumentParser(DocumentFragment* fragment, Element* contextElement, ParserContentPolicy parserContentPolicy)
127 : ScriptableDocumentParser(fragment->document(), parserContentPolicy)
128 , m_options(&fragment->document())
129 , m_token(adoptPtr(new HTMLToken))
130 , m_tokenizer(HTMLTokenizer::create(m_options))
131 , m_treeBuilder(HTMLTreeBuilder::create(this, fragment, contextElement, this->parserContentPolicy(), m_options))
132 , m_xssAuditorDelegate(&fragment->document())
133 , m_weakFactory(this)
134 , m_isPinnedToMainThread(true)
135 , m_endWasDelayed(false)
136 , m_haveBackgroundParser(false)
137 , m_pumpSessionNestingLevel(0)
139 ASSERT(!shouldUseThreading());
140 bool reportErrors = false; // For now document fragment parsing never reports errors.
141 m_tokenizer->setState(tokenizerStateForContextElement(contextElement, reportErrors, m_options));
142 m_xssAuditor.initForFragment();
145 HTMLDocumentParser::~HTMLDocumentParser()
148 if (m_haveBackgroundParser)
149 stopBackgroundParser();
150 // In Oilpan, HTMLDocumentParser can die together with Document, and
151 // detach() is not called in this case.
153 ASSERT(!m_parserScheduler);
154 ASSERT(!m_pumpSessionNestingLevel);
155 ASSERT(!m_preloadScanner);
156 ASSERT(!m_insertionPreloadScanner);
157 ASSERT(!m_haveBackgroundParser);
158 // FIXME: We should be able to ASSERT(m_speculations.isEmpty()),
159 // but there are cases where that's not true currently. For example,
160 // we we're told to stop parsing before we've consumed all the input.
164 void HTMLDocumentParser::trace(Visitor* visitor)
166 visitor->trace(m_treeBuilder);
167 visitor->trace(m_xssAuditorDelegate);
168 visitor->trace(m_scriptRunner);
169 visitor->trace(m_preloader);
170 ScriptableDocumentParser::trace(visitor);
171 HTMLScriptRunnerHost::trace(visitor);
174 void HTMLDocumentParser::pinToMainThread()
176 ASSERT(!m_haveBackgroundParser);
177 ASSERT(!m_isPinnedToMainThread);
178 m_isPinnedToMainThread = true;
181 m_token = adoptPtr(new HTMLToken);
182 m_tokenizer = HTMLTokenizer::create(m_options);
186 void HTMLDocumentParser::detach()
188 if (m_haveBackgroundParser)
189 stopBackgroundParser();
190 DocumentParser::detach();
192 m_scriptRunner->detach();
193 m_treeBuilder->detach();
194 // FIXME: It seems wrong that we would have a preload scanner here.
195 // Yet during fast/dom/HTMLScriptElement/script-load-events.html we do.
196 m_preloadScanner.clear();
197 m_insertionPreloadScanner.clear();
198 m_parserScheduler.clear(); // Deleting the scheduler will clear any timers.
201 void HTMLDocumentParser::stopParsing()
203 DocumentParser::stopParsing();
204 m_parserScheduler.clear(); // Deleting the scheduler will clear any timers.
205 if (m_haveBackgroundParser)
206 stopBackgroundParser();
209 // This kicks off "Once the user agent stops parsing" as described by:
210 // http://www.whatwg.org/specs/web-apps/current-work/multipage/the-end.html#the-end
211 void HTMLDocumentParser::prepareToStopParsing()
213 // FIXME: It may not be correct to disable this for the background parser.
214 // That means hasInsertionPoint() may not be correct in some cases.
215 ASSERT(!hasInsertionPoint() || m_haveBackgroundParser);
217 // pumpTokenizer can cause this parser to be detached from the Document,
218 // but we need to ensure it isn't deleted yet.
219 RefPtrWillBeRawPtr<HTMLDocumentParser> protect(this);
221 // NOTE: This pump should only ever emit buffered character tokens,
222 // so ForceSynchronous vs. AllowYield should be meaningless.
224 ASSERT(!m_haveBackgroundParser);
225 pumpTokenizerIfPossible(ForceSynchronous);
231 DocumentParser::prepareToStopParsing();
233 // We will not have a scriptRunner when parsing a DocumentFragment.
235 document()->setReadyState(Document::Interactive);
237 // Setting the ready state above can fire mutation event and detach us
238 // from underneath. In that case, just bail out.
242 attemptToRunDeferredScriptsAndEnd();
245 bool HTMLDocumentParser::isParsingFragment() const
247 return m_treeBuilder->isParsingFragment();
250 bool HTMLDocumentParser::processingData() const
252 return isScheduledForResume() || inPumpSession() || m_haveBackgroundParser;
255 void HTMLDocumentParser::pumpTokenizerIfPossible(SynchronousMode mode)
259 if (isWaitingForScripts())
262 // Once a resume is scheduled, HTMLParserScheduler controls when we next pump.
263 if (isScheduledForResume()) {
264 ASSERT(mode == AllowYield);
271 bool HTMLDocumentParser::isScheduledForResume() const
273 return m_parserScheduler && m_parserScheduler->isScheduledForResume();
276 // Used by HTMLParserScheduler
277 void HTMLDocumentParser::resumeParsingAfterYield()
279 ASSERT(!m_isPinnedToMainThread);
280 // pumpTokenizer can cause this parser to be detached from the Document,
281 // but we need to ensure it isn't deleted yet.
282 RefPtrWillBeRawPtr<HTMLDocumentParser> protect(this);
284 if (m_haveBackgroundParser) {
285 pumpPendingSpeculations();
289 // We should never be here unless we can pump immediately. Call pumpTokenizer()
290 // directly so that ASSERTS will fire if we're wrong.
291 pumpTokenizer(AllowYield);
295 void HTMLDocumentParser::runScriptsForPausedTreeBuilder()
297 ASSERT(scriptingContentIsAllowed(parserContentPolicy()));
299 TextPosition scriptStartPosition = TextPosition::belowRangePosition();
300 RefPtrWillBeRawPtr<Element> scriptElement = m_treeBuilder->takeScriptToProcess(scriptStartPosition);
301 // We will not have a scriptRunner when parsing a DocumentFragment.
303 m_scriptRunner->execute(scriptElement.release(), scriptStartPosition);
306 bool HTMLDocumentParser::canTakeNextToken(SynchronousMode mode, PumpSession& session)
311 ASSERT(!m_haveBackgroundParser || mode == ForceSynchronous);
313 if (isWaitingForScripts()) {
314 if (mode == AllowYield)
315 session.didSeeScript = true;
317 // If we don't run the script, we cannot allow the next token to be taken.
318 if (session.needsYield)
321 // If we're paused waiting for a script, we try to execute scripts before continuing.
322 runScriptsForPausedTreeBuilder();
325 if (isWaitingForScripts())
329 // FIXME: It's wrong for the HTMLDocumentParser to reach back to the
330 // LocalFrame, but this approach is how the old parser handled
331 // stopping when the page assigns window.location. What really
332 // should happen is that assigning window.location causes the
333 // parser to stop parsing cleanly. The problem is we're not
334 // perpared to do that at every point where we run JavaScript.
335 if (!isParsingFragment()
336 && document()->frame() && document()->frame()->navigationScheduler().locationChangePending())
339 if (mode == AllowYield)
340 m_parserScheduler->checkForYieldBeforeToken(session);
345 void HTMLDocumentParser::didReceiveParsedChunkFromBackgroundParser(PassOwnPtr<ParsedChunk> chunk)
347 TRACE_EVENT0("blink", "HTMLDocumentParser::didReceiveParsedChunkFromBackgroundParser");
349 // alert(), runModalDialog, and the JavaScript Debugger all run nested event loops
350 // which can cause this method to be re-entered. We detect re-entry using
351 // hasActiveParser(), save the chunk as a speculation, and return.
352 if (isWaitingForScripts() || !m_speculations.isEmpty() || document()->activeParserCount() > 0) {
353 m_preloader->takeAndPreload(chunk->preloads);
354 m_speculations.append(chunk);
358 // processParsedChunkFromBackgroundParser can cause this parser to be detached from the Document,
359 // but we need to ensure it isn't deleted yet.
360 RefPtrWillBeRawPtr<HTMLDocumentParser> protect(this);
362 ASSERT(m_speculations.isEmpty());
363 chunk->preloads.clear(); // We don't need to preload because we're going to parse immediately.
364 m_speculations.append(chunk);
365 pumpPendingSpeculations();
368 void HTMLDocumentParser::didReceiveEncodingDataFromBackgroundParser(const DocumentEncodingData& data)
370 document()->setEncodingData(data);
373 void HTMLDocumentParser::validateSpeculations(PassOwnPtr<ParsedChunk> chunk)
376 if (isWaitingForScripts()) {
377 // We're waiting on a network script, just save the chunk, we'll get
378 // a second validateSpeculations call after the script completes.
379 // This call should have been made immediately after runScriptsForPausedTreeBuilder
380 // which may have started a network load and left us waiting.
381 ASSERT(!m_lastChunkBeforeScript);
382 m_lastChunkBeforeScript = chunk;
386 ASSERT(!m_lastChunkBeforeScript);
387 OwnPtr<HTMLTokenizer> tokenizer = m_tokenizer.release();
388 OwnPtr<HTMLToken> token = m_token.release();
391 // There must not have been any changes to the HTMLTokenizer state on
392 // the main thread, which means the speculation buffer is correct.
396 // Currently we're only smart enough to reuse the speculation buffer if the tokenizer
397 // both starts and ends in the DataState. That state is simplest because the HTMLToken
398 // is always in the Uninitialized state. We should consider whether we can reuse the
399 // speculation buffer in other states, but we'd likely need to do something more
400 // sophisticated with the HTMLToken.
401 if (chunk->tokenizerState == HTMLTokenizer::DataState
402 && tokenizer->state() == HTMLTokenizer::DataState
403 && m_input.current().isEmpty()
404 && chunk->treeBuilderState == HTMLTreeBuilderSimulator::stateFor(m_treeBuilder.get())) {
405 ASSERT(token->isUninitialized());
409 discardSpeculationsAndResumeFrom(chunk, token.release(), tokenizer.release());
412 void HTMLDocumentParser::discardSpeculationsAndResumeFrom(PassOwnPtr<ParsedChunk> lastChunkBeforeScript, PassOwnPtr<HTMLToken> token, PassOwnPtr<HTMLTokenizer> tokenizer)
414 m_weakFactory.revokeAll();
415 m_speculations.clear();
417 OwnPtr<BackgroundHTMLParser::Checkpoint> checkpoint = adoptPtr(new BackgroundHTMLParser::Checkpoint);
418 checkpoint->parser = m_weakFactory.createWeakPtr();
419 checkpoint->token = token;
420 checkpoint->tokenizer = tokenizer;
421 checkpoint->treeBuilderState = HTMLTreeBuilderSimulator::stateFor(m_treeBuilder.get());
422 checkpoint->inputCheckpoint = lastChunkBeforeScript->inputCheckpoint;
423 checkpoint->preloadScannerCheckpoint = lastChunkBeforeScript->preloadScannerCheckpoint;
424 checkpoint->unparsedInput = m_input.current().toString().isolatedCopy();
425 m_input.current().clear(); // FIXME: This should be passed in instead of cleared.
427 ASSERT(checkpoint->unparsedInput.isSafeToSendToAnotherThread());
428 HTMLParserThread::shared()->postTask(bind(&BackgroundHTMLParser::resumeFrom, m_backgroundParser, checkpoint.release()));
431 void HTMLDocumentParser::processParsedChunkFromBackgroundParser(PassOwnPtr<ParsedChunk> popChunk)
433 TRACE_EVENT0("blink", "HTMLDocumentParser::processParsedChunkFromBackgroundParser");
435 ASSERT_WITH_SECURITY_IMPLICATION(!document()->activeParserCount());
436 ASSERT(!isParsingFragment());
437 ASSERT(!isWaitingForScripts());
438 ASSERT(!isStopped());
440 // ASSERT that this object is both attached to the Document and protected.
441 ASSERT(refCount() >= 2);
443 ASSERT(shouldUseThreading());
444 ASSERT(!m_tokenizer);
446 ASSERT(!m_lastChunkBeforeScript);
448 ActiveParserSession session(contextForParsingSession());
450 OwnPtr<ParsedChunk> chunk(popChunk);
451 OwnPtr<CompactHTMLTokenStream> tokens = chunk->tokens.release();
453 HTMLParserThread::shared()->postTask(bind(&BackgroundHTMLParser::startedChunkWithCheckpoint, m_backgroundParser, chunk->inputCheckpoint));
455 for (XSSInfoStream::const_iterator it = chunk->xssInfos.begin(); it != chunk->xssInfos.end(); ++it) {
456 m_textPosition = (*it)->m_textPosition;
457 m_xssAuditorDelegate.didBlockScript(**it);
462 for (Vector<CompactHTMLToken>::const_iterator it = tokens->begin(); it != tokens->end(); ++it) {
463 ASSERT(!isWaitingForScripts());
465 if (document()->frame() && document()->frame()->navigationScheduler().locationChangePending()) {
467 // To match main-thread parser behavior (which never checks locationChangePending on the EOF path)
468 // we peek to see if this chunk has an EOF and process it anyway.
469 if (tokens->last().type() == HTMLToken::EndOfFile) {
470 ASSERT(m_speculations.isEmpty()); // There should never be any chunks after the EOF.
471 prepareToStopParsing();
476 m_textPosition = it->textPosition();
478 constructTreeFromCompactHTMLToken(*it);
483 if (isWaitingForScripts()) {
484 ASSERT(it + 1 == tokens->end()); // The </script> is assumed to be the last token of this bunch.
485 runScriptsForPausedTreeBuilder();
486 validateSpeculations(chunk.release());
490 if (it->type() == HTMLToken::EndOfFile) {
491 ASSERT(it + 1 == tokens->end()); // The EOF is assumed to be the last token of this bunch.
492 ASSERT(m_speculations.isEmpty()); // There should never be any chunks after the EOF.
493 prepareToStopParsing();
497 ASSERT(!m_tokenizer);
501 // Make sure all required pending text nodes are emitted before returning.
502 // This leaves "script", "style" and "svg" nodes text nodes intact.
504 m_treeBuilder->flush(FlushIfAtTextLimit);
507 void HTMLDocumentParser::pumpPendingSpeculations()
509 // FIXME: Share this constant with the parser scheduler.
510 const double parserTimeLimit = 0.500;
513 // ASSERT that this object is both attached to the Document and protected.
514 ASSERT(refCount() >= 2);
516 // If this assert fails, you need to call validateSpeculations to make sure
517 // m_tokenizer and m_token don't have state that invalidates m_speculations.
518 ASSERT(!m_tokenizer);
520 ASSERT(!m_lastChunkBeforeScript);
521 ASSERT(!isWaitingForScripts());
522 ASSERT(!isStopped());
524 // FIXME: Pass in current input length.
525 TRACE_EVENT_BEGIN1(TRACE_DISABLED_BY_DEFAULT("devtools.timeline"), "ParseHTML", "beginData", InspectorParseHtmlEvent::beginData(document(), lineNumber().zeroBasedInt()));
526 TRACE_EVENT_INSTANT1(TRACE_DISABLED_BY_DEFAULT("devtools.timeline.stack"), "CallStack", "stack", InspectorCallStackEvent::currentCallStack());
527 // FIXME(361045): remove InspectorInstrumentation calls once DevTools Timeline migrates to tracing.
528 InspectorInstrumentationCookie cookie = InspectorInstrumentation::willWriteHTML(document(), lineNumber().zeroBasedInt());
530 double startTime = currentTime();
532 while (!m_speculations.isEmpty()) {
533 processParsedChunkFromBackgroundParser(m_speculations.takeFirst());
535 // Always check isStopped first as m_document may be null.
536 if (isStopped() || isWaitingForScripts())
539 if (currentTime() - startTime > parserTimeLimit && !m_speculations.isEmpty()) {
540 m_parserScheduler->scheduleForResume();
545 TRACE_EVENT_END1(TRACE_DISABLED_BY_DEFAULT("devtools.timeline"), "ParseHTML", "endLine", lineNumber().zeroBasedInt());
546 // FIXME(361045): remove InspectorInstrumentation calls once DevTools Timeline migrates to tracing.
547 InspectorInstrumentation::didWriteHTML(cookie, lineNumber().zeroBasedInt());
548 TRACE_EVENT_INSTANT1(TRACE_DISABLED_BY_DEFAULT("devtools.timeline"), "UpdateCounters", "data", InspectorUpdateCountersEvent::data());
551 void HTMLDocumentParser::forcePlaintextForTextDocument()
553 if (shouldUseThreading()) {
554 // This method is called before any data is appended, so we have to start
555 // the background parser ourselves.
556 if (!m_haveBackgroundParser)
557 startBackgroundParser();
559 HTMLParserThread::shared()->postTask(bind(&BackgroundHTMLParser::forcePlaintextForTextDocument, m_backgroundParser));
561 m_tokenizer->setState(HTMLTokenizer::PLAINTEXTState);
564 Document* HTMLDocumentParser::contextForParsingSession()
566 // The parsing session should interact with the document only when parsing
567 // non-fragments. Otherwise, we might delay the load event mistakenly.
568 if (isParsingFragment())
573 static PassRefPtr<MediaValues> createMediaValues(Document* document)
576 RefPtr<MediaValues> mediaValues = MediaValuesCached::create(*document);
577 ASSERT(mediaValues->isSafeToSendToAnotherThread());
581 void HTMLDocumentParser::pumpTokenizer(SynchronousMode mode)
583 ASSERT(!isStopped());
584 ASSERT(!isScheduledForResume());
586 // ASSERT that this object is both attached to the Document and protected.
587 ASSERT(refCount() >= 2);
591 ASSERT(!m_haveBackgroundParser || mode == ForceSynchronous);
593 PumpSession session(m_pumpSessionNestingLevel, contextForParsingSession());
595 // We tell the InspectorInstrumentation about every pump, even if we
596 // end up pumping nothing. It can filter out empty pumps itself.
597 // FIXME: m_input.current().length() is only accurate if we
598 // end up parsing the whole buffer in this pump. We should pass how
599 // much we parsed as part of didWriteHTML instead of willWriteHTML.
600 TRACE_EVENT_BEGIN1(TRACE_DISABLED_BY_DEFAULT("devtools.timeline"), "ParseHTML", "beginData", InspectorParseHtmlEvent::beginData(document(), m_input.current().currentLine().zeroBasedInt()));
601 TRACE_EVENT_INSTANT1(TRACE_DISABLED_BY_DEFAULT("devtools.timeline.stack"), "CallStack", "stack", InspectorCallStackEvent::currentCallStack());
602 // FIXME(361045): remove InspectorInstrumentation calls once DevTools Timeline migrates to tracing.
603 InspectorInstrumentationCookie cookie = InspectorInstrumentation::willWriteHTML(document(), m_input.current().currentLine().zeroBasedInt());
605 m_xssAuditor.init(document(), &m_xssAuditorDelegate);
607 while (canTakeNextToken(mode, session) && !session.needsYield) {
608 if (!isParsingFragment())
609 m_sourceTracker.start(m_input.current(), m_tokenizer.get(), token());
611 if (!m_tokenizer->nextToken(m_input.current(), token()))
614 if (!isParsingFragment()) {
615 m_sourceTracker.end(m_input.current(), m_tokenizer.get(), token());
617 // We do not XSS filter innerHTML, which means we (intentionally) fail
618 // http/tests/security/xssAuditor/dom-write-innerHTML.html
619 if (OwnPtr<XSSInfo> xssInfo = m_xssAuditor.filterToken(FilterTokenRequest(token(), m_sourceTracker, m_tokenizer->shouldAllowCDATA())))
620 m_xssAuditorDelegate.didBlockScript(*xssInfo);
623 constructTreeFromHTMLToken(token());
624 ASSERT(token().isUninitialized());
628 // Ensure we haven't been totally deref'ed after pumping. Any caller of this
629 // function should be holding a RefPtr to this to ensure we weren't deleted.
630 ASSERT(refCount() >= 1);
636 // There should only be PendingText left since the tree-builder always flushes
637 // the task queue before returning. In case that ever changes, crash.
638 if (mode == ForceSynchronous)
639 m_treeBuilder->flush(FlushAlways);
640 RELEASE_ASSERT(!isStopped());
642 if (session.needsYield)
643 m_parserScheduler->scheduleForResume();
645 if (isWaitingForScripts()) {
646 ASSERT(m_tokenizer->state() == HTMLTokenizer::DataState);
647 if (!m_preloadScanner) {
648 m_preloadScanner = adoptPtr(new HTMLPreloadScanner(m_options, document()->url(), createMediaValues(document())));
649 m_preloadScanner->appendToEnd(m_input.current());
651 m_preloadScanner->scan(m_preloader.get(), document()->baseElementURL());
654 TRACE_EVENT_END1(TRACE_DISABLED_BY_DEFAULT("devtools.timeline"), "ParseHTML", "endLine", m_input.current().currentLine().zeroBasedInt());
655 // FIXME(361045): remove InspectorInstrumentation calls once DevTools Timeline migrates to tracing.
656 InspectorInstrumentation::didWriteHTML(cookie, m_input.current().currentLine().zeroBasedInt());
659 void HTMLDocumentParser::constructTreeFromHTMLToken(HTMLToken& rawToken)
661 AtomicHTMLToken token(rawToken);
663 // We clear the rawToken in case constructTreeFromAtomicToken
664 // synchronously re-enters the parser. We don't clear the token immedately
665 // for Character tokens because the AtomicHTMLToken avoids copying the
666 // characters by keeping a pointer to the underlying buffer in the
667 // HTMLToken. Fortunately, Character tokens can't cause us to re-enter
670 // FIXME: Stop clearing the rawToken once we start running the parser off
671 // the main thread or once we stop allowing synchronous JavaScript
672 // execution from parseAttribute.
673 if (rawToken.type() != HTMLToken::Character)
676 m_treeBuilder->constructTree(&token);
678 if (!rawToken.isUninitialized()) {
679 ASSERT(rawToken.type() == HTMLToken::Character);
684 void HTMLDocumentParser::constructTreeFromCompactHTMLToken(const CompactHTMLToken& compactToken)
686 AtomicHTMLToken token(compactToken);
687 m_treeBuilder->constructTree(&token);
690 bool HTMLDocumentParser::hasInsertionPoint()
692 // FIXME: The wasCreatedByScript() branch here might not be fully correct.
693 // Our model of the EOF character differs slightly from the one in
694 // the spec because our treatment is uniform between network-sourced
695 // and script-sourced input streams whereas the spec treats them
697 return m_input.hasInsertionPoint() || (wasCreatedByScript() && !m_input.haveSeenEndOfFile());
700 void HTMLDocumentParser::insert(const SegmentedString& source)
705 TRACE_EVENT1("blink", "HTMLDocumentParser::insert", "source_length", source.length());
707 // pumpTokenizer can cause this parser to be detached from the Document,
708 // but we need to ensure it isn't deleted yet.
709 RefPtrWillBeRawPtr<HTMLDocumentParser> protect(this);
712 ASSERT(!inPumpSession());
713 ASSERT(m_haveBackgroundParser || wasCreatedByScript());
714 m_token = adoptPtr(new HTMLToken);
715 m_tokenizer = HTMLTokenizer::create(m_options);
718 SegmentedString excludedLineNumberSource(source);
719 excludedLineNumberSource.setExcludeLineNumbers();
720 m_input.insertAtCurrentInsertionPoint(excludedLineNumberSource);
721 pumpTokenizerIfPossible(ForceSynchronous);
723 if (isWaitingForScripts()) {
724 // Check the document.write() output with a separate preload scanner as
725 // the main scanner can't deal with insertions.
726 if (!m_insertionPreloadScanner)
727 m_insertionPreloadScanner = adoptPtr(new HTMLPreloadScanner(m_options, document()->url(), createMediaValues(document())));
729 m_insertionPreloadScanner->appendToEnd(source);
730 m_insertionPreloadScanner->scan(m_preloader.get(), document()->baseElementURL());
736 void HTMLDocumentParser::startBackgroundParser()
738 ASSERT(!isStopped());
739 ASSERT(shouldUseThreading());
740 ASSERT(!m_haveBackgroundParser);
741 m_haveBackgroundParser = true;
743 RefPtr<WeakReference<BackgroundHTMLParser> > reference = WeakReference<BackgroundHTMLParser>::createUnbound();
744 m_backgroundParser = WeakPtr<BackgroundHTMLParser>(reference);
746 // TODO(oysteine): Disabled due to crbug.com/398076 until a full fix can be implemented.
747 if (RuntimeEnabledFeatures::threadedParserDataReceiverEnabled()) {
748 if (DocumentLoader* loader = document()->loader())
749 loader->attachThreadedDataReceiver(adoptPtr(new ParserDataReceiver(m_backgroundParser)));
752 OwnPtr<BackgroundHTMLParser::Configuration> config = adoptPtr(new BackgroundHTMLParser::Configuration);
753 config->options = m_options;
754 config->parser = m_weakFactory.createWeakPtr();
755 config->xssAuditor = adoptPtr(new XSSAuditor);
756 config->xssAuditor->init(document(), &m_xssAuditorDelegate);
757 config->preloadScanner = adoptPtr(new TokenPreloadScanner(document()->url().copy(), createMediaValues(document())));
758 config->decoder = takeDecoder();
760 ASSERT(config->xssAuditor->isSafeToSendToAnotherThread());
761 ASSERT(config->preloadScanner->isSafeToSendToAnotherThread());
762 HTMLParserThread::shared()->postTask(bind(&BackgroundHTMLParser::start, reference.release(), config.release()));
765 void HTMLDocumentParser::stopBackgroundParser()
767 ASSERT(shouldUseThreading());
768 ASSERT(m_haveBackgroundParser);
769 m_haveBackgroundParser = false;
771 HTMLParserThread::shared()->postTask(bind(&BackgroundHTMLParser::stop, m_backgroundParser));
772 m_weakFactory.revokeAll();
775 void HTMLDocumentParser::append(PassRefPtr<StringImpl> inputSource)
780 // We should never reach this point if we're using a parser thread,
781 // as appendBytes() will directly ship the data to the thread.
782 ASSERT(!shouldUseThreading());
784 // pumpTokenizer can cause this parser to be detached from the Document,
785 // but we need to ensure it isn't deleted yet.
786 RefPtrWillBeRawPtr<HTMLDocumentParser> protect(this);
787 TRACE_EVENT1("net", "HTMLDocumentParser::append", "size", inputSource->length());
788 String source(inputSource);
790 if (m_preloadScanner) {
791 if (m_input.current().isEmpty() && !isWaitingForScripts()) {
792 // We have parsed until the end of the current input and so are now moving ahead of the preload scanner.
793 // Clear the scanner so we know to scan starting from the current input point if we block again.
794 m_preloadScanner.clear();
796 m_preloadScanner->appendToEnd(source);
797 if (isWaitingForScripts())
798 m_preloadScanner->scan(m_preloader.get(), document()->baseElementURL());
802 m_input.appendToEnd(source);
804 if (inPumpSession()) {
805 // We've gotten data off the network in a nested write.
806 // We don't want to consume any more of the input stream now. Do
807 // not worry. We'll consume this data in a less-nested write().
811 // A couple pinToMainThread() callers require synchronous parsing, but can't
812 // easily use the insert() method, so we hack append() for them to be synchronous.
813 // javascript: url handling is one such caller.
814 // FIXME: This is gross, and we should separate the concept of synchronous parsing
815 // from insert() so that only document.write() uses insert.
816 if (m_isPinnedToMainThread)
817 pumpTokenizerIfPossible(ForceSynchronous);
819 pumpTokenizerIfPossible(AllowYield);
824 void HTMLDocumentParser::end()
826 ASSERT(!isDetached());
827 ASSERT(!isScheduledForResume());
829 if (m_haveBackgroundParser)
830 stopBackgroundParser();
832 // Informs the the rest of WebCore that parsing is really finished (and deletes this).
833 m_treeBuilder->finished();
835 DocumentParser::stopParsing();
838 void HTMLDocumentParser::attemptToRunDeferredScriptsAndEnd()
840 ASSERT(isStopping());
841 // FIXME: It may not be correct to disable this for the background parser.
842 // That means hasInsertionPoint() may not be correct in some cases.
843 ASSERT(!hasInsertionPoint() || m_haveBackgroundParser);
844 if (m_scriptRunner && !m_scriptRunner->executeScriptsWaitingForParsing())
849 void HTMLDocumentParser::attemptToEnd()
851 // finish() indicates we will not receive any more data. If we are waiting on
852 // an external script to load, we can't finish parsing quite yet.
854 if (shouldDelayEnd()) {
855 m_endWasDelayed = true;
858 prepareToStopParsing();
861 void HTMLDocumentParser::endIfDelayed()
863 // If we've already been detached, don't bother ending.
867 if (!m_endWasDelayed || shouldDelayEnd())
870 m_endWasDelayed = false;
871 prepareToStopParsing();
874 void HTMLDocumentParser::finish()
876 // FIXME: We should ASSERT(!m_parserStopped) here, since it does not
877 // makes sense to call any methods on DocumentParser once it's been stopped.
878 // However, FrameLoader::stop calls DocumentParser::finish unconditionally.
880 // flush may ending up executing arbitrary script, and possibly detach the parser.
881 RefPtrWillBeRawPtr<HTMLDocumentParser> protect(this);
886 // Empty documents never got an append() call, and thus have never started
887 // a background parser. In those cases, we ignore shouldUseThreading()
888 // and fall through to the non-threading case.
889 if (m_haveBackgroundParser) {
890 if (!m_input.haveSeenEndOfFile())
891 m_input.closeWithoutMarkingEndOfFile();
892 HTMLParserThread::shared()->postTask(bind(&BackgroundHTMLParser::finish, m_backgroundParser));
898 // We're finishing before receiving any data. Rather than booting up
899 // the background parser just to spin it down, we finish parsing
901 m_token = adoptPtr(new HTMLToken);
902 m_tokenizer = HTMLTokenizer::create(m_options);
905 // We're not going to get any more data off the network, so we tell the
906 // input stream we've reached the end of file. finish() can be called more
907 // than once, if the first time does not call end().
908 if (!m_input.haveSeenEndOfFile())
909 m_input.markEndOfFile();
914 bool HTMLDocumentParser::isExecutingScript() const
918 return m_scriptRunner->isExecutingScript();
921 OrdinalNumber HTMLDocumentParser::lineNumber() const
923 if (m_haveBackgroundParser)
924 return m_textPosition.m_line;
926 return m_input.current().currentLine();
929 TextPosition HTMLDocumentParser::textPosition() const
931 if (m_haveBackgroundParser)
932 return m_textPosition;
934 const SegmentedString& currentString = m_input.current();
935 OrdinalNumber line = currentString.currentLine();
936 OrdinalNumber column = currentString.currentColumn();
938 return TextPosition(line, column);
941 bool HTMLDocumentParser::isWaitingForScripts() const
943 // When the TreeBuilder encounters a </script> tag, it returns to the HTMLDocumentParser
944 // where the script is transfered from the treebuilder to the script runner.
945 // The script runner will hold the script until its loaded and run. During
946 // any of this time, we want to count ourselves as "waiting for a script" and thus
947 // run the preload scanner, as well as delay completion of parsing.
948 bool treeBuilderHasBlockingScript = m_treeBuilder->hasParserBlockingScript();
949 bool scriptRunnerHasBlockingScript = m_scriptRunner && m_scriptRunner->hasParserBlockingScript();
950 // Since the parser is paused while a script runner has a blocking script, it should
951 // never be possible to end up with both objects holding a blocking script.
952 ASSERT(!(treeBuilderHasBlockingScript && scriptRunnerHasBlockingScript));
953 // If either object has a blocking script, the parser should be paused.
954 return treeBuilderHasBlockingScript || scriptRunnerHasBlockingScript;
957 void HTMLDocumentParser::resumeParsingAfterScriptExecution()
959 ASSERT(!isExecutingScript());
960 ASSERT(!isWaitingForScripts());
962 if (m_haveBackgroundParser) {
963 validateSpeculations(m_lastChunkBeforeScript.release());
964 ASSERT(!m_lastChunkBeforeScript);
965 // processParsedChunkFromBackgroundParser can cause this parser to be detached from the Document,
966 // but we need to ensure it isn't deleted yet.
967 RefPtrWillBeRawPtr<HTMLDocumentParser> protect(this);
968 pumpPendingSpeculations();
972 m_insertionPreloadScanner.clear();
973 pumpTokenizerIfPossible(AllowYield);
977 void HTMLDocumentParser::appendCurrentInputStreamToPreloadScannerAndScan()
979 ASSERT(m_preloadScanner);
980 m_preloadScanner->appendToEnd(m_input.current());
981 m_preloadScanner->scan(m_preloader.get(), document()->baseElementURL());
984 void HTMLDocumentParser::notifyScriptLoaded(Resource* cachedResource)
986 // pumpTokenizer can cause this parser to be detached from the Document,
987 // but we need to ensure it isn't deleted yet.
988 RefPtrWillBeRawPtr<HTMLDocumentParser> protect(this);
990 ASSERT(m_scriptRunner);
991 ASSERT(!isExecutingScript());
993 attemptToRunDeferredScriptsAndEnd();
997 m_scriptRunner->executeScriptsWaitingForLoad(cachedResource);
998 if (!isWaitingForScripts())
999 resumeParsingAfterScriptExecution();
1002 void HTMLDocumentParser::executeScriptsWaitingForResources()
1004 // Document only calls this when the Document owns the DocumentParser
1005 // so this will not be called in the DocumentFragment case.
1006 ASSERT(m_scriptRunner);
1007 // Ignore calls unless we have a script blocking the parser waiting on a
1008 // stylesheet load. Otherwise we are currently parsing and this
1009 // is a re-entrant call from encountering a </ style> tag.
1010 if (!m_scriptRunner->hasScriptsWaitingForResources())
1013 // pumpTokenizer can cause this parser to be detached from the Document,
1014 // but we need to ensure it isn't deleted yet.
1015 RefPtrWillBeRawPtr<HTMLDocumentParser> protect(this);
1016 m_scriptRunner->executeScriptsWaitingForResources();
1017 if (!isWaitingForScripts())
1018 resumeParsingAfterScriptExecution();
1021 void HTMLDocumentParser::parseDocumentFragment(const String& source, DocumentFragment* fragment, Element* contextElement, ParserContentPolicy parserContentPolicy)
1023 RefPtrWillBeRawPtr<HTMLDocumentParser> parser = HTMLDocumentParser::create(fragment, contextElement, parserContentPolicy);
1024 parser->insert(source); // Use insert() so that the parser will not yield.
1026 ASSERT(!parser->processingData()); // Make sure we're done. <rdar://problem/3963151>
1027 parser->detach(); // Allows ~DocumentParser to assert it was detached before destruction.
1030 void HTMLDocumentParser::suspendScheduledTasks()
1032 if (m_parserScheduler)
1033 m_parserScheduler->suspend();
1036 void HTMLDocumentParser::resumeScheduledTasks()
1038 if (m_parserScheduler)
1039 m_parserScheduler->resume();
1042 void HTMLDocumentParser::appendBytes(const char* data, size_t length)
1044 if (!length || isStopped())
1047 if (shouldUseThreading()) {
1048 if (!m_haveBackgroundParser)
1049 startBackgroundParser();
1051 OwnPtr<Vector<char> > buffer = adoptPtr(new Vector<char>(length));
1052 memcpy(buffer->data(), data, length);
1053 TRACE_EVENT1("net", "HTMLDocumentParser::appendBytes", "size", (unsigned)length);
1055 HTMLParserThread::shared()->postTask(bind(&BackgroundHTMLParser::appendRawBytesFromMainThread, m_backgroundParser, buffer.release()));
1059 DecodedDataDocumentParser::appendBytes(data, length);
1062 void HTMLDocumentParser::flush()
1064 // If we've got no decoder, we never received any data.
1065 if (isDetached() || needsDecoder())
1068 if (m_haveBackgroundParser)
1069 HTMLParserThread::shared()->postTask(bind(&BackgroundHTMLParser::flush, m_backgroundParser));
1071 DecodedDataDocumentParser::flush();
1074 void HTMLDocumentParser::setDecoder(PassOwnPtr<TextResourceDecoder> decoder)
1077 DecodedDataDocumentParser::setDecoder(decoder);
1079 if (m_haveBackgroundParser)
1080 HTMLParserThread::shared()->postTask(bind(&BackgroundHTMLParser::setDecoder, m_backgroundParser, takeDecoder()));