1 /*-------------------------------------------------------------------------
2 * drawElements Quality Program OpenGL ES 3.0 Module
3 * -------------------------------------------------
5 * Copyright 2014 The Android Open Source Project
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
21 * \brief Flush and finish tests.
22 *//*--------------------------------------------------------------------*/
24 #include "es3fFlushFinishTests.hpp"
26 #include "gluRenderContext.hpp"
27 #include "gluObjectWrapper.hpp"
28 #include "gluShaderProgram.hpp"
29 #include "gluDrawUtil.hpp"
31 #include "glsCalibration.hpp"
33 #include "tcuTestLog.hpp"
34 #include "tcuRenderTarget.hpp"
35 #include "tcuCPUWarmup.hpp"
37 #include "glwEnums.hpp"
38 #include "glwFunctions.hpp"
40 #include "deRandom.hpp"
41 #include "deStringUtil.hpp"
59 using deqp::gls::theilSenLinearRegression;
60 using deqp::gls::LineParameters;
67 MAX_VIEWPORT_SIZE = 256,
68 MAX_SAMPLE_DURATION_US = 150*1000,
70 MIN_DRAW_CALL_COUNT = 10,
71 MAX_DRAW_CALL_COUNT = 1<<20,
72 MAX_SHADER_ITER_COUNT = 1<<10,
74 NUM_VERIFICATION_SAMPLES = 3,
75 MAX_CALIBRATION_ATTEMPTS = 5
78 DE_STATIC_ASSERT(MAX_SAMPLE_DURATION_US < 1000*WAIT_TIME_MS);
80 const float NO_CORR_COEF_THRESHOLD = 0.1f;
81 const float FLUSH_COEF_THRESHOLD = 0.2f;
82 const float CORRELATED_COEF_THRESHOLD = 0.5f;
83 const float CALIBRATION_VERIFICATION_THRESHOLD = 0.10f; // Rendering time needs to be within 10% of MAX_SAMPLE_DURATION_US
85 static void busyWait (int milliseconds)
87 const deUint64 startTime = deGetMicroseconds();
92 for (int i = 0; i < 10; i++)
95 if (deGetMicroseconds()-startTime >= deUint64(1000*milliseconds))
100 class CalibrationFailedException : public std::runtime_error
103 CalibrationFailedException (const std::string& reason) : std::runtime_error(reason) {}
106 class FlushFinishCase : public TestCase
109 enum ExpectedBehavior
111 EXPECT_COEF_LESS_THAN = 0,
112 EXPECT_COEF_GREATER_THAN,
115 FlushFinishCase (Context& context,
117 const char* description,
118 ExpectedBehavior waitBehavior,
120 ExpectedBehavior readBehavior,
121 float readThreshold);
122 ~FlushFinishCase (void);
126 IterateResult iterate (void);
133 deUint64 readPixelsTime;
136 struct CalibrationParams
138 int numItersInShader;
143 virtual void waitForGL (void) = 0;
146 FlushFinishCase (const FlushFinishCase&);
147 FlushFinishCase& operator= (const FlushFinishCase&);
149 CalibrationParams calibrate (void);
150 void verifyCalibration (const CalibrationParams& params);
152 void analyzeResults (const std::vector<Sample>& samples, const CalibrationParams& calibrationParams);
154 void setupRenderState (void);
155 void setShaderIterCount (int numIters);
156 void render (int numDrawCalls);
157 void readPixels (void);
159 const ExpectedBehavior m_waitBehavior;
160 const float m_waitThreshold;
161 const ExpectedBehavior m_readBehavior;
162 const float m_readThreshold;
164 glu::ShaderProgram* m_program;
168 FlushFinishCase::FlushFinishCase (Context& context, const char* name, const char* description, ExpectedBehavior waitBehavior, float waitThreshold, ExpectedBehavior readBehavior, float readThreshold)
169 : TestCase (context, name, description)
170 , m_waitBehavior (waitBehavior)
171 , m_waitThreshold (waitThreshold)
172 , m_readBehavior (readBehavior)
173 , m_readThreshold (readThreshold)
174 , m_program (DE_NULL)
179 FlushFinishCase::~FlushFinishCase (void)
181 FlushFinishCase::deinit();
184 void FlushFinishCase::init (void)
186 DE_ASSERT(!m_program);
188 m_program = new glu::ShaderProgram(m_context.getRenderContext(),
189 glu::ProgramSources()
190 << glu::VertexSource(
192 "in highp vec4 a_position;\n"
193 "out highp vec4 v_coord;\n"
196 " gl_Position = a_position;\n"
197 " v_coord = a_position;\n"
199 << glu::FragmentSource(
201 "uniform highp int u_numIters;\n"
202 "in highp vec4 v_coord;\n"
203 "out mediump vec4 o_color;\n"
206 " highp vec4 color = v_coord;\n"
207 " for (int i = 0; i < u_numIters; i++)\n"
208 " color = sin(color);\n"
209 " o_color = color;\n"
212 if (!m_program->isOk())
214 m_testCtx.getLog() << *m_program;
217 TCU_FAIL("Compile failed");
220 m_iterCountLoc = m_context.getRenderContext().getFunctions().getUniformLocation(m_program->getProgram(), "u_numIters");
221 TCU_CHECK(m_iterCountLoc >= 0);
224 void FlushFinishCase::deinit (void)
230 tcu::TestLog& operator<< (tcu::TestLog& log, const FlushFinishCase::Sample& sample)
232 log << TestLog::Message << sample.numDrawCalls << " calls:\t" << sample.submitTime << " us submit,\t" << sample.waitTime << " us wait,\t" << sample.readPixelsTime << " us read" << TestLog::EndMessage;
236 void FlushFinishCase::setupRenderState (void)
238 const glw::Functions& gl = m_context.getRenderContext().getFunctions();
239 const int posLoc = gl.getAttribLocation(m_program->getProgram(), "a_position");
240 const int viewportW = de::min<int>(m_context.getRenderTarget().getWidth(), MAX_VIEWPORT_SIZE);
241 const int viewportH = de::min<int>(m_context.getRenderTarget().getHeight(), MAX_VIEWPORT_SIZE);
243 static const float s_positions[] =
251 TCU_CHECK(posLoc >= 0);
253 gl.viewport(0, 0, viewportW, viewportH);
254 gl.useProgram(m_program->getProgram());
255 gl.enableVertexAttribArray(posLoc);
256 gl.vertexAttribPointer(posLoc, 2, GL_FLOAT, GL_FALSE, 0, &s_positions[0]);
258 gl.blendFunc(GL_ONE, GL_ONE);
259 gl.blendEquation(GL_FUNC_ADD);
260 GLU_EXPECT_NO_ERROR(gl.getError(), "Failed to set up render state");
263 void FlushFinishCase::setShaderIterCount (int numIters)
265 const glw::Functions& gl = m_context.getRenderContext().getFunctions();
266 gl.uniform1i(m_iterCountLoc, numIters);
269 void FlushFinishCase::render (int numDrawCalls)
271 const glw::Functions& gl = m_context.getRenderContext().getFunctions();
273 const deUint8 indices[] = { 0, 1, 2, 2, 1, 3 };
275 gl.clear(GL_COLOR_BUFFER_BIT|GL_DEPTH_BUFFER_BIT|GL_STENCIL_BUFFER_BIT);
277 for (int ndx = 0; ndx < numDrawCalls; ndx++)
278 gl.drawElements(GL_TRIANGLES, DE_LENGTH_OF_ARRAY(indices), GL_UNSIGNED_BYTE, &indices[0]);
281 void FlushFinishCase::readPixels (void)
283 const glw::Functions& gl = m_context.getRenderContext().getFunctions();
286 gl.readPixels(0, 0, 1, 1, GL_RGBA, GL_UNSIGNED_BYTE, &tmp);
289 FlushFinishCase::CalibrationParams FlushFinishCase::calibrate (void)
291 tcu::ScopedLogSection section (m_testCtx.getLog(), "CalibrationInfo", "Calibration info");
292 CalibrationParams params;
294 // Step 1: find iteration count that results in rougly 1/10th of target maximum sample duration.
296 const deUint64 targetDurationUs = MAX_SAMPLE_DURATION_US/100;
297 deUint64 prevDuration = 0;
298 int prevIterCount = 1;
299 int curIterCount = 1;
301 m_testCtx.getLog() << TestLog::Message << "Calibrating shader iteration count, target duration = " << targetDurationUs << " us" << TestLog::EndMessage;
305 deUint64 curDuration;
307 setShaderIterCount(curIterCount);
308 render(1); // \note Submit time is ignored
311 const deUint64 startTime = deGetMicroseconds();
313 curDuration = deGetMicroseconds()-startTime;
316 m_testCtx.getLog() << TestLog::Message << "Duration with " << curIterCount << " iterations = " << curDuration << " us" << TestLog::EndMessage;
318 if (curDuration > targetDurationUs)
320 if (curIterCount > 1)
322 // Compute final count by using linear estimation.
323 const float a = float(curDuration - prevDuration) / float(curIterCount - prevIterCount);
324 const float b = float(prevDuration) - a*float(prevIterCount);
325 const float est = (float(targetDurationUs) - b) / a;
327 curIterCount = de::clamp(deFloorFloatToInt32(est), 1, int(MAX_SHADER_ITER_COUNT));
329 // else: Settle on 1.
333 else if (curIterCount >= MAX_SHADER_ITER_COUNT)
334 break; // Settle on maximum.
337 prevIterCount = curIterCount;
338 prevDuration = curDuration;
339 curIterCount = curIterCount*2;
343 params.numItersInShader = curIterCount;
345 m_testCtx.getLog() << TestLog::Integer("ShaderIterCount", "Shader iteration count", "", QP_KEY_TAG_NONE, params.numItersInShader);
348 // Step 2: Find draw call count that results in desired maximum time.
350 deUint64 prevDuration = 0;
351 int prevDrawCount = 1;
352 int curDrawCount = 1;
354 m_testCtx.getLog() << TestLog::Message << "Calibrating maximum draw call count, target duration = " << int(MAX_SAMPLE_DURATION_US) << " us" << TestLog::EndMessage;
356 setShaderIterCount(params.numItersInShader);
360 deUint64 curDuration;
362 render(curDrawCount); // \note Submit time is ignored
365 const deUint64 startTime = deGetMicroseconds();
367 curDuration = deGetMicroseconds()-startTime;
370 m_testCtx.getLog() << TestLog::Message << "Duration with " << curDrawCount << " draw calls = " << curDuration << " us" << TestLog::EndMessage;
372 if (curDuration > MAX_SAMPLE_DURATION_US)
374 if (curDrawCount > 1)
376 // Compute final count by using linear estimation.
377 const float a = float(curDuration - prevDuration) / float(curDrawCount - prevDrawCount);
378 const float b = float(prevDuration) - a*float(prevDrawCount);
379 const float est = (float(MAX_SAMPLE_DURATION_US) - b) / a;
381 curDrawCount = de::clamp(deFloorFloatToInt32(est), 1, int(MAX_DRAW_CALL_COUNT));
383 // else: Settle on 1.
387 else if (curDrawCount >= MAX_DRAW_CALL_COUNT)
388 break; // Settle on maximum.
391 prevDrawCount = curDrawCount;
392 prevDuration = curDuration;
393 curDrawCount = curDrawCount*2;
397 params.maxDrawCalls = curDrawCount;
399 m_testCtx.getLog() << TestLog::Integer("MaxDrawCalls", "Maximum number of draw calls", "", QP_KEY_TAG_NONE, params.maxDrawCalls);
403 if (params.maxDrawCalls < MIN_DRAW_CALL_COUNT)
404 throw CalibrationFailedException("Calibration failed, maximum draw call count is too low");
409 void FlushFinishCase::verifyCalibration (const CalibrationParams& params)
411 setShaderIterCount(params.numItersInShader);
413 for (int sampleNdx = 0; sampleNdx < NUM_VERIFICATION_SAMPLES; sampleNdx++)
415 deUint64 readStartTime;
417 render(params.maxDrawCalls);
419 readStartTime = deGetMicroseconds();
423 const deUint64 renderDuration = deGetMicroseconds()-readStartTime;
424 const float relativeDelta = float(double(renderDuration) / double(MAX_SAMPLE_DURATION_US)) - 1.0f;
426 if (!de::inBounds(relativeDelta, -CALIBRATION_VERIFICATION_THRESHOLD, CALIBRATION_VERIFICATION_THRESHOLD))
428 std::ostringstream msg;
429 msg << "ERROR: Unstable performance, got " << renderDuration << " us read time, "
430 << de::floatToString(relativeDelta*100.0f, 1) << "% diff to estimated " << (int)MAX_SAMPLE_DURATION_US << " us";
431 throw CalibrationFailedException(msg.str());
437 struct CompareSampleDrawCount
439 bool operator() (const FlushFinishCase::Sample& a, const FlushFinishCase::Sample& b) const { return a.numDrawCalls < b.numDrawCalls; }
442 std::vector<Vec2> getPointsFromSamples (const std::vector<FlushFinishCase::Sample>& samples, const deUint64 FlushFinishCase::Sample::*field)
444 vector<Vec2> points(samples.size());
446 for (size_t ndx = 0; ndx < samples.size(); ndx++)
447 points[ndx] = Vec2(float(samples[ndx].numDrawCalls), float(samples[ndx].*field));
453 T getMaximumValue (const std::vector<FlushFinishCase::Sample>& samples, const T FlushFinishCase::Sample::*field)
455 DE_ASSERT(!samples.empty());
457 T maxVal = samples[0].*field;
459 for (size_t ndx = 1; ndx < samples.size(); ndx++)
460 maxVal = de::max(maxVal, samples[ndx].*field);
465 void FlushFinishCase::analyzeResults (const std::vector<Sample>& samples, const CalibrationParams& calibrationParams)
467 const vector<Vec2> waitTimes = getPointsFromSamples(samples, &Sample::waitTime);
468 const vector<Vec2> readTimes = getPointsFromSamples(samples, &Sample::readPixelsTime);
469 const LineParameters waitLine = theilSenLinearRegression(waitTimes);
470 const LineParameters readLine = theilSenLinearRegression(readTimes);
471 const float normWaitCoef = waitLine.coefficient * float(calibrationParams.maxDrawCalls) / float(MAX_SAMPLE_DURATION_US);
472 const float normReadCoef = readLine.coefficient * float(calibrationParams.maxDrawCalls) / float(MAX_SAMPLE_DURATION_US);
476 tcu::ScopedLogSection section (m_testCtx.getLog(), "Samples", "Samples");
477 vector<Sample> sortedSamples (samples.begin(), samples.end());
479 std::sort(sortedSamples.begin(), sortedSamples.end(), CompareSampleDrawCount());
481 for (vector<Sample>::const_iterator iter = sortedSamples.begin(); iter != sortedSamples.end(); ++iter)
482 m_testCtx.getLog() << *iter;
485 m_testCtx.getLog() << TestLog::Float("WaitCoefficient", "Wait coefficient", "", QP_KEY_TAG_NONE, waitLine.coefficient)
486 << TestLog::Float("ReadCoefficient", "Read coefficient", "", QP_KEY_TAG_NONE, readLine.coefficient)
487 << TestLog::Float("NormalizedWaitCoefficient", "Normalized wait coefficient", "", QP_KEY_TAG_NONE, normWaitCoef)
488 << TestLog::Float("NormalizedReadCoefficient", "Normalized read coefficient", "", QP_KEY_TAG_NONE, normReadCoef);
491 const bool waitCorrelated = normWaitCoef > CORRELATED_COEF_THRESHOLD;
492 const bool readCorrelated = normReadCoef > CORRELATED_COEF_THRESHOLD;
493 const bool waitNotCorr = normWaitCoef < NO_CORR_COEF_THRESHOLD;
494 const bool readNotCorr = normReadCoef < NO_CORR_COEF_THRESHOLD;
496 if (waitCorrelated || waitNotCorr)
497 m_testCtx.getLog() << TestLog::Message << "Wait time is" << (waitCorrelated ? "" : " NOT") << " correlated to rendering workload size." << TestLog::EndMessage;
499 m_testCtx.getLog() << TestLog::Message << "Warning: Wait time correlation to rendering workload size is unclear." << TestLog::EndMessage;
501 if (readCorrelated || readNotCorr)
502 m_testCtx.getLog() << TestLog::Message << "Read time is" << (readCorrelated ? "" : " NOT") << " correlated to rendering workload size." << TestLog::EndMessage;
504 m_testCtx.getLog() << TestLog::Message << "Warning: Read time correlation to rendering workload size is unclear." << TestLog::EndMessage;
507 for (int ndx = 0; ndx < 2; ndx++)
509 const float coef = ndx == 0 ? normWaitCoef : normReadCoef;
510 const char* name = ndx == 0 ? "wait" : "read";
511 const ExpectedBehavior behavior = ndx == 0 ? m_waitBehavior : m_readBehavior;
512 const float threshold = ndx == 0 ? m_waitThreshold : m_readThreshold;
513 const bool isOk = behavior == EXPECT_COEF_GREATER_THAN ? coef > threshold :
514 behavior == EXPECT_COEF_LESS_THAN ? coef < threshold : false;
515 const char* cmpName = behavior == EXPECT_COEF_GREATER_THAN ? "greater than" :
516 behavior == EXPECT_COEF_LESS_THAN ? "less than" : DE_NULL;
520 m_testCtx.getLog() << TestLog::Message << "ERROR: Expected " << name << " coefficient to be " << cmpName << " " << threshold << TestLog::EndMessage;
525 m_testCtx.setTestResult(allOk ? QP_TEST_RESULT_PASS : QP_TEST_RESULT_FAIL,
526 allOk ? "Pass" : "Suspicious performance behavior");
529 FlushFinishCase::IterateResult FlushFinishCase::iterate (void)
531 vector<Sample> samples (NUM_SAMPLES);
532 CalibrationParams params;
538 // Do one full render cycle.
540 setShaderIterCount(1);
546 for (int calibrationRoundNdx = 0; /* until done */; calibrationRoundNdx++)
550 m_testCtx.touchWatchdog();
551 params = calibrate();
552 verifyCalibration(params);
555 catch (const CalibrationFailedException& e)
557 m_testCtx.getLog() << e;
559 if (calibrationRoundNdx < MAX_CALIBRATION_ATTEMPTS)
561 m_testCtx.getLog() << TestLog::Message
562 << "Retrying calibration (" << (calibrationRoundNdx+1) << " / " << (int)MAX_CALIBRATION_ATTEMPTS << ")"
563 << TestLog::EndMessage;
567 m_testCtx.setTestResult(QP_TEST_RESULT_COMPATIBILITY_WARNING, e.what());
575 de::Random rnd (123);
577 setShaderIterCount(params.numItersInShader);
579 for (size_t ndx = 0; ndx < samples.size(); ndx++)
581 const int drawCallCount = rnd.getInt(1, params.maxDrawCalls);
582 const deUint64 submitStartTime = deGetMicroseconds();
583 deUint64 waitStartTime;
584 deUint64 readStartTime;
585 deUint64 readFinishTime;
587 render(drawCallCount);
589 waitStartTime = deGetMicroseconds();
592 readStartTime = deGetMicroseconds();
594 readFinishTime = deGetMicroseconds();
596 samples[ndx].numDrawCalls = drawCallCount;
597 samples[ndx].submitTime = waitStartTime-submitStartTime;
598 samples[ndx].waitTime = readStartTime-waitStartTime;
599 samples[ndx].readPixelsTime = readFinishTime-readStartTime;
601 m_testCtx.touchWatchdog();
605 // Analyze - sets test case result.
606 analyzeResults(samples, params);
611 class WaitOnlyCase : public FlushFinishCase
614 WaitOnlyCase (Context& context)
615 : FlushFinishCase(context, "wait", "Wait only", EXPECT_COEF_LESS_THAN, NO_CORR_COEF_THRESHOLD, EXPECT_COEF_GREATER_THAN, -1000.0f /* practically nothing is expected */)
621 m_testCtx.getLog() << TestLog::Message << int(WAIT_TIME_MS) << " ms busy wait" << TestLog::EndMessage;
622 FlushFinishCase::init();
626 void waitForGL (void)
628 busyWait(WAIT_TIME_MS);
632 class FlushOnlyCase : public FlushFinishCase
635 FlushOnlyCase (Context& context)
636 : FlushFinishCase(context, "flush", "Flush only", EXPECT_COEF_LESS_THAN, FLUSH_COEF_THRESHOLD, EXPECT_COEF_GREATER_THAN, CORRELATED_COEF_THRESHOLD)
642 m_testCtx.getLog() << TestLog::Message << "Single call to glFlush()" << TestLog::EndMessage;
643 FlushFinishCase::init();
647 void waitForGL (void)
649 m_context.getRenderContext().getFunctions().flush();
653 class FlushWaitCase : public FlushFinishCase
656 FlushWaitCase (Context& context)
657 : FlushFinishCase(context, "flush_wait", "Wait after flushing", EXPECT_COEF_LESS_THAN, FLUSH_COEF_THRESHOLD, EXPECT_COEF_LESS_THAN, NO_CORR_COEF_THRESHOLD)
663 m_testCtx.getLog() << TestLog::Message << "glFlush() followed by " << int(WAIT_TIME_MS) << " ms busy wait" << TestLog::EndMessage;
664 FlushFinishCase::init();
668 void waitForGL (void)
670 m_context.getRenderContext().getFunctions().flush();
671 busyWait(WAIT_TIME_MS);
675 class FinishOnlyCase : public FlushFinishCase
678 FinishOnlyCase (Context& context)
679 : FlushFinishCase(context, "finish", "Finish only", EXPECT_COEF_GREATER_THAN, CORRELATED_COEF_THRESHOLD, EXPECT_COEF_LESS_THAN, NO_CORR_COEF_THRESHOLD)
685 m_testCtx.getLog() << TestLog::Message << "Single call to glFinish()" << TestLog::EndMessage;
686 FlushFinishCase::init();
690 void waitForGL (void)
692 m_context.getRenderContext().getFunctions().finish();
696 class FinishWaitCase : public FlushFinishCase
699 FinishWaitCase (Context& context)
700 : FlushFinishCase(context, "finish_wait", "Finish and wait", EXPECT_COEF_GREATER_THAN, CORRELATED_COEF_THRESHOLD, EXPECT_COEF_LESS_THAN, NO_CORR_COEF_THRESHOLD)
706 m_testCtx.getLog() << TestLog::Message << "glFinish() followed by " << int(WAIT_TIME_MS) << " ms busy wait" << TestLog::EndMessage;
707 FlushFinishCase::init();
711 void waitForGL (void)
713 m_context.getRenderContext().getFunctions().finish();
714 busyWait(WAIT_TIME_MS);
720 FlushFinishTests::FlushFinishTests (Context& context)
721 : TestCaseGroup(context, "flush_finish", "Flush and Finish tests")
725 FlushFinishTests::~FlushFinishTests (void)
729 void FlushFinishTests::init (void)
731 addChild(new WaitOnlyCase (m_context));
732 addChild(new FlushOnlyCase (m_context));
733 addChild(new FlushWaitCase (m_context));
734 addChild(new FinishOnlyCase (m_context));
735 addChild(new FinishWaitCase (m_context));