Update rive-cpp to 2.0 version
[platform/core/uifw/rive-tizen.git] / submodule / skia / infra / bots / task_drivers / perf_puppeteer_skottie_frames / perf_puppeteer_skottie_frames_test.go
1 // Copyright 2020 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 package main
6
7 import (
8         "context"
9         "io/ioutil"
10         "os"
11         "path/filepath"
12         "testing"
13
14         "github.com/stretchr/testify/assert"
15         "github.com/stretchr/testify/require"
16         "go.skia.org/infra/go/exec"
17         "go.skia.org/infra/go/testutils"
18         "go.skia.org/infra/task_driver/go/td"
19 )
20
21 func TestSetup_NPMInitializedBenchmarkOutCreated(t *testing.T) {
22         benchmarkPath, err := ioutil.TempDir("", "benchmark")
23         require.NoError(t, err)
24         defer testutils.RemoveAll(t, benchmarkPath)
25
26         const fakeNodeBinPath = "/fake/path/to/node/bin"
27
28         res := td.RunTestSteps(t, false, func(ctx context.Context) error {
29                 mock := exec.CommandCollector{}
30                 ctx = td.WithExecRunFn(ctx, mock.Run)
31                 err := setup(ctx, benchmarkPath, fakeNodeBinPath)
32                 if err != nil {
33                         assert.NoError(t, err)
34                         return err
35                 }
36                 require.Len(t, mock.Commands(), 1)
37                 cmd := mock.Commands()[0]
38                 assert.Equal(t, "/fake/path/to/node/bin/npm", cmd.Name)
39                 assert.Equal(t, []string{"ci"}, cmd.Args)
40                 return nil
41         })
42         require.Empty(t, res.Errors)
43         require.Empty(t, res.Exceptions)
44
45         fi, err := os.Stat(filepath.Join(benchmarkPath, "out"))
46         require.NoError(t, err)
47         assert.True(t, fi.IsDir())
48 }
49
50 func TestBenchSkottieFrames_CPUHasNoUseGPUFlag(t *testing.T) {
51         lotties, err := ioutil.TempDir("", "lotties")
52         require.NoError(t, err)
53         defer testutils.RemoveAll(t, lotties)
54
55         require.NoError(t, os.MkdirAll(filepath.Join(lotties, "animation_1"), 0777))
56
57         const fakeNodeBinPath = "/fake/path/to/node/bin"
58         const fakeCanvasKitPath = "/fake/path/to/canvaskit"
59         const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer"
60
61         perfObj := perfJSONFormat{
62                 Key: map[string]string{
63                         perfKeyCpuOrGPU: "CPU",
64                 },
65         }
66
67         res := td.RunTestSteps(t, false, func(ctx context.Context) error {
68                 mock := exec.CommandCollector{}
69                 ctx = td.WithExecRunFn(ctx, mock.Run)
70                 err := benchSkottieFrames(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, lotties, fakeNodeBinPath)
71                 if err != nil {
72                         assert.NoError(t, err)
73                         return err
74                 }
75                 require.Len(t, mock.Commands(), 1)
76                 cmd := mock.Commands()[0]
77                 assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name)
78                 assert.Equal(t, []string{"perf-canvaskit-with-puppeteer",
79                         "--bench_html", "skottie-frames.html",
80                         "--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js",
81                         "--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm",
82                         "--input_lottie", filepath.Join(lotties, "animation_1", "data.json"),
83                         "--assets", filepath.Join(lotties, "animation_1", "images"),
84                         "--output", "/fake/path/to/perf-puppeteer/out/animation_1.json"}, cmd.Args)
85                 return nil
86         })
87         require.Empty(t, res.Errors)
88         require.Empty(t, res.Exceptions)
89 }
90
91 func TestBenchSkottieFrames_GPUHasFlag(t *testing.T) {
92         lotties, err := ioutil.TempDir("", "lotties")
93         require.NoError(t, err)
94         defer testutils.RemoveAll(t, lotties)
95
96         require.NoError(t, os.MkdirAll(filepath.Join(lotties, "animation_1"), 0777))
97
98         const fakeNodeBinPath = "/fake/path/to/node/bin"
99         const fakeCanvasKitPath = "/fake/path/to/canvaskit"
100         const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer"
101
102         perfObj := perfJSONFormat{
103                 Key: map[string]string{
104                         perfKeyCpuOrGPU: "GPU",
105                 },
106         }
107
108         res := td.RunTestSteps(t, false, func(ctx context.Context) error {
109                 mock := exec.CommandCollector{}
110                 ctx = td.WithExecRunFn(ctx, mock.Run)
111                 err := benchSkottieFrames(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, lotties, fakeNodeBinPath)
112                 if err != nil {
113                         assert.NoError(t, err)
114                         return err
115                 }
116                 require.Len(t, mock.Commands(), 1)
117                 cmd := mock.Commands()[0]
118                 assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name)
119                 assert.Equal(t, []string{"perf-canvaskit-with-puppeteer",
120                         "--bench_html", "skottie-frames.html",
121                         "--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js",
122                         "--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm",
123                         "--input_lottie", filepath.Join(lotties, "animation_1", "data.json"),
124                         "--assets", filepath.Join(lotties, "animation_1", "images"),
125                         "--output", "/fake/path/to/perf-puppeteer/out/animation_1.json",
126                         "--use_gpu"}, cmd.Args)
127                 return nil
128         })
129         require.Empty(t, res.Errors)
130         require.Empty(t, res.Exceptions)
131 }
132
133 // TestProcessSkottieFramesData_CPUTwoInputsGetSummarizedAndCombined tests the scenario where we
134 // have multiple inputs to process. The input directory should get scanned for all json files;
135 // these JSON files should be read in and converted to perf results, using the name of the file
136 // as the name (w/o the .json suffix).
137 func TestProcessSkottieFramesData_CPUTwoInputsGetSummarizedAndCombined(t *testing.T) {
138         input, err := ioutil.TempDir("", "inputs")
139         require.NoError(t, err)
140         defer testutils.RemoveAll(t, input)
141         err = writeFilesToDisk(filepath.Join(input, "out"), map[string]string{
142                 "first_animation.json":  skottieFramesSampleOne,
143                 "second_animation.json": skottieFramesSampleTwo,
144         })
145         require.NoError(t, err)
146         output, err := ioutil.TempDir("", "perf")
147         require.NoError(t, err)
148         defer testutils.RemoveAll(t, output)
149
150         keys := map[string]string{
151                 "os":               "Debian10",
152                 "model":            "GCE",
153                 perfKeyCpuOrGPU:    "CPU",
154                 "cpu_or_gpu_value": "AVX2",
155         }
156
157         perfObj, err := makePerfObj(someGitHash, someTaskID, someMachineID, keys)
158         require.NoError(t, err)
159
160         outputFile := filepath.Join(output, "perf-taskid1352.json")
161         res := td.RunTestSteps(t, false, func(ctx context.Context) error {
162                 return processSkottieFramesData(ctx, perfObj, input, outputFile)
163         })
164         require.Empty(t, res.Errors)
165         require.Empty(t, res.Exceptions)
166
167         b, err := ioutil.ReadFile(outputFile)
168         require.NoError(t, err)
169
170         assert.Equal(t, `{
171   "gitHash": "032631e490db494128e0610a19adce4cab9706d1",
172   "swarming_task_id": "4bdd43ed7c906c11",
173   "swarming_machine_id": "skia-e-gce-203",
174   "key": {
175     "arch": "wasm",
176     "binary": "CanvasKit",
177     "browser": "Chromium",
178     "configuration": "Release",
179     "cpu_or_gpu": "CPU",
180     "cpu_or_gpu_value": "AVX2",
181     "extra_config": "SkottieFrames",
182     "model": "GCE",
183     "os": "Debian10"
184   },
185   "results": {
186     "first_animation": {
187       "software": {
188         "1st_frame_ms": 31.555,
189         "2nd_frame_ms": 87.795,
190         "3rd_frame_ms": 0.43,
191         "4th_frame_ms": 1.845,
192         "5th_frame_ms": 3.61,
193         "90th_percentile_frame_ms": 4.455,
194         "95th_percentile_frame_ms": 31.555,
195         "99th_percentile_frame_ms": 87.795,
196         "avg_first_five_frames_ms": 25.047,
197         "avg_render_frame_ms": 5.662692,
198         "avg_render_with_flush_ms": 1.75,
199         "avg_render_without_flush_ms": 1.875,
200         "json_load_ms": 16.05,
201         "median_render_frame_ms": 0.795,
202         "median_render_with_flush_ms": 1.8,
203         "median_render_without_flush_ms": 1.88,
204         "stddev_render_frame_ms": 17.463467,
205         "stddev_render_with_flush_ms": 0.74999994,
206         "stddev_render_without_flush_ms": 0.07500001
207       }
208     },
209     "second_animation": {
210       "software": {
211         "1st_frame_ms": 210.555,
212         "2nd_frame_ms": 770.795,
213         "3rd_frame_ms": 10.43,
214         "4th_frame_ms": 31.845,
215         "5th_frame_ms": 3.61,
216         "90th_percentile_frame_ms": 210.555,
217         "95th_percentile_frame_ms": 400.455,
218         "99th_percentile_frame_ms": 770.795,
219         "avg_first_five_frames_ms": 205.44699,
220         "avg_render_frame_ms": 55.58577,
221         "avg_render_with_flush_ms": 3.75,
222         "avg_render_without_flush_ms": 5.125,
223         "json_load_ms": 28.15,
224         "median_render_frame_ms": 0.8,
225         "median_render_with_flush_ms": 3.8,
226         "median_render_without_flush_ms": 5.13,
227         "stddev_render_frame_ms": 166.36926,
228         "stddev_render_with_flush_ms": 0.75,
229         "stddev_render_without_flush_ms": 0.074999936
230       }
231     }
232   }
233 }`, string(b))
234 }
235
236 func TestProcessSkottieFramesData_GPUTwoInputsGetSummarizedAndCombined(t *testing.T) {
237         input, err := ioutil.TempDir("", "inputs")
238         require.NoError(t, err)
239         defer testutils.RemoveAll(t, input)
240         err = writeFilesToDisk(filepath.Join(input, "out"), map[string]string{
241                 "first_animation.json":  skottieFramesSampleOne,
242                 "second_animation.json": skottieFramesSampleTwo,
243         })
244         require.NoError(t, err)
245         output, err := ioutil.TempDir("", "perf")
246         require.NoError(t, err)
247         defer testutils.RemoveAll(t, output)
248
249         // These are based off of realistic values.
250         keys := map[string]string{
251                 "os":               "Ubuntu18",
252                 "model":            "Golo",
253                 perfKeyCpuOrGPU:    "GPU",
254                 "cpu_or_gpu_value": "QuadroP400",
255         }
256
257         perfObj, err := makePerfObj(someGitHash, someTaskID, someMachineID, keys)
258         require.NoError(t, err)
259
260         outputFile := filepath.Join(output, "perf-taskid1352.json")
261         res := td.RunTestSteps(t, false, func(ctx context.Context) error {
262                 return processSkottieFramesData(ctx, perfObj, input, outputFile)
263         })
264         require.Empty(t, res.Errors)
265
266         b, err := ioutil.ReadFile(outputFile)
267         require.NoError(t, err)
268
269         assert.Equal(t, `{
270   "gitHash": "032631e490db494128e0610a19adce4cab9706d1",
271   "swarming_task_id": "4bdd43ed7c906c11",
272   "swarming_machine_id": "skia-e-gce-203",
273   "key": {
274     "arch": "wasm",
275     "binary": "CanvasKit",
276     "browser": "Chromium",
277     "configuration": "Release",
278     "cpu_or_gpu": "GPU",
279     "cpu_or_gpu_value": "QuadroP400",
280     "extra_config": "SkottieFrames",
281     "model": "Golo",
282     "os": "Ubuntu18"
283   },
284   "results": {
285     "first_animation": {
286       "webgl2": {
287         "1st_frame_ms": 31.555,
288         "2nd_frame_ms": 87.795,
289         "3rd_frame_ms": 0.43,
290         "4th_frame_ms": 1.845,
291         "5th_frame_ms": 3.61,
292         "90th_percentile_frame_ms": 4.455,
293         "95th_percentile_frame_ms": 31.555,
294         "99th_percentile_frame_ms": 87.795,
295         "avg_first_five_frames_ms": 25.047,
296         "avg_render_frame_ms": 5.662692,
297         "avg_render_with_flush_ms": 1.75,
298         "avg_render_without_flush_ms": 1.875,
299         "json_load_ms": 16.05,
300         "median_render_frame_ms": 0.795,
301         "median_render_with_flush_ms": 1.8,
302         "median_render_without_flush_ms": 1.88,
303         "stddev_render_frame_ms": 17.463467,
304         "stddev_render_with_flush_ms": 0.74999994,
305         "stddev_render_without_flush_ms": 0.07500001
306       }
307     },
308     "second_animation": {
309       "webgl2": {
310         "1st_frame_ms": 210.555,
311         "2nd_frame_ms": 770.795,
312         "3rd_frame_ms": 10.43,
313         "4th_frame_ms": 31.845,
314         "5th_frame_ms": 3.61,
315         "90th_percentile_frame_ms": 210.555,
316         "95th_percentile_frame_ms": 400.455,
317         "99th_percentile_frame_ms": 770.795,
318         "avg_first_five_frames_ms": 205.44699,
319         "avg_render_frame_ms": 55.58577,
320         "avg_render_with_flush_ms": 3.75,
321         "avg_render_without_flush_ms": 5.125,
322         "json_load_ms": 28.15,
323         "median_render_frame_ms": 0.8,
324         "median_render_with_flush_ms": 3.8,
325         "median_render_without_flush_ms": 5.13,
326         "stddev_render_frame_ms": 166.36926,
327         "stddev_render_with_flush_ms": 0.75,
328         "stddev_render_without_flush_ms": 0.074999936
329       }
330     }
331   }
332 }`, string(b))
333 }
334
335 func writeFilesToDisk(path string, fileNamesToContent map[string]string) error {
336         if err := os.MkdirAll(path, 0777); err != nil {
337                 return err
338         }
339         for name, content := range fileNamesToContent {
340                 if err := ioutil.WriteFile(filepath.Join(path, name), []byte(content), 0666); err != nil {
341                         return err
342                 }
343         }
344         return nil
345 }
346
347 const (
348         someGitHash   = "032631e490db494128e0610a19adce4cab9706d1"
349         someTaskID    = "4bdd43ed7c906c11"
350         someMachineID = "skia-e-gce-203"
351 )
352
353 const skottieFramesSampleOne = `
354 {
355   "total_frame_ms": [
356     31.555,
357     87.795,
358     0.430,
359     1.845,
360     3.610,
361     1.105,
362     0.545,
363     2.315,
364     1.685,
365     0.615,
366     0.425,
367     0.815,
368     0.355,
369     0.655,
370     0.390,
371     4.455,
372     0.800,
373     0.685,
374     2.630,
375     0.325,
376     0.355,
377     0.740,
378     0.785,
379     0.795,
380     0.72,
381     0.80
382   ],
383   "without_flush_ms": [
384     2.0,
385     1.99,
386     1.98,
387     1.97,
388     1.96,
389     1.95,
390     1.94,
391     1.93,
392     1.92,
393     1.91,
394     1.9,
395     1.89,
396     1.88,
397     1.87,
398     1.86,
399     1.85,
400     1.84,
401     1.83,
402     1.82,
403     1.81,
404     1.8,
405     1.79,
406     1.78,
407     1.77,
408     1.76,
409     1.75
410   ],
411   "with_flush_ms": [
412     3.0,
413     2.9,
414     2.8,
415     2.7,
416     2.6,
417     2.5,
418     2.4,
419     2.3,
420     2.2,
421     2.1,
422     2.0,
423     1.9,
424     1.8,
425     1.7,
426     1.6,
427     1.5,
428     1.4,
429     1.3,
430     1.2,
431     1.1,
432     1.0,
433     0.9,
434     0.8,
435     0.7,
436     0.6,
437     0.5
438   ],
439   "json_load_ms": 16.05
440 }`
441
442 const skottieFramesSampleTwo = `
443 {
444   "total_frame_ms": [
445     210.555,
446     770.795,
447     10.430,
448     31.845,
449     3.610,
450     1.105,
451     0.545,
452     2.315,
453     1.685,
454     0.615,
455     0.425,
456     0.815,
457     0.355,
458     0.655,
459     0.390,
460     400.455,
461     0.800,
462     0.685,
463     2.630,
464     0.325,
465     0.355,
466     0.740,
467     0.785,
468     0.795,
469     0.72,
470     0.80
471   ],
472   "without_flush_ms": [
473     5.0,
474     5.01,
475     5.02,
476     5.03,
477     5.04,
478     5.05,
479     5.06,
480     5.07,
481     5.08,
482     5.09,
483     5.1,
484     5.11,
485     5.12,
486     5.13,
487     5.14,
488     5.15,
489     5.16,
490     5.17,
491     5.18,
492     5.19,
493     5.2,
494     5.21,
495     5.22,
496     5.23,
497     5.24,
498     5.25
499   ],
500   "with_flush_ms": [
501     5.0,
502     4.9,
503     4.8,
504     4.7,
505     4.6,
506     4.5,
507     4.4,
508     4.3,
509     4.2,
510     4.1,
511     4.0,
512     3.9,
513     3.8,
514     3.7,
515     3.6,
516     3.5,
517     3.4,
518     3.3,
519     3.2,
520     3.1,
521     3.0,
522     2.9,
523     2.8,
524     2.7,
525     2.6,
526     2.5
527   ],
528   "json_load_ms": 28.15
529 }`