submodule/skia/infra/bots/task_drivers/perf_puppeteer_skottie_frames/perf_puppeteer_skottie_frames_test.go

   1 // Copyright 2020 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 package main
   6
   7 import (
   8         "context"
   9         "io/ioutil"
  10         "os"
  11         "path/filepath"
  12         "testing"
  13
  14         "github.com/stretchr/testify/assert"
  15         "github.com/stretchr/testify/require"
  16         "go.skia.org/infra/go/exec"
  17         "go.skia.org/infra/go/testutils"
  18         "go.skia.org/infra/task_driver/go/td"
  19 )
  20
  21 func TestSetup_NPMInitializedBenchmarkOutCreated(t *testing.T) {
  22         benchmarkPath, err := ioutil.TempDir("", "benchmark")
  23         require.NoError(t, err)
  24         defer testutils.RemoveAll(t, benchmarkPath)
  25
  26         const fakeNodeBinPath = "/fake/path/to/node/bin"
  27
  28         res := td.RunTestSteps(t, false, func(ctx context.Context) error {
  29                 mock := exec.CommandCollector{}
  30                 ctx = td.WithExecRunFn(ctx, mock.Run)
  31                 err := setup(ctx, benchmarkPath, fakeNodeBinPath)
  32                 if err != nil {
  33                         assert.NoError(t, err)
  34                         return err
  35                 }
  36                 require.Len(t, mock.Commands(), 1)
  37                 cmd := mock.Commands()[0]
  38                 assert.Equal(t, "/fake/path/to/node/bin/npm", cmd.Name)
  39                 assert.Equal(t, []string{"ci"}, cmd.Args)
  40                 return nil
  41         })
  42         require.Empty(t, res.Errors)
  43         require.Empty(t, res.Exceptions)
  44
  45         fi, err := os.Stat(filepath.Join(benchmarkPath, "out"))
  46         require.NoError(t, err)
  47         assert.True(t, fi.IsDir())
  48 }
  49
  50 func TestBenchSkottieFrames_CPUHasNoUseGPUFlag(t *testing.T) {
  51         lotties, err := ioutil.TempDir("", "lotties")
  52         require.NoError(t, err)
  53         defer testutils.RemoveAll(t, lotties)
  54
  55         require.NoError(t, os.MkdirAll(filepath.Join(lotties, "animation_1"), 0777))
  56
  57         const fakeNodeBinPath = "/fake/path/to/node/bin"
  58         const fakeCanvasKitPath = "/fake/path/to/canvaskit"
  59         const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer"
  60
  61         perfObj := perfJSONFormat{
  62                 Key: map[string]string{
  63                         perfKeyCpuOrGPU: "CPU",
  64                 },
  65         }
  66
  67         res := td.RunTestSteps(t, false, func(ctx context.Context) error {
  68                 mock := exec.CommandCollector{}
  69                 ctx = td.WithExecRunFn(ctx, mock.Run)
  70                 err := benchSkottieFrames(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, lotties, fakeNodeBinPath)
  71                 if err != nil {
  72                         assert.NoError(t, err)
  73                         return err
  74                 }
  75                 require.Len(t, mock.Commands(), 1)
  76                 cmd := mock.Commands()[0]
  77                 assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name)
  78                 assert.Equal(t, []string{"perf-canvaskit-with-puppeteer",
  79                         "--bench_html", "skottie-frames.html",
  80                         "--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js",
  81                         "--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm",
  82                         "--input_lottie", filepath.Join(lotties, "animation_1", "data.json"),
  83                         "--assets", filepath.Join(lotties, "animation_1", "images"),
  84                         "--output", "/fake/path/to/perf-puppeteer/out/animation_1.json"}, cmd.Args)
  85                 return nil
  86         })
  87         require.Empty(t, res.Errors)
  88         require.Empty(t, res.Exceptions)
  89 }
  90
  91 func TestBenchSkottieFrames_GPUHasFlag(t *testing.T) {
  92         lotties, err := ioutil.TempDir("", "lotties")
  93         require.NoError(t, err)
  94         defer testutils.RemoveAll(t, lotties)
  95
  96         require.NoError(t, os.MkdirAll(filepath.Join(lotties, "animation_1"), 0777))
  97
  98         const fakeNodeBinPath = "/fake/path/to/node/bin"
  99         const fakeCanvasKitPath = "/fake/path/to/canvaskit"
 100         const fakeBenchmarkPath = "/fake/path/to/perf-puppeteer"
 101
 102         perfObj := perfJSONFormat{
 103                 Key: map[string]string{
 104                         perfKeyCpuOrGPU: "GPU",
 105                 },
 106         }
 107
 108         res := td.RunTestSteps(t, false, func(ctx context.Context) error {
 109                 mock := exec.CommandCollector{}
 110                 ctx = td.WithExecRunFn(ctx, mock.Run)
 111                 err := benchSkottieFrames(ctx, perfObj, fakeBenchmarkPath, fakeCanvasKitPath, lotties, fakeNodeBinPath)
 112                 if err != nil {
 113                         assert.NoError(t, err)
 114                         return err
 115                 }
 116                 require.Len(t, mock.Commands(), 1)
 117                 cmd := mock.Commands()[0]
 118                 assert.Equal(t, "/fake/path/to/node/bin/node", cmd.Name)
 119                 assert.Equal(t, []string{"perf-canvaskit-with-puppeteer",
 120                         "--bench_html", "skottie-frames.html",
 121                         "--canvaskit_js", "/fake/path/to/canvaskit/canvaskit.js",
 122                         "--canvaskit_wasm", "/fake/path/to/canvaskit/canvaskit.wasm",
 123                         "--input_lottie", filepath.Join(lotties, "animation_1", "data.json"),
 124                         "--assets", filepath.Join(lotties, "animation_1", "images"),
 125                         "--output", "/fake/path/to/perf-puppeteer/out/animation_1.json",
 126                         "--use_gpu"}, cmd.Args)
 127                 return nil
 128         })
 129         require.Empty(t, res.Errors)
 130         require.Empty(t, res.Exceptions)
 131 }
 132
 133 // TestProcessSkottieFramesData_CPUTwoInputsGetSummarizedAndCombined tests the scenario where we
 134 // have multiple inputs to process. The input directory should get scanned for all json files;
 135 // these JSON files should be read in and converted to perf results, using the name of the file
 136 // as the name (w/o the .json suffix).
 137 func TestProcessSkottieFramesData_CPUTwoInputsGetSummarizedAndCombined(t *testing.T) {
 138         input, err := ioutil.TempDir("", "inputs")
 139         require.NoError(t, err)
 140         defer testutils.RemoveAll(t, input)
 141         err = writeFilesToDisk(filepath.Join(input, "out"), map[string]string{
 142                 "first_animation.json":  skottieFramesSampleOne,
 143                 "second_animation.json": skottieFramesSampleTwo,
 144         })
 145         require.NoError(t, err)
 146         output, err := ioutil.TempDir("", "perf")
 147         require.NoError(t, err)
 148         defer testutils.RemoveAll(t, output)
 149
 150         keys := map[string]string{
 151                 "os":               "Debian10",
 152                 "model":            "GCE",
 153                 perfKeyCpuOrGPU:    "CPU",
 154                 "cpu_or_gpu_value": "AVX2",
 155         }
 156
 157         perfObj, err := makePerfObj(someGitHash, someTaskID, someMachineID, keys)
 158         require.NoError(t, err)
 159
 160         outputFile := filepath.Join(output, "perf-taskid1352.json")
 161         res := td.RunTestSteps(t, false, func(ctx context.Context) error {
 162                 return processSkottieFramesData(ctx, perfObj, input, outputFile)
 163         })
 164         require.Empty(t, res.Errors)
 165         require.Empty(t, res.Exceptions)
 166
 167         b, err := ioutil.ReadFile(outputFile)
 168         require.NoError(t, err)
 169
 170         assert.Equal(t, `{
 171   "gitHash": "032631e490db494128e0610a19adce4cab9706d1",
 172   "swarming_task_id": "4bdd43ed7c906c11",
 173   "swarming_machine_id": "skia-e-gce-203",
 174   "key": {
 175     "arch": "wasm",
 176     "binary": "CanvasKit",
 177     "browser": "Chromium",
 178     "configuration": "Release",
 179     "cpu_or_gpu": "CPU",
 180     "cpu_or_gpu_value": "AVX2",
 181     "extra_config": "SkottieFrames",
 182     "model": "GCE",
 183     "os": "Debian10"
 184   },
 185   "results": {
 186     "first_animation": {
 187       "software": {
 188         "1st_frame_ms": 31.555,
 189         "2nd_frame_ms": 87.795,
 190         "3rd_frame_ms": 0.43,
 191         "4th_frame_ms": 1.845,
 192         "5th_frame_ms": 3.61,
 193         "90th_percentile_frame_ms": 4.455,
 194         "95th_percentile_frame_ms": 31.555,
 195         "99th_percentile_frame_ms": 87.795,
 196         "avg_first_five_frames_ms": 25.047,
 197         "avg_render_frame_ms": 5.662692,
 198         "avg_render_with_flush_ms": 1.75,
 199         "avg_render_without_flush_ms": 1.875,
 200         "json_load_ms": 16.05,
 201         "median_render_frame_ms": 0.795,
 202         "median_render_with_flush_ms": 1.8,
 203         "median_render_without_flush_ms": 1.88,
 204         "stddev_render_frame_ms": 17.463467,
 205         "stddev_render_with_flush_ms": 0.74999994,
 206         "stddev_render_without_flush_ms": 0.07500001
 207       }
 208     },
 209     "second_animation": {
 210       "software": {
 211         "1st_frame_ms": 210.555,
 212         "2nd_frame_ms": 770.795,
 213         "3rd_frame_ms": 10.43,
 214         "4th_frame_ms": 31.845,
 215         "5th_frame_ms": 3.61,
 216         "90th_percentile_frame_ms": 210.555,
 217         "95th_percentile_frame_ms": 400.455,
 218         "99th_percentile_frame_ms": 770.795,
 219         "avg_first_five_frames_ms": 205.44699,
 220         "avg_render_frame_ms": 55.58577,
 221         "avg_render_with_flush_ms": 3.75,
 222         "avg_render_without_flush_ms": 5.125,
 223         "json_load_ms": 28.15,
 224         "median_render_frame_ms": 0.8,
 225         "median_render_with_flush_ms": 3.8,
 226         "median_render_without_flush_ms": 5.13,
 227         "stddev_render_frame_ms": 166.36926,
 228         "stddev_render_with_flush_ms": 0.75,
 229         "stddev_render_without_flush_ms": 0.074999936
 230       }
 231     }
 232   }
 233 }`, string(b))
 234 }
 235
 236 func TestProcessSkottieFramesData_GPUTwoInputsGetSummarizedAndCombined(t *testing.T) {
 237         input, err := ioutil.TempDir("", "inputs")
 238         require.NoError(t, err)
 239         defer testutils.RemoveAll(t, input)
 240         err = writeFilesToDisk(filepath.Join(input, "out"), map[string]string{
 241                 "first_animation.json":  skottieFramesSampleOne,
 242                 "second_animation.json": skottieFramesSampleTwo,
 243         })
 244         require.NoError(t, err)
 245         output, err := ioutil.TempDir("", "perf")
 246         require.NoError(t, err)
 247         defer testutils.RemoveAll(t, output)
 248
 249         // These are based off of realistic values.
 250         keys := map[string]string{
 251                 "os":               "Ubuntu18",
 252                 "model":            "Golo",
 253                 perfKeyCpuOrGPU:    "GPU",
 254                 "cpu_or_gpu_value": "QuadroP400",
 255         }
 256
 257         perfObj, err := makePerfObj(someGitHash, someTaskID, someMachineID, keys)
 258         require.NoError(t, err)
 259
 260         outputFile := filepath.Join(output, "perf-taskid1352.json")
 261         res := td.RunTestSteps(t, false, func(ctx context.Context) error {
 262                 return processSkottieFramesData(ctx, perfObj, input, outputFile)
 263         })
 264         require.Empty(t, res.Errors)
 265
 266         b, err := ioutil.ReadFile(outputFile)
 267         require.NoError(t, err)
 268
 269         assert.Equal(t, `{
 270   "gitHash": "032631e490db494128e0610a19adce4cab9706d1",
 271   "swarming_task_id": "4bdd43ed7c906c11",
 272   "swarming_machine_id": "skia-e-gce-203",
 273   "key": {
 274     "arch": "wasm",
 275     "binary": "CanvasKit",
 276     "browser": "Chromium",
 277     "configuration": "Release",
 278     "cpu_or_gpu": "GPU",
 279     "cpu_or_gpu_value": "QuadroP400",
 280     "extra_config": "SkottieFrames",
 281     "model": "Golo",
 282     "os": "Ubuntu18"
 283   },
 284   "results": {
 285     "first_animation": {
 286       "webgl2": {
 287         "1st_frame_ms": 31.555,
 288         "2nd_frame_ms": 87.795,
 289         "3rd_frame_ms": 0.43,
 290         "4th_frame_ms": 1.845,
 291         "5th_frame_ms": 3.61,
 292         "90th_percentile_frame_ms": 4.455,
 293         "95th_percentile_frame_ms": 31.555,
 294         "99th_percentile_frame_ms": 87.795,
 295         "avg_first_five_frames_ms": 25.047,
 296         "avg_render_frame_ms": 5.662692,
 297         "avg_render_with_flush_ms": 1.75,
 298         "avg_render_without_flush_ms": 1.875,
 299         "json_load_ms": 16.05,
 300         "median_render_frame_ms": 0.795,
 301         "median_render_with_flush_ms": 1.8,
 302         "median_render_without_flush_ms": 1.88,
 303         "stddev_render_frame_ms": 17.463467,
 304         "stddev_render_with_flush_ms": 0.74999994,
 305         "stddev_render_without_flush_ms": 0.07500001
 306       }
 307     },
 308     "second_animation": {
 309       "webgl2": {
 310         "1st_frame_ms": 210.555,
 311         "2nd_frame_ms": 770.795,
 312         "3rd_frame_ms": 10.43,
 313         "4th_frame_ms": 31.845,
 314         "5th_frame_ms": 3.61,
 315         "90th_percentile_frame_ms": 210.555,
 316         "95th_percentile_frame_ms": 400.455,
 317         "99th_percentile_frame_ms": 770.795,
 318         "avg_first_five_frames_ms": 205.44699,
 319         "avg_render_frame_ms": 55.58577,
 320         "avg_render_with_flush_ms": 3.75,
 321         "avg_render_without_flush_ms": 5.125,
 322         "json_load_ms": 28.15,
 323         "median_render_frame_ms": 0.8,
 324         "median_render_with_flush_ms": 3.8,
 325         "median_render_without_flush_ms": 5.13,
 326         "stddev_render_frame_ms": 166.36926,
 327         "stddev_render_with_flush_ms": 0.75,
 328         "stddev_render_without_flush_ms": 0.074999936
 329       }
 330     }
 331   }
 332 }`, string(b))
 333 }
 334
 335 func writeFilesToDisk(path string, fileNamesToContent map[string]string) error {
 336         if err := os.MkdirAll(path, 0777); err != nil {
 337                 return err
 338         }
 339         for name, content := range fileNamesToContent {
 340                 if err := ioutil.WriteFile(filepath.Join(path, name), []byte(content), 0666); err != nil {
 341                         return err
 342                 }
 343         }
 344         return nil
 345 }
 346
 347 const (
 348         someGitHash   = "032631e490db494128e0610a19adce4cab9706d1"
 349         someTaskID    = "4bdd43ed7c906c11"
 350         someMachineID = "skia-e-gce-203"
 351 )
 352
 353 const skottieFramesSampleOne = `
 354 {
 355   "total_frame_ms": [
 356     31.555,
 357     87.795,
 358     0.430,
 359     1.845,
 360     3.610,
 361     1.105,
 362     0.545,
 363     2.315,
 364     1.685,
 365     0.615,
 366     0.425,
 367     0.815,
 368     0.355,
 369     0.655,
 370     0.390,
 371     4.455,
 372     0.800,
 373     0.685,
 374     2.630,
 375     0.325,
 376     0.355,
 377     0.740,
 378     0.785,
 379     0.795,
 380     0.72,
 381     0.80
 382   ],
 383   "without_flush_ms": [
 384     2.0,
 385     1.99,
 386     1.98,
 387     1.97,
 388     1.96,
 389     1.95,
 390     1.94,
 391     1.93,
 392     1.92,
 393     1.91,
 394     1.9,
 395     1.89,
 396     1.88,
 397     1.87,
 398     1.86,
 399     1.85,
 400     1.84,
 401     1.83,
 402     1.82,
 403     1.81,
 404     1.8,
 405     1.79,
 406     1.78,
 407     1.77,
 408     1.76,
 409     1.75
 410   ],
 411   "with_flush_ms": [
 412     3.0,
 413     2.9,
 414     2.8,
 415     2.7,
 416     2.6,
 417     2.5,
 418     2.4,
 419     2.3,
 420     2.2,
 421     2.1,
 422     2.0,
 423     1.9,
 424     1.8,
 425     1.7,
 426     1.6,
 427     1.5,
 428     1.4,
 429     1.3,
 430     1.2,
 431     1.1,
 432     1.0,
 433     0.9,
 434     0.8,
 435     0.7,
 436     0.6,
 437     0.5
 438   ],
 439   "json_load_ms": 16.05
 440 }`
 441
 442 const skottieFramesSampleTwo = `
 443 {
 444   "total_frame_ms": [
 445     210.555,
 446     770.795,
 447     10.430,
 448     31.845,
 449     3.610,
 450     1.105,
 451     0.545,
 452     2.315,
 453     1.685,
 454     0.615,
 455     0.425,
 456     0.815,
 457     0.355,
 458     0.655,
 459     0.390,
 460     400.455,
 461     0.800,
 462     0.685,
 463     2.630,
 464     0.325,
 465     0.355,
 466     0.740,
 467     0.785,
 468     0.795,
 469     0.72,
 470     0.80
 471   ],
 472   "without_flush_ms": [
 473     5.0,
 474     5.01,
 475     5.02,
 476     5.03,
 477     5.04,
 478     5.05,
 479     5.06,
 480     5.07,
 481     5.08,
 482     5.09,
 483     5.1,
 484     5.11,
 485     5.12,
 486     5.13,
 487     5.14,
 488     5.15,
 489     5.16,
 490     5.17,
 491     5.18,
 492     5.19,
 493     5.2,
 494     5.21,
 495     5.22,
 496     5.23,
 497     5.24,
 498     5.25
 499   ],
 500   "with_flush_ms": [
 501     5.0,
 502     4.9,
 503     4.8,
 504     4.7,
 505     4.6,
 506     4.5,
 507     4.4,
 508     4.3,
 509     4.2,
 510     4.1,
 511     4.0,
 512     3.9,
 513     3.8,
 514     3.7,
 515     3.6,
 516     3.5,
 517     3.4,
 518     3.3,
 519     3.2,
 520     3.1,
 521     3.0,
 522     2.9,
 523     2.8,
 524     2.7,
 525     2.6,
 526     2.5
 527   ],
 528   "json_load_ms": 28.15
 529 }`