From a86a02f5c1d39c44b071c8ba9891864bfe37f86c Mon Sep 17 00:00:00 2001 From: Dave Marchevsky Date: Mon, 20 Dec 2021 15:23:37 -0500 Subject: [PATCH] docker + tests: Run tests using python3, refactor Dockerfile In #3707, I added ubuntu-20.04 to the list of OS's bcc-test uses to run tests. I expected that this would result in the tests running on both 18.04 and 20.04. Unfortunately this change was effectively a no-op as the tests are run in a Docker container and the os field in bcc-test.yml dictates the type/version of the _host_ OS, not the container's OS. So it's not necessary to run on both, just run on 20.04. It will be necessary to modify Dockerfile.test to use both 18.04 and 20.04. To prepare for this, move all python tests to use python3 interpreter, as 20.04 doesn't have python2 pip readily available. Also, refactor Dockerfile.tests a bit so that it's possible to provide ubuntu version and shortname as build input. This commit does not result in the docker test container working/running both 18.04 and 20.04, rather lays groundwork for future commits to do so. Signed-off-by: Dave Marchevsky --- .github/workflows/bcc-test.yml | 2 +- Dockerfile.tests | 20 ++++++++++++-------- src/python/CMakeLists.txt | 2 +- tests/python/test_array.py | 2 +- tests/python/test_attach_perf_event.py | 2 +- tests/python/test_bpf_log.py | 2 +- tests/python/test_brb.py | 2 +- tests/python/test_brb2.py | 2 +- tests/python/test_call1.py | 2 +- tests/python/test_clang.py | 2 +- tests/python/test_debuginfo.py | 2 +- tests/python/test_disassembler.py | 2 +- tests/python/test_dump_func.py | 2 +- tests/python/test_flags.py | 2 +- tests/python/test_free_bcc_memory.py | 2 +- tests/python/test_histogram.py | 2 +- tests/python/test_license.py | 2 +- tests/python/test_lpm_trie.py | 2 +- tests/python/test_lru.py | 2 +- tests/python/test_map_batch_ops.py | 2 +- tests/python/test_map_in_map.py | 2 +- tests/python/test_percpu.py | 2 +- tests/python/test_perf_event.py | 2 +- tests/python/test_probe_count.py | 2 +- tests/python/test_queuestack.py | 2 +- tests/python/test_ringbuf.py | 2 +- tests/python/test_rlimit.py | 2 +- tests/python/test_shared_table.py | 2 +- tests/python/test_stackid.py | 2 +- tests/python/test_stat1.py | 2 +- tests/python/test_tools_memleak.py | 2 +- tests/python/test_tools_smoke.py | 2 +- tests/python/test_trace2.py | 2 +- tests/python/test_trace3.py | 2 +- tests/python/test_trace4.py | 2 +- tests/python/test_trace_maxactive.py | 2 +- tests/python/test_tracepoint.py | 2 +- tests/python/test_uprobes.py | 2 +- tests/python/test_uprobes2.py | 2 +- tests/python/test_usdt.py | 2 +- tests/python/test_usdt2.py | 2 +- tests/python/test_usdt3.py | 2 +- tests/python/test_utils.py | 2 +- tests/python/test_xlate1.py | 2 +- tests/wrapper.sh.in | 2 +- 45 files changed, 56 insertions(+), 52 deletions(-) diff --git a/.github/workflows/bcc-test.yml b/.github/workflows/bcc-test.yml index 324495f7..4f52d7ad 100644 --- a/.github/workflows/bcc-test.yml +++ b/.github/workflows/bcc-test.yml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-18.04, ubuntu-20.04] # 18.04.3 release has 5.0.0 kernel + os: [ubuntu-20.04] env: - TYPE: Debug PYTHON_TEST_LOGFILE: critical.log diff --git a/Dockerfile.tests b/Dockerfile.tests index 99b6a445..3b082364 100644 --- a/Dockerfile.tests +++ b/Dockerfile.tests @@ -1,17 +1,23 @@ -FROM ubuntu:18.04 +ARG UBUNTU_VERSION="18.04" +FROM ubuntu:${UBUNTU_VERSION} ARG LLVM_VERSION="8" ENV LLVM_VERSION=$LLVM_VERSION +ARG UBUNTU_SHORTNAME="bionic" + RUN apt-get update && apt-get install -y curl gnupg &&\ llvmRepository="\n\ -deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic main\n\ -deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic main\n\ -deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-${LLVM_VERSION} main\n\ -deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-${LLVM_VERSION} main\n" &&\ +deb http://apt.llvm.org/${UBUNTU_SHORTNAME}/ llvm-toolchain-${UBUNTU_SHORTNAME} main\n\ +deb-src http://apt.llvm.org/${UBUNTU_SHORTNAME}/ llvm-toolchain-${UBUNTU_SHORTNAME} main\n\ +deb http://apt.llvm.org/${UBUNTU_SHORTNAME}/ llvm-toolchain-${UBUNTU_SHORTNAME}-${LLVM_VERSION} main\n\ +deb-src http://apt.llvm.org/${UBUNTU_SHORTNAME}/ llvm-toolchain-${UBUNTU_SHORTNAME}-${LLVM_VERSION} main\n" &&\ echo $llvmRepository >> /etc/apt/sources.list && \ curl -L https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - +ARG DEBIAN_FRONTEND="noninteractive" +ENV TZ="Etc/UTC" + RUN apt-get update && apt-get install -y \ util-linux \ bison \ @@ -40,7 +46,6 @@ RUN apt-get update && apt-get install -y \ iproute2 \ python3 \ python3-pip \ - python-pip \ ethtool \ arping \ netperf \ @@ -50,8 +55,7 @@ RUN apt-get update && apt-get install -y \ libtinfo5 \ libtinfo-dev -RUN pip3 install pyroute2 netaddr dnslib cachetools -RUN pip install pyroute2==0.5.18 netaddr==0.8.0 dnslib==0.9.14 cachetools==3.1.1 +RUN pip3 install pyroute2==0.5.18 netaddr==0.8.0 dnslib==0.9.14 cachetools==3.1.1 # FIXME this is faster than building from source, but it seems there is a bug # in probing libruby.so rather than ruby binary diff --git a/src/python/CMakeLists.txt b/src/python/CMakeLists.txt index fa602397..f5c40b95 100644 --- a/src/python/CMakeLists.txt +++ b/src/python/CMakeLists.txt @@ -2,7 +2,7 @@ # Licensed under the Apache License, Version 2.0 (the "License") if(NOT PYTHON_CMD) - set(PYTHON_CMD "python") + set(PYTHON_CMD "python3") endif() if(EXISTS "/etc/debian_version") diff --git a/tests/python/test_array.py b/tests/python/test_array.py index 1461226d..0ca995d4 100755 --- a/tests/python/test_array.py +++ b/tests/python/test_array.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_attach_perf_event.py b/tests/python/test_attach_perf_event.py index a843b575..f4bae4ff 100755 --- a/tests/python/test_attach_perf_event.py +++ b/tests/python/test_attach_perf_event.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2021, Athira Rajeev, IBM Corp. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_bpf_log.py b/tests/python/test_bpf_log.py index cb3d0038..e38c5eae 100755 --- a/tests/python/test_bpf_log.py +++ b/tests/python/test_bpf_log.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_brb.py b/tests/python/test_brb.py index 74617566..ed4cb7f7 100755 --- a/tests/python/test_brb.py +++ b/tests/python/test_brb.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_brb2.py b/tests/python/test_brb2.py index f983de16..c7ef90ee 100755 --- a/tests/python/test_brb2.py +++ b/tests/python/test_brb2.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_call1.py b/tests/python/test_call1.py index 6766cab3..db40e1a7 100755 --- a/tests/python/test_call1.py +++ b/tests/python/test_call1.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_clang.py b/tests/python/test_clang.py index f6c05bae..7bf12cc3 100755 --- a/tests/python/test_clang.py +++ b/tests/python/test_clang.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_debuginfo.py b/tests/python/test_debuginfo.py index 28f29e6f..3bf5a2b9 100755 --- a/tests/python/test_debuginfo.py +++ b/tests/python/test_debuginfo.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) Sasha Goldshtein # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_disassembler.py b/tests/python/test_disassembler.py index 89c4ec56..bf324d2b 100755 --- a/tests/python/test_disassembler.py +++ b/tests/python/test_disassembler.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) Clevernet # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_dump_func.py b/tests/python/test_dump_func.py index 6fd3b49c..56872f5d 100755 --- a/tests/python/test_dump_func.py +++ b/tests/python/test_dump_func.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_flags.py b/tests/python/test_flags.py index a5d2b42d..1e0cb5a4 100644 --- a/tests/python/test_flags.py +++ b/tests/python/test_flags.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_free_bcc_memory.py b/tests/python/test_free_bcc_memory.py index 7d6f6f43..232baa6e 100755 --- a/tests/python/test_free_bcc_memory.py +++ b/tests/python/test_free_bcc_memory.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # USAGE: test_usdt.py # diff --git a/tests/python/test_histogram.py b/tests/python/test_histogram.py index cb878c6d..92eee91e 100755 --- a/tests/python/test_histogram.py +++ b/tests/python/test_histogram.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_license.py b/tests/python/test_license.py index 1358579c..f9b070da 100755 --- a/tests/python/test_license.py +++ b/tests/python/test_license.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2018 Clevernet, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_lpm_trie.py b/tests/python/test_lpm_trie.py index 638362a6..e77ae3f4 100755 --- a/tests/python/test_lpm_trie.py +++ b/tests/python/test_lpm_trie.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2017 Facebook, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_lru.py b/tests/python/test_lru.py index 2946edcc..5d4a049e 100644 --- a/tests/python/test_lru.py +++ b/tests/python/test_lru.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_map_batch_ops.py b/tests/python/test_map_batch_ops.py index ffc9bde6..8efa6855 100755 --- a/tests/python/test_map_batch_ops.py +++ b/tests/python/test_map_batch_ops.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # USAGE: test_map_batch_ops.py # diff --git a/tests/python/test_map_in_map.py b/tests/python/test_map_in_map.py index 751eb79e..46e62927 100755 --- a/tests/python/test_map_in_map.py +++ b/tests/python/test_map_in_map.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # USAGE: test_map_in_map.py # diff --git a/tests/python/test_percpu.py b/tests/python/test_percpu.py index b493752e..e1c646c0 100755 --- a/tests/python/test_percpu.py +++ b/tests/python/test_percpu.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_perf_event.py b/tests/python/test_perf_event.py index 882e71a1..6119ec72 100755 --- a/tests/python/test_perf_event.py +++ b/tests/python/test_perf_event.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2016 PLUMgrid # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_probe_count.py b/tests/python/test_probe_count.py index 1e40305f..3fbfc180 100755 --- a/tests/python/test_probe_count.py +++ b/tests/python/test_probe_count.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) Suchakra Sharma # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_queuestack.py b/tests/python/test_queuestack.py index c00283db..47ee7925 100755 --- a/tests/python/test_queuestack.py +++ b/tests/python/test_queuestack.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_ringbuf.py b/tests/python/test_ringbuf.py index 93245be5..87adbff4 100755 --- a/tests/python/test_ringbuf.py +++ b/tests/python/test_ringbuf.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_rlimit.py b/tests/python/test_rlimit.py index deda8a78..8e93baba 100755 --- a/tests/python/test_rlimit.py +++ b/tests/python/test_rlimit.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # USAGE: test_usdt.py # diff --git a/tests/python/test_shared_table.py b/tests/python/test_shared_table.py index 10dd63f8..7b8f9d5b 100644 --- a/tests/python/test_shared_table.py +++ b/tests/python/test_shared_table.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2016 Facebook, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_stackid.py b/tests/python/test_stackid.py index 34a756b4..707604e9 100755 --- a/tests/python/test_stackid.py +++ b/tests/python/test_stackid.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_stat1.py b/tests/python/test_stat1.py index 10433099..313e2a9e 100755 --- a/tests/python/test_stat1.py +++ b/tests/python/test_stat1.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_tools_memleak.py b/tests/python/test_tools_memleak.py index 45528b83..df023667 100755 --- a/tests/python/test_tools_memleak.py +++ b/tests/python/test_tools_memleak.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from unittest import main, skipUnless, TestCase from utils import kernel_version_ge diff --git a/tests/python/test_tools_smoke.py b/tests/python/test_tools_smoke.py index 64bf500a..182e3e91 100755 --- a/tests/python/test_tools_smoke.py +++ b/tests/python/test_tools_smoke.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) Sasha Goldshtein, 2017 # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_trace2.py b/tests/python/test_trace2.py index 8614bca7..d7094c3b 100755 --- a/tests/python/test_trace2.py +++ b/tests/python/test_trace2.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_trace3.py b/tests/python/test_trace3.py index 7843d742..a0493011 100755 --- a/tests/python/test_trace3.py +++ b/tests/python/test_trace3.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_trace4.py b/tests/python/test_trace4.py index 8fb680e6..a4b8ab02 100755 --- a/tests/python/test_trace4.py +++ b/tests/python/test_trace4.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_trace_maxactive.py b/tests/python/test_trace_maxactive.py index 677ca8bb..e75d7e87 100755 --- a/tests/python/test_trace_maxactive.py +++ b/tests/python/test_trace_maxactive.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_tracepoint.py b/tests/python/test_tracepoint.py index ddc2c979..a2dd40f4 100755 --- a/tests/python/test_tracepoint.py +++ b/tests/python/test_tracepoint.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) Sasha Goldshtein # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_uprobes.py b/tests/python/test_uprobes.py index f7c78e75..621b45cf 100755 --- a/tests/python/test_uprobes.py +++ b/tests/python/test_uprobes.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_uprobes2.py b/tests/python/test_uprobes2.py index 6118754a..0c0b0101 100755 --- a/tests/python/test_uprobes2.py +++ b/tests/python/test_uprobes2.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # USAGE: test_uprobe2.py # diff --git a/tests/python/test_usdt.py b/tests/python/test_usdt.py index d84ccc99..d026a503 100755 --- a/tests/python/test_usdt.py +++ b/tests/python/test_usdt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # USAGE: test_usdt.py # diff --git a/tests/python/test_usdt2.py b/tests/python/test_usdt2.py index c2b2daeb..f8da339d 100755 --- a/tests/python/test_usdt2.py +++ b/tests/python/test_usdt2.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # USAGE: test_usdt2.py # diff --git a/tests/python/test_usdt3.py b/tests/python/test_usdt3.py index 5fe2ef4f..a378fd48 100755 --- a/tests/python/test_usdt3.py +++ b/tests/python/test_usdt3.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # USAGE: test_usdt3.py # diff --git a/tests/python/test_utils.py b/tests/python/test_utils.py index 54b97cfb..9a3b7b7a 100755 --- a/tests/python/test_utils.py +++ b/tests/python/test_utils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) Catalysts GmbH # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/python/test_xlate1.py b/tests/python/test_xlate1.py index 5183e2a2..3057c45d 100755 --- a/tests/python/test_xlate1.py +++ b/tests/python/test_xlate1.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) PLUMgrid, Inc. # Licensed under the Apache License, Version 2.0 (the "License") diff --git a/tests/wrapper.sh.in b/tests/wrapper.sh.in index 88229dcd..a096ac87 100755 --- a/tests/wrapper.sh.in +++ b/tests/wrapper.sh.in @@ -8,7 +8,7 @@ name=$1; shift kind=$1; shift cmd=$1; shift -PYTHONPATH=@CMAKE_BINARY_DIR@/src/python/bcc-python +PYTHONPATH=@CMAKE_BINARY_DIR@/src/python/bcc-python3 LD_LIBRARY_PATH=@CMAKE_BINARY_DIR@:@CMAKE_BINARY_DIR@/src/cc ns=$name -- 2.34.1