From: Gichan Jang Date: Fri, 20 Dec 2024 01:28:32 +0000 (+0900) Subject: [Action] Add actions to replace TAOS-CI X-Git-Tag: accepted/tizen/unified/20250108.103324~11 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=f589e738e0df11d7a5ea820d2b72f1de786a3de4;p=platform%2Fupstream%2Fnnstreamer-edge.git [Action] Add actions to replace TAOS-CI Add actions to replace TAOS-CI Signed-off-by: Gichan Jang --- diff --git a/.github/actions/check-rebuild/action.yml b/.github/actions/check-rebuild/action.yml new file mode 100644 index 0000000..cdf20e9 --- /dev/null +++ b/.github/actions/check-rebuild/action.yml @@ -0,0 +1,20 @@ +name: Check if rebuild required +description: + +inputs: + mode: + description: build mode to be checked + required: false + default: build + +runs: + using: composite + steps: + - run: | + tmpfile=$(mktemp) + git show --pretty="format:" --name-only --diff-filter=AMRC ${{ github.event.pull_request.head.sha}} -${{ github.event.pull_request.commits }} | sort | uniq | awk NF > ${tmpfile} + echo "changed_file_list=${tmpfile}" >> "$GITHUB_ENV" + rebuild=`bash .github/actions/check-rebuild/check_if_rebuild_requires.sh ${tmpfile} ${{ inputs.mode }} | grep "REBUILD=YES" | wc -l` + echo "Rebuild required: ${rebuild}" + echo "rebuild=${rebuild}" >> "$GITHUB_ENV" + shell: sh diff --git a/.github/actions/check-rebuild/check_if_rebuild_requires.sh b/.github/actions/check-rebuild/check_if_rebuild_requires.sh new file mode 100644 index 0000000..651226d --- /dev/null +++ b/.github/actions/check-rebuild/check_if_rebuild_requires.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +## +# Copyright (c) 2024 Samsung Electronics Co., Ltd. All Rights Reserved. +# +# @file: check_if_rebuild_requires.sh +# @brief Check if rebuild & unit-test is required with the given PR. +# @see https://github.com/nnstreamer/nnstreamer +# @author MyungJoo Ham +# +# Argument 1 ($1): the file containing list of files to be checked. +# Argument 2 ($2): build mode to be checked +# gbs: check if Tizen GBS build is required +# debian: check if pdebuild is required +# build (default): check if general meson rebuild is required. + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi + +if [ -z $2 ]; then + mode="build" +else + mode=$2 +fi + +rebuild=0 +regbs=0 +redebian=0 + +for file in `cat $1`; do + case $file in + *.md|*.png|*.webp|*.css|*.html ) + ;; + packaging/* ) + regbs='1' + ;; + debian/* ) + redebian='1' + ;; + * ) + rebuild='1' + regbs='1' + redebian='1' + ;; + esac +done + +case $mode in + gbs) + if [[ "$regbs" == "1" ]]; then + echo "REBUILD=YES" + exit 0 + fi + ;; + debian) + if [[ "$redebian" == "1" ]]; then + echo "REBUILD=YES" + exit 0 + fi + ;; + *) + if [[ "$rebuild" == "1" ]]; then + echo "REBUILD=YES" + exit 0 + fi + ;; +esac + +echo "REBUILD=NO" diff --git a/.github/actions/gitpush/action.yml b/.github/actions/gitpush/action.yml new file mode 100644 index 0000000..a60cccb --- /dev/null +++ b/.github/actions/gitpush/action.yml @@ -0,0 +1,38 @@ +name: 'Push to github.io' +description: 'Update the github action result to nnstreamer.github.io' +inputs: + source: + description: 'source path of the file or directory to be copied' + required: true + dest: + description: 'destination directory in nnstreamer.github.io repository' + required: true + message: + description: 'commit message' + required: false + default: 'Update the result from nnstreamer github action.' + taos_account: + required: true + taos_account_token: + required: true + +runs: + using: "composite" + steps: + - name: update the result + run: | + git clone https://${{ inputs.taos_account }}:${{ inputs.taos_account_token }}@github.com/nnstreamer/nnstreamer.github.io.git + pushd nnstreamer.github.io + mkdir -p ${{ inputs.dest }} + cp -r ${{ inputs.source }} ${{ inputs.dest }} + if git diff --shortstat | grep -q changed; then + git config user.email "nnsuite@samsung.com" + git config user.name "nnsuite" + git add * + git commit -s -m "${{ inputs.message }}" + git push origin main -f + else + echo "Nothing to commit. Skip the push operation." + fi + popd + shell: bash diff --git a/.github/workflows/daily-build-gbs.yml b/.github/workflows/daily-build-gbs.yml new file mode 100644 index 0000000..f3c3cd1 --- /dev/null +++ b/.github/workflows/daily-build-gbs.yml @@ -0,0 +1,78 @@ +name: gbs daily build and test + +on: + schedule: + # 05:00 AM (KST) Mon-Fri + - cron: "00 20 * * 0-4" + + # Allow manually triggering the workflow + workflow_dispatch: + +jobs: + build: + outputs: + x86_64: ${{ steps.gbs-result.outputs.x86_64 }} + armv7l: ${{ steps.gbs-result.outputs.armv7l }} + aarch64: ${{ steps.gbs-result.outputs.aarch64 }} + strategy: + fail-fast: false + matrix: + include: + - gbs_build_arch: "x86_64" + gbs_build_option: "--define \"unit_test 1\" --define \"testcoverage 1\"" + - gbs_build_arch: "armv7l" + gbs_build_option: "--define \"unit_test 1\"" + - gbs_build_arch: "aarch64" + gbs_build_option: "--define \"unit_test 1\"" + + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v1 + + - name: prepare deb sources for GBS + run: echo "deb [trusted=yes] http://download.tizen.org/tools/latest-release/Ubuntu_22.04/ /" | sudo tee /etc/apt/sources.list.d/tizen.list + + - name: install GBS + run: sudo apt-get update && sudo apt-get install -y gbs + + - name: configure GBS + run: cp .github/workflows/tizen.gbs.conf ~/.gbs.conf + + - name: get date + id: get-date + run: | + echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: run GBS + id: gbs-build + run: gbs build ${{ matrix.gbs_build_option }} --define "_skip_debug_rpm 1" -A ${{ matrix.gbs_build_arch }} + + - name: save gbs cache + uses: actions/cache/save@v4 + if: ${{ always() && github.ref == 'refs/heads/main' }} + with: + path: ~/GBS-ROOT/local/cache + key: gbs-cache-${{ matrix.gbs_build_arch }}-${{ steps.get-date.outputs.date }} + + - if: matrix.gbs_build_arch == 'x86_64' && steps.gbs-build.outcome == 'success' + name: extract test coverage result + run: | + pip install pybadges beautifulsoup4 setuptools + mkdir -p ~/testresult/ + pushd ~/testresult/ + cp ~/GBS-ROOT/local/repos/tizen/x86_64/RPMS/*-coverage*.rpm . + rpm2cpio *-coverage*.rpm | cpio -idv + popd + python3 .github/workflows/gen_coverage_badge.py ~/testresult/usr/share/nnstreamer-edge/unittest/result/index.html ~/testresult/usr/share/nnstreamer-edge/unittest/result/coverage_badge.svg + + - if: matrix.gbs_build_arch == 'x86_64' && steps.gbs-build.outcome == 'success' + name: update test coverage result to github.io + uses: ./.github/actions/gitpush + with: + source: ~/testresult/usr/share/nnstreamer-edge/unittest/result/* + dest: testresult/nnstreamer-edge + message: "${{ steps.get-date.outputs.date }} : Update test coverage result." + taos_account: ${{ secrets.TAOS_ACCOUNT }} + taos_account_token: ${{ secrets.TAOS_ACCOUNT_TOKEN }} diff --git a/.github/workflows/gbs_build.yml b/.github/workflows/gbs_build.yml new file mode 100644 index 0000000..e6be9f2 --- /dev/null +++ b/.github/workflows/gbs_build.yml @@ -0,0 +1,57 @@ +name: Build and unit test/ Tizen/GBS + +on: + pull_request: + branches: [ main ] + +jobs: + build: + name: Tizen GBS build on Ubuntu + strategy: + matrix: + include: + - gbs_build_arch: "x86_64" + gbs_build_option: "--define \"unit_test 1\"" + - gbs_build_arch: "armv7l" + gbs_build_option: "--define \"unit_test 0\"" + - gbs_build_arch: "aarch64" + gbs_build_option: "--define \"unit_test 0\"" + + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: -${{ github.event.pull_request.commits }} + - name: Check if rebuild required + uses: ./.github/actions/check-rebuild + with: + mode: gbs + - uses: actions/setup-python@v1 + + - name: prepare GBS + if: env.rebuild == '1' + run: | + echo "deb [trusted=yes] http://download.tizen.org/tools/latest-release/Ubuntu_22.04/ /" | sudo tee /etc/apt/sources.list.d/tizen.list + sudo apt-get update && sudo apt-get install -y gbs + cp .github/workflows/tizen.gbs.conf ~/.gbs.conf + + - name: get date + id: get-date + run: | + echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: restore gbs cache from main branch + if: env.rebuild == '1' + uses: actions/cache/restore@v4 + with: + path: ~/GBS-ROOT/local/cache + key: gbs-cache-${{ matrix.gbs_build_arch }}-${{ steps.get-date.outputs.date }} + restore-keys: | + gbs-cache-${{ matrix.gbs_build_arch }}- + + - name: run GBS + if: env.rebuild == '1' + run: | + gbs build --skip-srcrpm --define "_skip_debug_rpm 1" -A ${{ matrix.gbs_build_arch }} ${{ matrix.gbs_build_option }} diff --git a/.github/workflows/pdebuild.yml b/.github/workflows/pdebuild.yml new file mode 100644 index 0000000..4f43b18 --- /dev/null +++ b/.github/workflows/pdebuild.yml @@ -0,0 +1,110 @@ +name: Build and unit test/ Pdebuild Ubuntu 22.04 + +on: + pull_request: + branches: [ main ] + + schedule: + # 04:00 AM (KST) Mon-Fri + - cron: "00 19 * * 0-4" + + # Allow manually triggering the workflow + workflow_dispatch: + +jobs: + build: + strategy: + matrix: + os: [ ubuntu-22.04 ] + arch: [ amd64 ] + include: + - distroname: jammy + os: ubuntu-22.04 + + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: -${{ github.event.pull_request.commits }} + - name: Check if rebuild required + uses: ./.github/actions/check-rebuild + with: + mode: build + - uses: actions/setup-python@v5 + + - name: Check trigger event + id: rebuild + run: | + if ${{ env.rebuild == '1' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}; then + echo "rebuild=1" >> $GITHUB_OUTPUT + else + echo "rebuild=0" >> $GITHUB_OUTPUT + fi + + - name: get date + id: get-date + run: | + echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: make cache dir for pbuilder + ## prevent permission error + run: sudo mkdir --mode a=rwx --parents /var/cache/pbuilder + + - name: restore pbuilder cache for pull-request + id: restore-pbuilder-cache + if: env.rebuild == '1' && github.event_name == 'pull_request' + uses: actions/cache/restore@v4 + with: + path: | + /var/cache/pbuilder/aptcache + /var/cache/pbuilder/base.tgz + key: pbuilder-cache-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles('**/debian/control') }}-${{ steps.get-date.outputs.date }} + restore-keys: | + pbuilder-cache-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles('**/debian/control') }}- + pbuilder-cache-${{ matrix.os }}-${{ matrix.arch }}- + + - name: prepare pdebuild + if: steps.rebuild.outputs.rebuild == '1' + run: | + echo "Installing build tools" + sudo add-apt-repository ppa:nnstreamer/ppa + echo "::group::apt-get update && apt-get install" + sudo apt-get update && sudo apt-get install -y pbuilder debootstrap curl ubuntu-dev-tools qemu-user-static debian-archive-keyring ubuntu-keyring debhelper + echo "::endgroup::" + echo "DISTRIBUTION=${{ matrix.distroname }}" > ~/.pbuilderrc + echo "OTHERMIRROR=\"deb [trusted=yes] http://archive.ubuntu.com/ubuntu ${{ matrix.distroname }}-backports universe |deb [trusted=yes] http://ppa.launchpad.net/nnstreamer/ppa/ubuntu ${{ matrix.distroname }} main\"" >> ~/.pbuilderrc + cat ~/.pbuilderrc + sudo mkdir -p /root/ + sudo ln -s ~/.pbuilderrc /root/ + + - name: make pbuilder base.tgz + if: ${{ steps.rebuild.outputs.rebuild == '1' && steps.restore-pbuilder-cache.outputs.cache-hit != 'true' }} + run: | + echo "=== pbuilder create" + echo "::group::pbuilder create --allow-untrusted" + sudo pbuilder create --allow-untrusted + echo "::endgroup::" + echo "=== pbuilder update" + echo "::group::pbuilder update --distribution ${{ matrix.distroname }}" + sudo pbuilder update --distribution ${{ matrix.distroname }} + echo "::endgroup" + echo "::group::pbuilder update" + sudo pbuilder update + echo "::endgroup" + + - name: run pdebuild + if: steps.rebuild.outputs.rebuild == '1' + id: pdebuild + run: | + mkdir -p ~/daily_build/ubuntu + pdebuild --logfile ~/daily_build/pdebuild_log.txt --buildresult ~/daily_build/ubuntu --architecture ${{ matrix.arch }} -- --distribution ${{ matrix.distroname }} + + - name: save pbuilder cache + if: always() && ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' }} + uses: actions/cache/save@v4 + with: + path: | + /var/cache/pbuilder/aptcache + /var/cache/pbuilder/base.tgz + key: pbuilder-cache-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles('**/debian/control') }}-${{ steps.get-date.outputs.date }} diff --git a/.github/workflows/static.check.scripts/cppcheck.sh b/.github/workflows/static.check.scripts/cppcheck.sh new file mode 100644 index 0000000..1031631 --- /dev/null +++ b/.github/workflows/static.check.scripts/cppcheck.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the file containing the list of files to be checked +# Argument 2 ($2): check level. (default 0) + +## +# @file cppcheck.sh +# @brief Check dangerous coding constructs in source codes (*.c, *.cpp) with a cppcheck tool +# Originally pr-prebuild-cppcheck.sh +# +# The possible severities (e.g., --enable=warning,unusedFunction) for messages are as following: +# Note that by default Cppcheck only writes error messages if it is certain. +# 1. error : used when bugs are found +# 2. warning: suggestions about defensive programming to prevent bugs +# 3. style : stylistic issues related to code cleanup (unused functions, redundant code, constness, and such) +# 4. performance: Suggestions for making the code faster. These suggestions are only based on common knowledge. +# 5. portability: portability warnings. 64-bit portability. code might work different on different compilers. etc. +# 6. information: Informational messages about checking problems. +# 7. unusedFunction: enable unusedFunction checking. This is not enabled by --enable=style +# because it does not work well on libraries. +# 8. all: enable all messages. It should also only be used when the whole program is scanned. +# +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/danmar/cppcheck +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham +# + +function version(){ + echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; +} + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi +if [ -z $2 ]; then + 2=0 +fi + +function check(){ + which $1 + if [[ $? -ne 0 ]]; then + echo "::error The command $1 is required, but not found." + exit 1 + fi +} + +check cppcheck +check file +check grep +check cat +check wc +check awk + +files=$1 +failed=0 + +if [ ! -f $files ]; then + echo "::error The file $files does not exists." + exit 1 +fi + +# Display the cppcheck version that is installed in the CI server. +# Note that the out-of-date version can generate an incorrect result. +cppcheck_ver=$(cppcheck --version | awk {'print $2'}) +default_cmd="--std=posix" +# --std=posix is deprecated and removed in 2.0.5 +if [[ $(version $cppcheck_ver) -ge $(version "2.0.5") ]]; then + default_cmd="--library=posix" +fi + +static_analysis_sw="cppcheck" + +if [[ $2 -eq 0 ]]; then + echo "cppcheck: Default mode." + static_analysis_rules="$default_cmd" +elif [[ $2 -eq 1 ]]; then + echo "cppcheck: --enable=warning,performance added." + static_analysis_rules="--enable=warning,performance $default_cmd" +else + echo "cppcheck: $2 is an incorrect optiona. Overriding it as 0" + static_analysis_rules="$default_cmd" +fi + +errfile=$(mktemp) +error="no" +errors="" + +for file in `cat $files`; do + # skip obsolete folder + if [[ $file =~ ^obsolete/.* ]]; then + continue + fi + # skip external folder + if [[ $file =~ ^external/.* ]]; then + continue + fi + # Handle only text files in case that there are lots of files in one commit. + if [[ `file $file | grep "ASCII text" | wc -l` -gt 0 ]]; then + # in case of source code files (*.c, *.cpp) + case $file in + # in case of C/C++ code + *.c|*.cpp) + echo "( $file ) file is source code with the text format." + # Check C/C++ file, enable all checks. + $static_analysis_sw $static_analysis_rules $file 2> $errfile + bug_line=`$errfile | wc -l ` + if [[ $bug_line -gt 0 ]]; then + echo "$file cppcheck result shows some errors. There are $bug_line bug(s):" + echo ":group:cppcheck result of $file" + cat $errfile + echo ":endgroup:" + error="yes" + errors+=" $file" + else + echo "$file cppcheck result is ok." + fi + ;; + esac + fi +done + +if [[ "$error" == "yes" ]]; then + echo "cppcheck shows errors in: $errors" + exit 1 +fi diff --git a/.github/workflows/static.check.scripts/doxygen-build.sh b/.github/workflows/static.check.scripts/doxygen-build.sh new file mode 100644 index 0000000..c2026a1 --- /dev/null +++ b/.github/workflows/static.check.scripts/doxygen-build.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the file containing the list of files to be checked +# + +## +# @file doxygen-build.sh +# @brief Check a doxygen grammar if a doxygen can normally generates source code +# Originally, pr-prebuild-doxygen-build.sh +# +# Doxygen is the de facto standard tool for generating documentation from annotated C++ +# sources, but it also supports other popular programming languages such as C, Objective-C, +# C#, PHP, Java, Python, IDL (Corba, Microsoft, and UNO/OpenOffice flavors), Fortran, VHDL, +# Tcl, and to some extent D. +# +# @see http://www.doxygen.nl/ +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham +# @note +# Note that module developer has to execute a self evaluaton if the plug-in module includes incorrect grammar(s). +# + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi + +files=$1 +failed=0 + +if [ ! -f $files ]; then + echo "::error The file $files does not exists." + exit 1 +fi + +function check(){ + which $1 + if [[ $? -ne 0 ]]; then + echo "::error The command $1 is required, but not found." + exit 1 + fi +} +check file +check grep +check cat +check wc +check doxygen + +errorresult=$(mktemp) +doxygen_check_result="doxygen_build_result.txt" + +# Inspect all files that contributor modifed. +for file in `cat $files`; do + # skip obsolete folder + if [[ $file =~ ^obsolete/.* ]]; then + continue + fi + # skip external folder + if [[ $file =~ ^external/.* ]]; then + continue + fi + + # Handle only a source code sequentially in case that there are lots of files in one commit. + if [[ `file $file | grep "ASCII text" | wc -l` -gt 0 ]]; then + case $file in + # In case of source code + *.c | *.cpp | *.cc | *.hh | *.h | *.hpp | *.py | *.sh | *.php | *.java) + doxygen_analysis_sw="doxygen" + doxygen_analysis_rules=" - " + doxygen_analysis_config=".github/workflows/static.check.scripts/Doxyfile.prj" + + # Doxygen Usage: ( cat ../Doxyfile.ci ; echo "INPUT=./webhook.php" ) | doxygen - + ( cat $doxygen_analysis_config ; echo "INPUT=$file" ) | $doxygen_analysis_sw $doxygen_analysis_rules + result=$? + + if [[ $result != 0 ]]; then + failed=1 + echo "====================================================" >> $errorresult + echo "Doxygen has failed in file $file" >> $errorresult + echo "====================================================" >> $errorresult + cat $doxygen_check_result >> $errorresult + echo "====================================================\n\n" >> $errorresult + fi + ;; + esac + fi +done + +if [[ $failed == "1" ]]; then + echo "::group::There are doxygen build errors in the pull-requset" + cat $errorresult + echo "::endgroup::" + echo "::error Doxygen build test has failed." + exit 1 +fi diff --git a/.github/workflows/static.check.scripts/doxygen-tag.sh b/.github/workflows/static.check.scripts/doxygen-tag.sh new file mode 100755 index 0000000..d3f8696 --- /dev/null +++ b/.github/workflows/static.check.scripts/doxygen-tag.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the file containing the list of files to be checked + +## +# @file doxygen-tag.sh +# @brief Check if there is a doxygen-tag issue. Originally pr-prebuild-doxygen-tag.sh +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham +# + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi + +advanced=0 +if [ "$2" = "1" ]; then + advanced=1 +fi + +files=$1 +failed=0 + +if [ ! -f $files ]; then + echo "::error The file $files does not exists." + exit 1 +fi + +echo "::group::Doxygen tag check started" + +for file in `cat $files`; do + if [[ `file $file | grep "ASCII text" | wc -l` -gt 0 ]]; then + # In case of source code files: *.c|*.h|*.cpp|*.py|*.sh|*.php ) + case $file in + # In case of C/C++ code + *.c|*.h|*.cc|*.hh|*.cpp|*.hpp ) + echo "[DEBUG] ( $file ) file is source code with the text format." >> $report_path + doxygen_lang="doxygen-cncpp" + # Append a doxgen rule step by step + doxygen_basic_rules="@file @brief" # @file and @brief to inspect file + doxygen_advanced_rules="@author @bug" # @author, @bug to inspect file, @brief for to inspect function + + # Apply advanced doxygen rule if pr_doxygen_check_level=1 in config-environment.sh + if [[ $advanced == 1 ]]; then + doxygen_basic_rules="$doxygen_basic_rules $doxygen_advanced_rules" + fi + + for word in $doxygen_basic_rules + do + doxygen_rule_compare_count=`cat ${file} | grep "$word" | wc -l` + doxygen_rule_expect_count=1 + + # Doxygen_rule_compare_count: real number of Doxygen tags in a file + # Doxygen_rule_expect_count: required number of Doxygen tags + if [[ $doxygen_rule_compare_count -lt $doxygen_rule_expect_count ]]; then + echo "[ERROR] $doxygen_lang: failed. file name: $file, $word tag is required at the top of file" + failed=1 + fi + done + + # Checking tags for each function + if [[ $advanced == 1 ]]; then + declare -i idx=0 + function_positions="" # Line number of functions. + structure_positions="" # Line number of structure. + + local function_check_flag="f+p" # check document for function and prototype of the function + + if [[ $pr_doxygen_check_skip_function_definition == 1 && $file != *.h ]]; then + function_check_flag="p" # check document for only prototypes of the function for non-header file + fi + + # Find line number of functions using ctags, and append them. + while IFS='' read -r line || [[ -n "$line" ]]; do + temp=`echo $line | cut -d ' ' -f3` # line number of function place 3rd field when divided into ' ' >> $report_path + function_positions="$function_positions $temp " + done < <(ctags -x --c-kinds=$function_check_flag $file) # "--c-kinds=f" mean find function + + # Find line number of structure using ctags, and append them. + while IFS='' read -r line || [[ -n "$line" ]]; do + temp=`echo $line | cut -d ' ' -f3` # line number of structure place 3rd field when divided into ' ' >> $report_path + structure_positions="$structure_positions $temp " + done < <(ctags -x --c-kinds=sc $file) # "--c-kinds=sc" mean find 's'truct and 'c'lass + + # Checking committed file line by line for detailed hints when missing Doxygen tags. + while IFS='' read -r line || [[ -n "$line" ]]; do + idx+=1 + + # Check if a function has @brief tag or not. + # To pass correct line number not sub number, keep space " $idx ". + # ex) want to pass 143 not 14, 43, 1, 3, 4 + if [[ $function_positions =~ " $idx " && $brief -eq 0 ]]; then + echo "[ERROR] File name: $file, $idx line, `echo $line | cut -d ' ' -f1` function needs @brief tag " + failed=1 + fi + + # Check if a structure has @brief tag or not. + # To pass correct line number not sub number, keep space " $idx ". + # For example, we want to pass 143 not 14, 43, 1, 3, and 4. + if [[ $structure_positions =~ " $idx " && $brief -eq 0 ]]; then # same as above. + echo "[ERROR] File name: $file, $idx line, structure needs @brief tag " + failed=1 + fi + + # Find brief or copydoc tag in the comments between the codes. + if [[ $line =~ "@brief" || $line =~ "@copydoc" ]]; then + brief=1 + # Doxygen tags become zero in code section. + elif [[ $line != *"*"* && ( $line =~ ";" || $line =~ "}" || $line =~ "#") ]]; then + brief=0 + fi + + # Check a comment statement that begins with '/*'. + # Note that doxygen does not recognize a comment statement that start with '/*'. + # Let's skip the doxygen tag inspection such as "/**" in case of a single line comment. + if [[ $line =~ "/*" && $line != *"/**"* && ( $line != *"*/" || $line =~ "@" ) && ( $idx != 1 ) ]]; then + echo "[ERROR] File name: $file, $idx line, Doxygen or multi line comments should begin with /**" + failed=1 + fi + + # Check the doxygen tag written in upper case beacuase doxygen cannot use upper case tag such as '@TODO'. + # Let's check a comment statement that begins with '//','/*' or ' *'. + if [[ ($line =~ "@"[A-Z]) && ($line == *([[:blank:]])"/*"* || $line == *([[:blank:]])"//"* || $line == *([[:blank:]])"*"*) ]]; then + echo "[ERROR] File name: $file, $idx line, The doxygen tag sholud be written in lower case." + failed=1 + fi + + done < "$file" + fi + ;; + # In case of Python code + *.py ) + doxygen_lang="doxygen-python" + # Append a Doxgen rule step by step + doxygen_rules="@package @brief" + doxygen_rule_num=0 + doxygen_rule_all=0 + + for word in $doxygen_rules + do + doxygen_rule_all=$(( doxygen_rule_all + 1 )) + doxygen_rule[$doxygen_rule_all]=`cat ${file} | grep "$word" | wc -l` + doxygen_rule_num=$(( $doxygen_rule_num + ${doxygen_rule[$doxygen_rule_all]} )) + done + if [[ $doxygen_rule_num -le 0 ]]; then + echo "[ERROR] $doxygen_lang: failed. file name: $file, ($doxygen_rule_num)/$doxygen_rule_all tags are found." + failed=1 + break + else + echo "[DEBUG] $doxygen_lang: passed. file name: $file, ($doxygen_rule_num)/$doxygen_rule_all tags are found." >> $report_path + fi + ;; + esac + fi +done + +echo "::endgroup::" + +if [ $failed = 1 ]; then + echo "::error There is a doxygen tag missing or incorrect." + exit 1 +fi diff --git a/.github/workflows/static.check.scripts/executable.sh b/.github/workflows/static.check.scripts/executable.sh new file mode 100644 index 0000000..2f9d3f1 --- /dev/null +++ b/.github/workflows/static.check.scripts/executable.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the list of files changed in this pull request. + +## +# @file executable.sh +# @brief Check executable bits for .cpp, .c, .hpp, .h, .prototxt, .caffemodel, .txt., .init +# Originally, pr-prebuild-executable.sh +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham +# + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi + +files=$1 +failed=0 + +if [ ! -f $files ]; then + echo "::error The file $files does not exists." + exit 1 +fi + +errorreport=$(mktemp) + +for file in `cat $files`; do + if [[ $file =~ \.cpp$ || $file =~ \.c$ || $file =~ \.hpp$ || $file =~ \.h$ || $file =~ \.prototxt$ || $file =~ \.caffemodel$ || $file =~ \.txt$ || $file =~ \.ini$ ]]; then + if [[ -f "$file" && -x "$file" ]]; then + # It is a text file (.cpp, .c, ...) and is executable. This is invalid! + failed=1 + ls -la $file >> $errorreport + fi + fi +done + +if [ $failed = 1 ]; then + echo "::group::List of files with executable bits that should not be executable" + cat $errorreport + echo "::endgroup::" + echo "::error There are source code files with executable bits. Fix them." + exit 1 +fi diff --git a/.github/workflows/static.check.scripts/flawfinder.sh b/.github/workflows/static.check.scripts/flawfinder.sh new file mode 100644 index 0000000..78cd7b3 --- /dev/null +++ b/.github/workflows/static.check.scripts/flawfinder.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the file containing the list of files to be checked +# Argument 2 ($2): the flawfinder check level. (1 by default) +# + +## +# @file flawfinder.sh +# @brief This module examines C/C++ source code to find a possible security weaknesses. +# Originally, pr-prebuild-flawfinder.sh +# +# Flawfinder searches through C/C++ source code looking for potential security flaws. It examines +# all of the project's C/C++ source code. It is very useful for quickly finding and removing some +# security problems before a program is widely released. +# +# Flawfinder produces a list of `hits` (potential security flaws), sorted by risk; the riskiest +# hits are shown first. The risk level is shown inside square brackets and varies from 0 (very +# little risk) to 5 (great risk). This risk level depends not only on the function, +# but on the values of the parameters of the function. For example, constant strings are often less risky +# than fully variable strings in many contexts, and in those contexts the hit will have a lower risk level. +# Hit descriptions also note the relevant Common Weakness Enumeration (CWE) identifier(s) in parentheses. +# +# @see https://dwheeler.com/flawfinder/ +# @see https://sourceforge.net/projects/flawfinder/ +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi + +pr_flawfinder_check_level=1 +if [ -z $2 ]; then + echo "Flawfinder check level is not given. The default value 1 is applied" +else + if [ -n "$2" ] && [ "$2" -eq "$2" ] 2>/dev/null; then + echo "" + else + echo ":error The second argument '$2' should be a number." + exit 1 + fi + pr_flawfinder_check_level=$2 +fi + +files=$1 +failed=0 + +if [ ! -f $files ]; then + echo "::error The file $files does not exists." + exit 1 +fi + +function check(){ + which $1 + if [[ $? -ne 0 ]]; then + echo "::error The command $1 is required, but not found." + exit 1 + fi +} +# Check if server administrator install required commands +check flawfinder +check file +check grep +check cat +check wc +check git +check awk + +static_analysis_sw="flawfinder" +static_analysis_rules="--html --context --minlevel=$pr_flawfinder_check_level" +flawfinder_result=$(mktemp) + +# Display the flawfinder version that is installed in the CI server. +# Note that the out-of-date version can generate an incorrect result. +flawfinder --version + +# Read file names that a contributor modified (e.g., added, moved, deleted, and updated) from a last commit. +# Then, inspect C/C++ source code files from *.patch files of the last commit. +for file in `cat $file`; do + # Skip the obsolete folder + if [[ $file =~ ^obsolete/.* ]]; then + continue + fi + # Skip the external folder + if [[ $file =~ ^external/.* ]]; then + continue + fi + # Handle only text files in case that there are lots of files in one commit. + if [[ `file $file | grep "ASCII text" | wc -l` -gt 0 ]]; then + # in case of C/C++ source code + case $file in + # in case of C/C++ code + *.c|*.cc|*.cpp|*.c++) + $static_analysis_sw $static_analysis_rules $file > ${flawfinder_result} + bug_nums=`cat ${flawfinder_result} | grep "Hits = " | awk '{print $3}'` + # Report the execution result. + if [[ $bug_nums -gt 0 ]]; then + echo "[ERROR] $static_analysis_sw: failed. file name: $file, There are $bug_nums bug(s)." + echo "::group::The flawfinder result of $file:" + cat ${flawfinder_result} + echo "::endgroup::" + failed=1 + else + echo "[DEBUG] $static_analysis_sw: passed. file name: $file, There are $bug_nums bug(s)." + fi + ;; + esac + fi +done + +if [[ "$failed" == "1" ]]; then + echo "::error There is an error from flawfinder. Please review the detailed results above." + exit 1 +fi diff --git a/.github/workflows/static.check.scripts/hardcoded-path.sh b/.github/workflows/static.check.scripts/hardcoded-path.sh new file mode 100644 index 0000000..38b4d67 --- /dev/null +++ b/.github/workflows/static.check.scripts/hardcoded-path.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the file containing the list of files to be checked +# + +## +# @file hardcoded-path.sh +# @brief Check prohibited hardcoded paths (/home/* for now). Originally pr-prebuild-hardcoded-path.sh +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham +# + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi + +files=$1 +failed=0 + +if [ ! -f $files ]; then + echo "::error The file $files does not exists." + exit 1 +fi + +function check(){ + which $1 + if [[ $? -ne 0 ]]; then + echo "::error The command $1 is required, but not found." + exit 1 + fi +} +check grep +check wc + +errorreport=$(mktemp) + +for file in `cat $files`; do + if [[ $file =~ \.cpp$ || $file =~ \.c$ || $file =~ \.hpp$ || $file =~ \.h$ ]]; then + count=`grep "\"\/home\/" $file | wc -l` + if [[ $VIOLATION -gt 0 ]]; then + failed=1 + errstr=`grep "\"\/home\/" $file` + echo "At file $file, found a hardcoded path:$errstr\n" >> $errorreport + fi + fi +done + +if [[ "failed" = "1" ]]; then + echo "::group::Files with hardcoded paths are:" + cat $errorreport + echo "::endgroup::" + echo "::error A source code should not have a personal path hardcoded." + exit 1 +fi diff --git a/.github/workflows/static.check.scripts/indent.sh b/.github/workflows/static.check.scripts/indent.sh new file mode 100644 index 0000000..a2a7404 --- /dev/null +++ b/.github/workflows/static.check.scripts/indent.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the file containing the list of files to be checked + +## +# @file indent.sh +# @brief Check the code formatting style with GNU indent. Orignally pr-prebuild-indent.sh +# @see https://www.gnu.org/software/indent +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham +# + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi + +files=$1 +failed=0 + +if [ ! -f $files ]; then + echo "::error The file $files does not exists." + exit 1 +fi + +which indent +if [[ $? -ne 0 ]]; then + echo "::error The indent utility is not found." + exit 1 +fi + +echo "::group::Indent check started" + +for file in `cat $files`; do + if [[ `file $file | grep "ASCII text" | wc -l` -gt 0 ]]; then + case $file in + *.c|*.cpp) + indent \ + --braces-on-if-line \ + --case-brace-indentation0 \ + --case-indentation2 \ + --braces-after-struct-decl-line \ + --line-length80 \ + --no-tabs \ + --cuddle-else \ + --dont-line-up-parentheses \ + --continuation-indentation4 \ + --honour-newlines \ + --tab-size8 \ + --indent-level2 \ + --leave-preprocessor-space \ + $file + ;; + esac + fi +done + +tmpfile=$(mktemp) +git diff > ${tmpfile} +PATCHFILESIZE=$(stat -c%s ${tmpfile}) +if [[ $PATCHFILESIZE -ne 0 ]]; then + failed=1 +fi + +echo "::endgroup::" + +if [ $failed = 1 ]; then + echo "::warning There is an indentation style error." + echo "::group::The indentation style errors are..." + cat ${tmpfile} + echo "::endgroup::" +fi diff --git a/.github/workflows/static.check.scripts/misspelling.sh b/.github/workflows/static.check.scripts/misspelling.sh new file mode 100644 index 0000000..5846584 --- /dev/null +++ b/.github/workflows/static.check.scripts/misspelling.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the file containing the list of files to be checked +# + +## +# @file misspelling.sh +# @brief Check a misspelled statement in a text document file with GNU Aspell +# Originally, pr-prebuild-misspelling.sh +# +# GNU Aspell is a Free and Open Source spell checker designed to eventually replace Ispell. +# It can either be used as a library or as an independent spell checker. Its main feature +# is that it does a superior job of suggesting possible replacements for a misspelled word +# than just about any other spell checker out there for the English language. Unlike Ispell, +# Aspell can also easily check documents in UTF-8 without having to use a special dictionary. +# +# @see http://aspell.net/ +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham +# + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi + +files=$1 +failed=0 + +if [ ! -f $files ]; then + echo "::error The file $files does not exists." + exit 1 +fi + +function check(){ + which $1 + if [[ $? -ne 0 ]]; then + echo "::error The command $1 is required, but not found." + exit 1 + fi +} +check grep +check wc +check aspell +check file +check cat + +errorreport=$(mktemp) + +for file in `cat $files`; do + # skip obsolete folder + if [[ $file =~ ^obsolete/.* ]]; then + continue + fi + # skip external folder + if [[ $file =~ ^external/.* ]]; then + continue + fi + + if [[ `file $file | grep "ASCII text" | wc -l` -gt 0 ]]; then + # in case of document file: *.md, *.txt) + case $file in + # in case of MarkDown(MD) and text file + *.md | *.txt) + echo "Checking $file" + typo_analysis_sw="aspell" + typo_analysis_rules=" list -l en " + typo_check_result=$(mktemp) + + cat $file | $typo_analysis_sw $typo_analysis_rules > ${typo_check_result} + + line_count=`cat ${typo_check_result} | wc -l` + # 9,000 is declared by heuristic method from our experiment. + if [[ $line_count -gt 9000 ]]; then + echo "$typo_analysis_sw: failed. file name: $file, There are $line_count typo(s)." + error=1 + fi + ;; + esac + fi +done + +if [[ "$error" == "1" ]]; then + echo "::error There are typo error reported by $typo_analysis_sw." + exit 1 +fi diff --git a/.github/workflows/static.check.scripts/newline.sh b/.github/workflows/static.check.scripts/newline.sh new file mode 100644 index 0000000..dae9944 --- /dev/null +++ b/.github/workflows/static.check.scripts/newline.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the file containing the list of files to be checked + +## +# @file newline.sh +# @brief Check if there is a newline issue in a text file. Originally pr-prebuild-newline.sh +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham +# + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi + +files=$1 +failed=0 + +if [ ! -f $files ]; then + echo "::error The file $files does not exists." + exit 1 +fi + +echo "::group::Newline check started" + +num=0 +report=$(mktemp) +finalreport=$(mktemp) +for file in `cat $files`; do + newline_count=0 + + # If a file is "ASCII text" type, let's check a newline rule. + if [[ `file $file | grep "ASCII text" | wc -l` -gt 0 ]]; then + num=$(( $num + 1 )) + # fetch patch content of a specified file from a commit. + git show $file> ${report}.${num}.patch + # check if the last line of a patch file includes "\ No newline....." statement. + newline_count=$(cat ${report}.${num}.patch | tail -1 | grep '^\\ No newline' | wc -l) + if [[ $newline_count == 0 ]]; then + echo "$file is ok." + else + echo "$file has newline style error." + failed=1 + echo "=========================================" >> $finalreport + echo "$file has newline style error." >> $finalreport + cat $report.$num.patch >> $finalreport + echo "\n\n" >> $finalreport + fi + fi +done + +echo "::endgroup::" + +if [ $failed = 1 ]; then + echo "::error There is a newline style error." + echo "::group::The errors are..." + cat $finalreport + echo "::endgroup::" + exit 1 +fi diff --git a/.github/workflows/static.check.scripts/nobody.sh b/.github/workflows/static.check.scripts/nobody.sh new file mode 100644 index 0000000..27fae87 --- /dev/null +++ b/.github/workflows/static.check.scripts/nobody.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the number of commits in this pull-request + +## +# @file nobody.sh +# @brief Check the commit message body. Originally, pr-prebuild-nobody.sh +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham +# + +if [ -z $1 ]; then + echo "::error The argument (the number of commits in this PR) is not given." + exit 1 +fi + +if [ -n "$1" ] && [ "$1" -eq "$1" ] 2>/dev/null; then + echo "" +else + echo ":error The first argument '$1' should be a number." + exit 1 +fi + +function check(){ + which $1 + if [[ $? -ne 0 ]]; then + echo "::error The command $1 is required, but not found." + exit 1 + fi +} +check git +check seq + +for i in $(seq 0 $[$1 - 1]); do + wordcount=`git show --pretty="format:%b" --no-notes -s HEAD~$i | wc -w` + + if [[ $wordcount -lt 8 ]]; then + git show --stat HEAD~$i + echo "::error The commit has too short commit message." + exit 1 + fi +done diff --git a/.github/workflows/static.check.scripts/prohibited-words.sh b/.github/workflows/static.check.scripts/prohibited-words.sh new file mode 100644 index 0000000..29ea1b9 --- /dev/null +++ b/.github/workflows/static.check.scripts/prohibited-words.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the file containing the list of files to be checked +# + +## +# @file prohibited-words.sh +# @brief Check if there are prohibited words in the text files. +# Originally, pr-prebuild-prohibited-words.sh +# +# It to check a prohibited word if there are unnecessary words in the source codes. +# +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham +# + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi + +files=$1 +failed=0 + +if [ ! -f $files ]; then + echo "::error The file $files does not exists." + exit 1 +fi + +function check(){ + which $1 + if [[ $? -ne 0 ]]; then + echo "::error The command $1 is required, but not found." + exit 1 + fi +} + +check git +check grep +check wc + +# Inspect all files that contributor modifed. +target_files="" +for file in `cat $files`; do + # Skip obsolete folder + if [[ $file =~ ^obsolete/.* ]]; then + continue + fi + # Skip external folder + if [[ $file =~ ^external/.* ]]; then + continue + fi + # Handle only text files among files in one commit. + if [[ `file $file | grep "ASCII text" | wc -l` -gt 0 ]]; then + case $file in + # Declare source code files to inspect a prohibited word + *.c | *.h | *.cpp | *.hpp| *.py | *.sh | *.php | *.md ) + target_files="$target_files $file" + ;; + esac + fi +done + +bad_words_sw="grep" +bad_words_list=".github/workflows/static.check.scripts/prohibited-words.txt" +bad_words_rules="--color -n -r -H -f $bad_words_list" +bad_words_log_file=$(mktemp) + +# Run a prohibited-words module in case that a PR includes text files. +if [[ -n "${target_files/[ ]*\n/}" ]]; then + if [[ ! -f $bad_words_list ]]; then + echo "::error A prohibited word list file, $bad_words_list, doesn't exist." + exit 1 + fi + + # Step 1: Run this module to filter prohibited words from a text file. + # (e.g., grep --color -f "$PROHIBITED_WORDS" $filename) + $bad_words_sw $bad_words_rules $target_files > ${bad_words_log_file} + + # Step 2: Display the execution result for debugging in case of a failure + cat ${bad_words_log_file} + + # Step 3: Count prohibited words from variable result_content + result_count=$(cat ${bad_word_log_file} | grep -c '^' ) + + # Step 4: change a value of the check result + if [[ $result_count -gt 0 ]]; then + echo "::error There are prohibited words in this PR (counted $result_count)." + exit 1 + else + echo "There is no prohibited word found in this PR." + fi +fi diff --git a/.github/workflows/static.check.scripts/prohibited-words.txt b/.github/workflows/static.check.scripts/prohibited-words.txt new file mode 100644 index 0000000..9771f6a --- /dev/null +++ b/.github/workflows/static.check.scripts/prohibited-words.txt @@ -0,0 +1,3 @@ +samsung.net +FUCK +file diff --git a/.github/workflows/static.check.scripts/pylint.sh b/.github/workflows/static.check.scripts/pylint.sh new file mode 100644 index 0000000..820ff81 --- /dev/null +++ b/.github/workflows/static.check.scripts/pylint.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the file containing the list of files to be checked + +## +# @file pylint.sh +# @brief Check the code formatting style with GNU pylint. Originally pr-prebuild-pylint.sh +# @see https://www.pylint.org/ +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham +# + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi + +files=$1 +failed=0 + +if [ ! -f $files ]; then + echo "::error The file $files does not exists." + exit 1 +fi + +which pylint +if [[ $? -ne 0 ]]; then + echo "::error The pylint utility is not found." + exit 1 +fi + +echo "::group::Pylint check started" + +pylint --generate-rcfile > ~/.pylintrc +result=$(mktemp) +errlog=$(mktemp) + +for file in `cat $files`; do + if [[ $file =~ ^obsolete/.* ]]; then + continue + fi + if [[ $file =~ ^external/.* ]]; then + continue + fi + + if [[ `file $file | grep "ASCII text" | wc -l` -gt 0 ]]; then + case $file in + *.py) + pylint --reports=y $file > $result + line_count=$((`cat $result | grep W: | wc -l` + \ + `cat $result | grep C: | wc -l` + \ + `cat $result | grep E: | wc -l` + \ + `cat $result | grep R: | wc -l`)) + if [[ $line_count -gt 0 ]]; then + failed=1 + echo "======================================" >> $errlog + echo "pylint error from $file" >> $errlog + cat $result >> $errlog + echo "\n\n" >> $errlog + fi + ;; + esac + fi +done + +echo "::endgroup::" + +if [ $failed = 1 ]; then + echo "::error There is a doxygen tag missing or incorrect." + echo "::group::The pylint errors are..." + cat $errlog + echo "::endgroup::" + exit 1 +fi diff --git a/.github/workflows/static.check.scripts/rpm-spec.sh b/.github/workflows/static.check.scripts/rpm-spec.sh new file mode 100644 index 0000000..c07ca08 --- /dev/null +++ b/.github/workflows/static.check.scripts/rpm-spec.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the file containing the list of files to be checked +# Argument 2 ($2): the max number of errors to be tolerated + +## +# @file rpm-spec.sh +# @brief Check the spec file (packaging/*.spec) with rpmlint. Originally pr-prebuild-rpm-spec.sh +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham +# + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi + +tolerable=0 +if [ -z $2 ]; then + echo "Tolarable rpmlint errors = 0" +else + tolerable=$2 + echo "Tolarable rpmlint errors = $2" +fi + +files=$1 +failed=0 + +if [ ! -f $files ]; then + echo "::error The file $files does not exists." + exit 1 +fi + +spec_modified="false" +specfiles="" +resultfile=$(mktemp) + +for file in `cat $files`; do + if [[ $file =~ ^obsolete/.* ]]; then + continue + fi + if [[ $file =~ ^external/.* ]]; then + continue + fi + # Handle only spec file in case that there are lots of files in one commit. + if [[ `file $file | grep "ASCII text" | wc -l` -gt 0 ]]; then + case $file in + *.spec) + specfiles+=" ${file}" + echo "A .spec file found: $file" + spec_modified="true" + break + ;; + esac + fi +done + +if [[ spec_modified == "true" ]]; then + rpmlint $specfiles | aha --line-fix > $resultfile + echo "::group::rpmlint result" + cat $resultfile + echo "::ungroup::" + + count=`grep "[0-9]* errors, [0-9]* warnings." $resultfile | grep -o "[0-9]* errors" | grep -o "[0-9]*"` + if [ $count -gt $tolerable ]; then + echo "::error RPMLINT reports more errors ($count) than tolerated ($tolerable)." + exit 1 + fi +else + echo "There is no .spec file modified in this PR. RPMLINT is not executed." +fi diff --git a/.github/workflows/static.check.scripts/shellcheck.sh b/.github/workflows/static.check.scripts/shellcheck.sh new file mode 100644 index 0000000..f36ac51 --- /dev/null +++ b/.github/workflows/static.check.scripts/shellcheck.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the file containing the list of files to be checked +# + +## +# @file shellcheck.sh +# @brief This module is a static analysis tool for shell scripts such as sh, bash. +# Originally, pr-prebuild-shellcheck.sh +# +# It is mainly focused on handling typical beginner and intermediate level syntax errors +# and pitfalls where the shell just gives a cryptic error message or strange behavior, +# but it also reports on a few more advanced issues where corner cases can cause delayed +# failures. +# +# @see https://www.shellcheck.net/ +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham +# + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi + +files=$1 +failed=0 + +if [ ! -f $files ]; then + echo "::error The file $files does not exists." + exit 1 +fi + +function check(){ + which $1 + if [[ $? -ne 0 ]]; then + echo "::error The command $1 is required, but not found." + exit 1 + fi +} +# Check if required commands are installed by server administrator +check cat +check shellcheck +check file +check grep +check wc + +shell_syntax_analysis_sw="shellcheck" +shell_syntax_analysis_rules="-s bash" +shell_syntax_check_result=$(mktemp) + +# Inspect all files that contributor modifed. +for file in `cat $files`; do + # Skip obsolete folder + if [[ $file =~ ^obsolete/.* ]]; then + continue + fi + # Skip external folder + if [[ $file =~ ^external/.* ]]; then + continue + fi + # Handle only text files in case that there are lots of files in one commit. + echo "[DEBUG] file name is ($file)." + if [[ `file $file | grep "shell script" | wc -l` -gt 0 ]]; then + case $file in + # In case of .sh or .bash file + *.sh | *.bash) + echo "($file) file is a shell script file with the 'shell script' text format." + + cat $file | $shell_syntax_analysis_sw $shell_syntax_analysis_rules > ${shell_syntax_check_result} + line_count=`cat ${shell_syntax_check_result} | wc -l` + + echo "::group::shellcheck result of $file" + cat ${shell_syntax_check_result} + echo "::endgroup::" + + # TODO: 9,000 is declared by heuristic method from our experiment. + if [[ $line_count -gt 9000 ]]; then + echo "$shell_syntax_analysis_sw: failed. file name: $file There are $line_count lines." + failed=1 + else + echo "$shell_syntax_analysis_sw: passed. file name: $file There are $line_count lines." + fi + ;; + esac + fi +done + +if [[ "$failed" == "1" ]]; then + echo "::error These is a enough number of errors in a shell file with shellcheck. Please refer to the log above". + exit 1 +fi diff --git a/.github/workflows/static.check.scripts/signed-off-by.sh b/.github/workflows/static.check.scripts/signed-off-by.sh new file mode 100644 index 0000000..e1aaa58 --- /dev/null +++ b/.github/workflows/static.check.scripts/signed-off-by.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the number of commits in this PR +# + +## +# @file signed-off-by.sh +# @brief Check if contributor write Sigend-off-by message. Originally, pr-prebuild-signed-off-by.sh +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Sewon oh +# @author MyungJoo Ham +# + +if [ -z $1 ]; then + echo "::error The argument (the number of commits in this PR) is not given." + exit 1 +fi + +if [ -n "$1" ] && [ "$1" -eq "$1" ] 2>/dev/null; then + echo "" +else + echo ":error The first argument '$1' should be a number." + exit 1 +fi + +function check(){ + which $1 + if [[ $? -ne 0 ]]; then + echo "::error The command $1 is required, but not found." + exit 1 + fi +} + +# Check if server administrator install required commands +check git + +for i in $(seq 0 $[$1 - 1]); do + count=`git show --no-notes -s HEAD~$i | grep "Signed-off-by: [^ ].*[^ ].*<[^ ].*@[^ ].*>" | wc -l` + if [[ $count -lt 1 ]]; then + git show --stat HEAD~$i + echo "============================================" + echo "" + echo "::error This commit does not have a proper Signed-off-by. Refer to https://ltsi.linuxfoundation.org/software/signed-off-process/ for some information about Signed-off-by. We require Signed-off-by: NAME signed by indivisual contributors." + exit 1 + else + id=`git show --pretty="format:%h" --no-notes -s HEAD~$i` + echo "Commit $id has proper a signed-off-by string." + fi +done diff --git a/.github/workflows/static.check.scripts/sloccount.sh b/.github/workflows/static.check.scripts/sloccount.sh new file mode 100644 index 0000000..3d9aeb4 --- /dev/null +++ b/.github/workflows/static.check.scripts/sloccount.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the file containing the list of files to be checked +# + +## +# @file sloccount.sh +# @brief Count physical source lines of code (SLOC). Originally, pr-prebuild-sloccount.sh +# +# It is a set of tools for counting physical Source Lines of Code (SLOC) in a large +# number of languages of a potentially large set of programs. +# +# @see https://packages.ubuntu.com/search?keywords=sloccount +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham +# + +if [ -z $1 ]; then + echo "::error The argument (file path) is not given." + exit 1 +fi + +files=$1 +failed=0 + +if [ ! -f $files ]; then + echo "::error The file $files does not exists." + exit 1 +fi + +function check(){ + which $1 + if [[ $? -ne 0 ]]; then + echo "::error The command $1 is required, but not found." + exit 1 + fi +} + +# Check if server administrator install required commands +check sloccount +check git +check file +check mkdir +check grep + +sloc_analysis_sw="sloccount" +sloc_data_folder=$(mktemp -d) +sloc_analysis_rules="--wide --multiproject --datadir $sloc_data_folder" +sloc_check_result=$(mktemp) + +# Inspect all files that contributor modifed. +for file in `cat $files`; do + # skip obsolete folder + if [[ $file =~ ^obsolete/.* ]]; then + continue + fi + # skip external folder + if [[ $file =~ ^external/.* ]]; then + continue + fi + # Handle only text files in case that there are lots of files in one commit. + if [[ `file $file | grep "ASCII text" | wc -l` -gt 0 ]]; then + # Run a SLOCCount module in case that a PR includes source codes. + case $file in + *.c | *.cpp | *.py | *.sh | *.php ) + sloc_target_dir=${SRC_PATH} + + # Run this module + echo "::group::sloccount result" + $sloc_analysis_sw $sloc_analysis_rules . > ${sloc_check_result} + echo "::endgroup::" + run_result=$? + if [[ $run_result -eq 0 ]]; then + echo "" + exit 0 + else + echo "::error Sloccount failed." + exit 1 + fi + ;; + esac + fi +done diff --git a/.github/workflows/static.check.scripts/timestamp.sh b/.github/workflows/static.check.scripts/timestamp.sh new file mode 100644 index 0000000..d4a7718 --- /dev/null +++ b/.github/workflows/static.check.scripts/timestamp.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +## +# Imported from TAOS-CI +# Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved. +# Modified for Github-Action in 2024. +# +# Argument 1 ($1): the number of commits in this pull-request + +## +# @file timestamp.sh +# @brief Check the timestamp of the commit. Originally, pr-prebuild-timestamp.sh +# @see https://github.com/nnstreamer/TAOS-CI +# @see https://github.com/nnstreamer/nnstreamer +# @author Geunsik Lim +# @author MyungJoo Ham +# + +if [ -z $1 ]; then + echo "::error The argument (the number of commits in this PR) is not given." + exit 1 +fi + +if [ -n "$1" ] && [ "$1" -eq "$1" ] 2>/dev/null; then + echo "" +else + echo ":error The first argument '$1' should be a number." + exit 1 +fi + +function check(){ + which $1 + if [[ $? -ne 0 ]]; then + echo "::error The command $1 is required, but not found." + exit 1 + fi +} +check git +check seq + +NOW=`date +%s` +NOW_READ=`date` + +for i in $(seq 0 $[$1 - 1]); do + timestamp=`git show --pretty="%ct" --no-notes -s HEAD~$i` + timestamp_read=`git show --pretty="%cD" --no-notes -s HEAD~$i` + timestamp_3min=$(( $timestamp - 180 )) + # allow 3 minutes of clock drift. + + if [[ $timestamp_3min -gt $NOW ]]; then + git show --stat HEAD~$i + echo "::error The commit has timestamp error, coming from the future: $timestamp_read (now: $NOW_READ)." + exit 1 + fi +done diff --git a/.github/workflows/static.check.yml b/.github/workflows/static.check.yml new file mode 100644 index 0000000..762ee1e --- /dev/null +++ b/.github/workflows/static.check.yml @@ -0,0 +1,158 @@ +name: Static checkers and verifiers + +# "Pre-build" Scripts from TAOS-CI +# @todo Make this "reusable workflow" and publish for all projects. + +## Common variables and files +# - changed_file_list in GITHUB_ENV: the list of files updated in this pull-request. + +on: + pull_request: + branches: [ main ] + +jobs: + simple_script_checkers: + runs-on: ubuntu-latest + name: Static checks + steps: + - name: Preparing step 1... Checking out + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: -${{ github.event.pull_request.commits }} + - name: Preparing step 2... Installing packages + run: sudo apt-get update && sudo apt-get install clang-format git grep gawk exuberant-ctags indent pylint rpmlint aha cppcheck aspell doxygen sloccount shellcheck flawfinder + - name: Preparing step 3... Identify changed files + run: | + tmpfile_pre=$(mktemp) + tmpfile=$(mktemp) + git show --pretty="format:" --name-only --diff-filter=AMRC ${{ github.event.pull_request.head.sha}} -${{ github.event.pull_request.commits }} > ${tmpfile_pre} + + ####### Screen out deleted files from the file list!!! + + echo "::group::The list of changed files" + for file in `cat ${tmpfile_pre}`; do + if [[ -f $file ]]; then + echo "$file" + echo "$file" >> $tmpfile + else + echo "$file is deleted." + fi + done + echo "::endgroup::" + echo "changed_file_list=${tmpfile}" >> "$GITHUB_ENV" + - name: /Checker/ clang-format for .cc/.hh/.hpp/.cpp files + # Originally from "pr-prebuild-clang" + # Need "clang-format" + run: | + echo "Check .clang-format file" + if [ ! -f ".clang-format" ]; then + echo "::error .clang-format file not found" + exit 1 + fi + for file in `cat $changed_file_list`; do + echo "Checking $file" + if [[ "$file" =~ .*\.hh$ ]] || [[ "$file" =~ .*\.hpp ]] || [[ "$file" =~ .*\.cc$ ]] || [[ "$file" =~ .*\.cpp ]]; then + clang-format -i ${file} + fi + done + git diff -- *.cc *.hh *.hpp *.cpp > .ci.clang-format.patch + SIZE=$(stat -c%s .ci.clang-format.patch) + if [[ $SIZE -ne 0 ]]; then + echo "::group::The clang-format complaints..." + cat .ci.clang-format.patch + echo "::endgroup::" + echo "::error clang-format has found style errors in C++ files." + exit 1 + fi + echo "clang-format shows no style errors." + - name: /Checker/ File size check + # Originally from "pr-prebuild-file-size" + run: | + for file in `cat $changed_file_list`; do + echo "Checking $file" + FILESIZE=$(stat -c%s "$file") + FILESIZE_NUM=`echo $FILESIZE | sed ':a;s/\B[0-9]\{3\}\>/,&/;ta'` + if [[ $FILESIZE -gt $[ 5*1024*1024 ] ]]; then + echo "::error $file is too large: $FILESIZE > 5MiB" + exit 1 + fi + done + - name: /Checker/ Doxygen tag check + # Originally from "pr-prebuild-doxygen-tag" + # Need "grep" + run: | + bash .github/workflows/static.check.scripts/doxygen-tag.sh $changed_file_list 1 + - name: /Checker/ Indent check + # Originally from "pr-prebuild-indent" + # Need "indent" + run: | + bash .github/workflows/static.check.scripts/indent.sh $changed_file_list + - name: /Checker/ Pylint + # Originally from "pr-prebuild-pylint" + # Need "pylint" + run: | + bash .github/workflows/static.check.scripts/pylint.sh $changed_file_list + - name: /Checker/ Incorrect newlines + # Originally from "pr-prebuild-newline" + run: | + bash .github/workflows/static.check.scripts/newline.sh $changed_file_list + - name: /Checker/ RPM spec lint + # Originally from "pr-prebuild-rpm-spec" + # Need "rpmlint", "aha" + # Tolerated errors: 40 (make it 0 someday!!!) + run: | + bash .github/workflows/static.check.scripts/rpm-spec.sh $changed_file_list 40 + - name: /Checker/ CPPCheck errors + # Originally from "pr-prebuild-cppcheck" + # Need "cppcheck" + run: | + bash .github/workflows/static.check.scripts/cppcheck.sh $changed_file_list 0 + - name: /Checker/ Commit without proper message + # Originally from "pr-prebuild-nobody" + run: | + bash .github/workflows/static.check.scripts/nobody.sh ${{ github.event.pull_request.commits }} + - name: /Checker/ Timestamp from the future + # Originally from "pr-prebuild-timestamp" + run: | + bash .github/workflows/static.check.scripts/timestamp.sh ${{ github.event.pull_request.commits }} + - name: /Checker/ Executable bits in source code files + # Originally from "pr-prebuild-executable" + run: | + bash .github/workflows/static.check.scripts/executable.sh $changed_file_list + - name: /Checker/ Hardcoded paths + # Originally from "pr-prebuild-hardcoded-path" + run: | + bash .github/workflows/static.check.scripts/hardcoded-path.sh $changed_file_list + - name: /Checker/ Misspelling + # Originally from "pr-prebuild-misspelling" + run: | + bash .github/workflows/static.check.scripts/misspelling.sh $changed_file_list + - name: /Checker/ Doxygen build test + # Originally from "pr-prebuild-doxygen-build" + run: | + bash .github/workflows/static.check.scripts/doxygen-build.sh $changed_file_list + - name: /Checker/ sloccount limit + # Originally from "pr-prebuild-sloccount" + run: | + bash .github/workflows/static.check.scripts/sloccount.sh $changed_file_list + - name: /Checker/ prohibited words + # Originally from "pr-prebuild-prohibited-words" + run: | + bash .github/workflows/static.check.scripts/prohibited-words.sh $changed_file_list + - name: /Checker/ signed-off-by required + # Originally from "pr-prebuild-signed-off-by" + run: | + bash .github/workflows/static.check.scripts/signed-off-by.sh ${{ github.event.pull_request.commits }} + - name: /Checker/ shellcheck for shell scripts + # Originally from "pr-prebuild-shellcheck" + run: | + bash .github/workflows/static.check.scripts/shellcheck.sh $changed_file_list + - name: /Checker/ flawfinder for C/C++ files + # Originally from "pr-prebuild-flawfinder" + run: | + bash .github/workflows/static.check.scripts/flawfinder.sh $changed_file_list 1 + - name: /Checker/ covertity for C/C++ files + # Originally from "pr-prebuild-coverity" + run: | + #bash .github/workflows/static.check.scripts/coverity.sh $changed_file_list