From d8dfb16491ef689ae91ac536b77fd9c521b555f7 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 15 May 2019 21:55:44 +0200 Subject: [PATCH 1/5] Add a common utility function which makes debugging test failures easier and update one of the test files as an example. This makes debugging failures easier since we now include stdout, stderr and exit code for each bash command we run for which we save output to a variable. Previously we didn't do that so if an assertion failed you had no clue what was going on. --- cli/test_pack_python3.bats | 12 +++++------ test_helpers/common.sh | 42 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 7 deletions(-) create mode 100644 test_helpers/common.sh diff --git a/cli/test_pack_python3.bats b/cli/test_pack_python3.bats index f2ffa07..d2660a1 100644 --- a/cli/test_pack_python3.bats +++ b/cli/test_pack_python3.bats @@ -1,5 +1,6 @@ load '../test_helpers/bats-support/load' load '../test_helpers/bats-assert/load' +load '../test_helpers/common' setup() { if [[ ! -d /opt/stackstorm/packs/examples ]]; then @@ -42,7 +43,7 @@ skip_tests_if_python3_is_not_available_or_if_already_running_under_python3() { @test "packs.setup_virtualenv without python3 flags works and defaults to Python 2" { skip_tests_if_python3_is_not_available_or_if_already_running_under_python3 - SETUP_VENV_RESULTS=$(st2 run packs.setup_virtualenv packs=examples -j) + SETUP_VENV_RESULTS=$(run_command_and_log_output st2 run packs.setup_virtualenv packs=examples -j) run eval "echo '$SETUP_VENV_RESULTS' | jq -r '.result.result'" assert_success @@ -53,9 +54,6 @@ skip_tests_if_python3_is_not_available_or_if_already_running_under_python3() { assert_output "succeeded" - run st2-register-content --register-pack /opt/stackstorm/packs/examples/ --register-all - assert_success - run /opt/stackstorm/virtualenvs/examples/bin/python --version assert_output --partial "Python 2.7" @@ -66,7 +64,7 @@ skip_tests_if_python3_is_not_available_or_if_already_running_under_python3() { @test "packs.setup_virtualenv with python3 flag works" { skip_tests_if_python3_is_not_available_or_if_already_running_under_python3 - SETUP_VENV_RESULTS=$(st2 run packs.setup_virtualenv packs=examples python3=true -j) + SETUP_VENV_RESULTS=$(run_command_and_log_output st2 run packs.setup_virtualenv packs=examples python3=true -j) run eval "echo '$SETUP_VENV_RESULTS' | jq -r '.result.result'" assert_success @@ -81,7 +79,7 @@ skip_tests_if_python3_is_not_available_or_if_already_running_under_python3() { assert_output --partial "Python 3." - RESULT=$(st2 run examples.python_runner_print_python_version -j) + RESULT=$(run_command_and_log_output st2 run examples.python_runner_print_python_version -j) assert_success run eval "echo '$RESULT' | jq -r '.result.stdout'" @@ -108,7 +106,7 @@ skip_tests_if_python3_is_not_available_or_if_already_running_under_python3() { run st2 pack install python3_test --python3 -j assert_success - RESULT=$(st2 run python3_test.test_stdlib_import -j) + RESULT=$(run_command_and_log_output st2 run python3_test.test_stdlib_import -j) assert_success run eval "echo '$RESULT' | jq -r '.result.result'" diff --git a/test_helpers/common.sh b/test_helpers/common.sh new file mode 100644 index 0000000..d8f1164 --- /dev/null +++ b/test_helpers/common.sh @@ -0,0 +1,42 @@ +# Copyright 2019 Extreme Networks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Common bash utility functions used by test code + +run_command_and_log_output() { + # Utility function which runs a bash command and logs the command output (stdout) and exit + # code. + # + # This comes handy in scenarios where you want to save command stdout in a variable, but you + # also want to output the command stdout to make troubleshooting / debugging n test failures + # easier. + # + # Example usage: + # + # RESULT=$(run_command_and_log_output st2 run pack install packs=examples) + # + # 1. Run the command and capture the outputs + eval "$({ stderr=$({ stdout=$($@); exit_code=$?; } 2>&1; declare -p stdout exit_code >&2); declare -p stderr ; } 2>&1)" + + # 2. Log the output to stderr + >&2 echo "==========" + >&2 echo "Ran command: ${@}" + >&2 echo "Stdout: ${stdout}" + >&2 echo "stderr: ${stderr}" + >&2 echo "Exit code: ${exit_code}" + >&2 echo "==========" + + # 3. Return original command value + echo ${stdout} +} From 122ba6dc55119fa8f3d4e8ecb252bcfbd18d2e24 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 15 May 2019 22:04:15 +0200 Subject: [PATCH 2/5] Use consistent line breaks. --- cli/test_pack_python3.bats | 2 -- 1 file changed, 2 deletions(-) diff --git a/cli/test_pack_python3.bats b/cli/test_pack_python3.bats index d2660a1..b594db0 100644 --- a/cli/test_pack_python3.bats +++ b/cli/test_pack_python3.bats @@ -46,12 +46,10 @@ skip_tests_if_python3_is_not_available_or_if_already_running_under_python3() { SETUP_VENV_RESULTS=$(run_command_and_log_output st2 run packs.setup_virtualenv packs=examples -j) run eval "echo '$SETUP_VENV_RESULTS' | jq -r '.result.result'" assert_success - assert_output "Successfully set up virtualenv for the following packs: examples" run eval "echo '$SETUP_VENV_RESULTS' | jq -r '.status'" assert_success - assert_output "succeeded" run /opt/stackstorm/virtualenvs/examples/bin/python --version From 1f96aea7a54fc686ec4442b74c5d25da5ce4c348 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Wed, 15 May 2019 22:21:37 +0200 Subject: [PATCH 3/5] Pull in improvement from master. --- cli/test_pack_python3.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/test_pack_python3.bats b/cli/test_pack_python3.bats index b594db0..c35593d 100644 --- a/cli/test_pack_python3.bats +++ b/cli/test_pack_python3.bats @@ -95,7 +95,7 @@ skip_tests_if_python3_is_not_available_or_if_already_running_under_python3() { run eval "echo '$RESULT' | jq -r '.result.stdout'" assert_success - assert_output --regexp ".*PYTHONPATH: /usr/(local/)?lib/python3.*" + assert_output --regexp ".*PYTHONPATH: .*/usr/(local/)?lib/python3.*" } @test "python3 imports work correctly" { From 0c1ef09793f7c64e9744dc2af1edf19fb60e4775 Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 15 May 2019 16:36:26 -0700 Subject: [PATCH 4/5] Add function that echos stderr --- test_helpers/common.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test_helpers/common.sh b/test_helpers/common.sh index d8f1164..fc66f2e 100644 --- a/test_helpers/common.sh +++ b/test_helpers/common.sh @@ -40,3 +40,30 @@ run_command_and_log_output() { # 3. Return original command value echo ${stdout} } + +run_command_and_log_output_get_stderr() { + # Utility function which runs a bash command and logs the command output (stdout) and exit + # code. + # + # This comes handy in scenarios where you want to save command stdout in a variable, but you + # also want to output the command stdout to make troubleshooting / debugging n test failures + # easier. + # + # Example usage: + # + # RESULT=$(run_command_and_log_output_get_stderr st2 run pack install packs=examples) + # + # 1. Run the command and capture the outputs + eval "$({ stderr=$({ stdout=$($@); exit_code=$?; } 2>&1; declare -p stdout exit_code >&2); declare -p stderr ; } 2>&1)" + + # 2. Log the output to stderr + >&2 echo "==========" + >&2 echo "Ran command: ${@}" + >&2 echo "Stdout: ${stdout}" + >&2 echo "stderr: ${stderr}" + >&2 echo "Exit code: ${exit_code}" + >&2 echo "==========" + + # 3. Return original command error + echo ${stderr} +} From 13da9f6d367083e7a6cfe9ceaffd4a4d915fa265 Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 15 May 2019 16:38:54 -0700 Subject: [PATCH 5/5] Fix some typos and Bashisms --- test_helpers/common.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test_helpers/common.sh b/test_helpers/common.sh index fc66f2e..05b72e1 100644 --- a/test_helpers/common.sh +++ b/test_helpers/common.sh @@ -18,8 +18,8 @@ run_command_and_log_output() { # Utility function which runs a bash command and logs the command output (stdout) and exit # code. # - # This comes handy in scenarios where you want to save command stdout in a variable, but you - # also want to output the command stdout to make troubleshooting / debugging n test failures + # This comes in handy in scenarios where you want to save command stdout in a variable, but you + # also want to output the command stdout to make troubleshooting / debugging in test failures # easier. # # Example usage: @@ -32,21 +32,21 @@ run_command_and_log_output() { # 2. Log the output to stderr >&2 echo "==========" >&2 echo "Ran command: ${@}" - >&2 echo "Stdout: ${stdout}" + >&2 echo "stdout: ${stdout}" >&2 echo "stderr: ${stderr}" - >&2 echo "Exit code: ${exit_code}" + >&2 echo "exit code: ${exit_code}" >&2 echo "==========" # 3. Return original command value - echo ${stdout} + echo "${stdout}" } run_command_and_log_output_get_stderr() { # Utility function which runs a bash command and logs the command output (stdout) and exit # code. # - # This comes handy in scenarios where you want to save command stdout in a variable, but you - # also want to output the command stdout to make troubleshooting / debugging n test failures + # This comes in handy in scenarios where you want to save command stdout in a variable, but you + # also want to output the command stdout to make troubleshooting / debugging in test failures # easier. # # Example usage: @@ -59,11 +59,11 @@ run_command_and_log_output_get_stderr() { # 2. Log the output to stderr >&2 echo "==========" >&2 echo "Ran command: ${@}" - >&2 echo "Stdout: ${stdout}" + >&2 echo "stdout: ${stdout}" >&2 echo "stderr: ${stderr}" - >&2 echo "Exit code: ${exit_code}" + >&2 echo "exit code: ${exit_code}" >&2 echo "==========" # 3. Return original command error - echo ${stderr} + echo "${stderr}" }