Skip to content

Commit

Permalink
More test fixes, still WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
@seberm
seberm committed Dec 12, 2024
1 parent a3f7409 commit f4427d2
Showing 1 changed file with 69 additions and 54 deletions.
123 changes: 69 additions & 54 deletions tests/report/reportportal/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,20 @@ ARTIFACTS=$TMT_REPORT_ARTIFACTS_URL

PLAN_PREFIX='/plan'
PLAN_STATUS='FAILED'
TEST_PREFIX='/test'
#TEST_PREFIX='/test'
TEST_PREFIX=''

# TODO: Subresults for beakerlib
declare -A test=([1,'uuid']="" [1,'name']='/bad' [1,'status']='FAILED'
[2,'uuid']="" [2,'name']='/good' [2,'status']='PASSED'
[3,'uuid']="" [3,'name']='/subresults' [3,'status']='FAILED'
[4,'uuid']="" [4,'name']='/subresults-restraint/subtest-restraint/good' [4,'status']='PASSED'
[5,'uuid']="" [5,'name']='/subresults-restraint/subtest-restraint/fail' [5,'status']='FAILED'
[6,'uuid']="" [6,'name']='/subresults-restraint/subtest-restraint/weird' [6,'status']='FAILED'
[7,'uuid']="" [7,'name']='/weird' [7,'status']='FAILED')
declare -A test=([1,'uuid']="" [1,'name']='/test/bad' [1,'status']='FAILED'
[2,'uuid']="" [2,'name']='/test/good' [2,'status']='PASSED'
[3,'uuid']="" [3,'name']='/test/subresults' [3,'status']='FAILED'
[4,'uuid']="" [4,'name']='/subtest/good' [4,'status']='PASSED'
[5,'uuid']="" [5,'name']='/subtest/fail' [5,'status']='FAILED'
[6,'uuid']="" [6,'name']='/subtest/weird' [6,'status']='FAILED'
[7,'uuid']="" [7,'name']='/test/subresults-restraint/subtest-restraint/good' [7,'status']='PASSED'
[8,'uuid']="" [8,'name']='/test/subresults-restraint/subtest-restraint/fail' [8,'status']='FAILED'
[9,'uuid']="" [9,'name']='/test/subresults-restraint/subtest-restraint/weird' [9,'status']='FAILED'
[10,'uuid']="" [10,'name']='/test/weird' [10,'status']='FAILED')
DIV="|"


Expand Down Expand Up @@ -62,15 +66,16 @@ function identify_suite(){
# Read and verify reported test names and uuids from $rlRun_LOG
#
# GLOBALS:
# >> $test_uuid[1..7], $test_fullname[1..7]
# >> $test_uuid[1..10], $test_fullname[1..10]
function identify_tests(){
rlLog "Verify and get test data"

for i in {1..7}; do
for i in {1..10}; do
test_fullname[$i]=${TEST_PREFIX}${test[$i,'name']}

rlAssertGrep " test: ${test_fullname[$i]}" $rlRun_LOG
test_uuid[$i]=$(rlRun "grep -m$i -A1 ' test:' $rlRun_LOG | tail -n1 | awk '{print \$NF}' ")
rlAssertGrep " \(sub-\)\?test: ${test_fullname[$i]}" $rlRun_LOG
rlAssertGrep "test: ${test_fullname[$i]}" $rlRun_LOG
test_uuid[$i]=$(rlRun "grep -m$i -A1 ' \(sub-\)\?test:' $rlRun_LOG | tail -n1 | awk '{print \$NF}' ")
rlAssertNotEquals "Assert the test$i UUID is not empty" "${test_uuid[$i]}" ""
test[$i,'uuid']=${test_uuid[$i]}
done
Expand Down Expand Up @@ -118,7 +123,7 @@ rlJournalStart
rlRun -s "tmt run --id $run --verbose --all" 2
rlAssertGrep "url: https?://.*\.redhat\.com/ui/#${PROJECT}/launches/all/[0-9]+" $rlRun_LOG -Eq
identify_launch # >> $launch_uuid, $launch_id
identify_tests # >> $test_uuid[1..7], $test_fullname[1..7]
identify_tests # >> $test_uuid[1..10], $test_fullname[1..10]
rlPhaseEnd


Expand Down Expand Up @@ -154,7 +159,7 @@ rlJournalStart
done
rm tmp_attributes*

for i in {1..7}; do
for i in {1..10}; do
echo ""
test_name[$i]=${test[$i,'name']}
test_name=${test_name[$i]}
Expand All @@ -175,38 +180,40 @@ rlJournalStart
test_case_id=$(yq -r ".\"$test_name\".id" test.fmf)
[[ $test_case_id != null ]] && rlAssertEquals "Assert the testCaseId is correct" "$(echo $response | jq -r '.testCaseId')" "$test_case_id"

for jq_element in attributes parameters; do

# Check the test attributes only for parent test items, do not check them for subresults
if [[ ! "$test_name" =~ ^/subtest/ ]]; then
# Check all the common test attributes/parameters
[[ $jq_element == attributes ]] && fmf_label="context"
[[ $jq_element == parameters ]] && fmf_label="environment"
rlLogInfo "Check the $jq_element for test $test_name ($fmf_label)"
echo "$response" | jq -r ".$jq_element" > tmp_attributes.json || rlFail "$jq_element listing into tmp_attributes.json"
length=$(yq -r ".$fmf_label | length" plan.fmf)
for ((item_index=0; item_index<$length; item_index++ )); do
key=$(yq -r ".$fmf_label | keys | .[$item_index]" plan.fmf)
value=$(yq -r ".$fmf_label.$key" plan.fmf)
rlAssertGrep "$key" tmp_attributes.json -A1 > tmp_attributes_selection
rlAssertGrep "$value" tmp_attributes_selection
done

# Check the rarities in the test attributes/parameters
if [[ $jq_element == attributes ]]; then
key="contact"
value="$(yq -r ".\"$test_name\".$key" test.fmf)"
if [[ $value != null ]]; then
for jq_element in attributes parameters; do
[[ $jq_element == attributes ]] && fmf_label="context"
[[ $jq_element == parameters ]] && fmf_label="environment"
rlLogInfo "Check the $jq_element for test $test_name ($fmf_label)"
echo "$response" | jq -r ".$jq_element" > tmp_attributes.json || rlFail "$jq_element listing into tmp_attributes.json"
length=$(yq -r ".$fmf_label | length" plan.fmf)
for ((item_index=0; item_index<$length; item_index++ )); do
key=$(yq -r ".$fmf_label | keys | .[$item_index]" plan.fmf)
value=$(yq -r ".$fmf_label.$key" plan.fmf)
rlAssertGrep "$key" tmp_attributes.json -A1 > tmp_attributes_selection
rlAssertGrep "$value" tmp_attributes_selection
else
done

# Check the rarities in the test attributes/parameters
if [[ $jq_element == attributes ]]; then
key="contact"
value="$(yq -r ".\"$test_name\".$key" test.fmf)"
if [[ $value != null ]]; then
rlAssertGrep "$key" tmp_attributes.json -A1 > tmp_attributes_selection
rlAssertGrep "$value" tmp_attributes_selection
else
rlAssertNotGrep "$key" tmp_attributes.json
fi
elif [[ $jq_element == parameters ]]; then
key="TMT_TREE"
rlAssertNotGrep "$key" tmp_attributes.json
fi
elif [[ $jq_element == parameters ]]; then
key="TMT_TREE"
rlAssertNotGrep "$key" tmp_attributes.json
fi

rm tmp_attributes*
done
rm tmp_attributes*
done
fi

# REST API | [GET] log-controller | parent_id
rlLogInfo "Get all logs from the test $test_name"
Expand All @@ -217,7 +224,7 @@ rlJournalStart
rlAssertEquals "Assert the level of the info log is correct" "$(echo $response | jq -r .content[$content_index].level)" "${level[$content_index]}"

# Check the log message is correct (except the weird parent test)
if [[ $i -ne 7 ]]; then
if [[ $i -ne 10 ]]; then
log_message=$(yq -r ".\"$test_name\".test" test.fmf | awk -F '"' '{print $2}' )
rlAssertEquals "Assert the message of the info log is correct" "$(echo $response | jq -r .content[$content_index].message)" "$log_message"
fi
Expand All @@ -238,7 +245,7 @@ rlJournalStart
rlRun -s "tmt run --verbose --all report --how reportportal --launch-per-plan --launch '$launch_name' " 2 "" 1>/dev/null
identify_launch # >> $launch_uuid, $launch_id
rlAssertNotGrep "suite:" $rlRun_LOG
identify_tests # >> $test_uuid[1..7], $test_fullname[1..7]
identify_tests # >> $test_uuid[1..10], $test_fullname[1..10]

# REST API | [GET] test-item-controller | launch_id
rlLogInfo "Get info about all launch items"
Expand Down Expand Up @@ -271,7 +278,7 @@ rlJournalStart
rlRun -s "tmt run --verbose --all report --how reportportal --suite-per-plan --launch '$launch_name' --launch-description '$launch_description'" 2 "" 1>/dev/null
identify_launch # >> $launch_uuid, $launch_id
identify_suite # >> $suite_uuid, $suite_id
identify_tests # >> $test_uuid[1..7], $test_fullname[1..7]
identify_tests # >> $test_uuid[1..10], $test_fullname[1..10]
echo ""

# REST API | [GET] launch-controller | uuid
Expand Down Expand Up @@ -312,20 +319,26 @@ rlJournalStart
rlAssertEquals "Assert the name of suite item ${suite_name}" "$(echo $parent_item_json | jq -r .name)" "${suite_name}"
rlAssertEquals "Assert the description of suite item ${suite_name}" "$(echo $parent_item_json | jq -r .description)" "${plan_summary}<br>${launch_description}"
else
# TODO: tady pokracovat... nejak to tady nevstupuje do podminky?
echo "$parent_item_json"
echo "$parent_item_name"
if jq -e '.name == "/test/subresults"' <<< "$parent_item_json" > /dev/null; then
# All parent tests with subresults must have child subresult items
rlAssertEquals "Assert the item (${parent_item_name}) has child items" "$(echo $parent_item_json | jq -r .hasChildren)" "true"
else
# Tests with no subresults must not have any child items
rlAssertEquals "Assert the item (${parent_item_name}) has no child items" "$(echo $parent_item_json | jq -r .hasChildren)" "false"
fi
i=$content_index
rlAssertEquals "Assert the item is no suite" "$(echo $response | jq -r .content[$content_index].hasChildren)" "false"
rlAssertEquals "Assert the name of test item ${test_name[$i]}" "$(echo $response | jq -r .content[$content_index].name)" "${test_fullname[$i]}"
rlAssertEquals "Assert the UUID of test item ${test_name[$i]}" "$(echo $response | jq -r .content[$content_index].uuid)" "${test_uuid[$i]}"
rlAssertEquals "Assert the name of test item ${test_fullname[$content_index]}" "$(echo $parent_item_json | jq -r .name)" "${test_fullname[$content_index]}"
rlAssertEquals "Assert the UUID of test item ${test_fullname[$content_index]}" "$(echo $parent_item_json | jq -r .uuid)" "${test_uuid[$content_index]}"
fi
done

#######################
## DEBUG
exit 99
#######################

rlPhaseEnd


Expand All @@ -336,7 +349,7 @@ rlJournalStart
# TMT RUN [0]
rlLogInfo "Initial run that creates a launch for history"
rlRun -s "tmt run --verbose --all report --how reportportal --suite-per-plan --launch '${launch_name}_1'" 2 "" 1>/dev/null
for i in {1..7}; do
for i in {1..10}; do
echo ""
test_fullname=${TEST_PREFIX}${test[$i,'name']}
test_uuid=$(rlRun "grep -m$i -A1 'test:' $rlRun_LOG | tail -n1 | awk '{print \$NF}' ")
Expand All @@ -354,7 +367,7 @@ rlJournalStart
# TMT RUN [1]
rlLogInfo "A run that creates a launch with filtered environment variables (by default)"
rlRun -s "tmt run --verbose --all report --how reportportal --suite-per-plan --launch '${launch_name}_2'" 2 "" 1>/dev/null
for i in {1..7}; do
for i in {1..10}; do
echo ""
test_name=${test[$i,'name']}
test_fullname=${TEST_PREFIX}${test_name}
Expand All @@ -378,7 +391,7 @@ rlJournalStart
# TMT RUN [2]
rlLogInfo "A run that creates a launch without filtering the environment variables that break history aggregation"
rlRun -s "tmt run --verbose --all report --how reportportal --suite-per-plan --launch '${launch_name}_3' --exclude-variables ''" 2 "" 1>/dev/null
for i in {1..7}; do
for i in {1..10}; do
echo ""
test_name=${test[$i,'name']}
test_fullname=${TEST_PREFIX}${test_name}
Expand Down Expand Up @@ -416,7 +429,7 @@ rlJournalStart
# TMT RUN [0]
rlLogInfo "Initial run that creates a launch"
rlRun -s "tmt run --verbose --all report --how reportportal --suite-per-plan --launch '$launch_name'" 2 "" 1>/dev/null
for i in {1..7}; do
for i in {1..10}; do
core_test_uuid[$i]=$(rlRun "grep -m$i -A1 'test:' $rlRun_LOG | tail -n1 | awk '{print \$NF}' ")
rlAssertNotEquals "Assert the test$i UUID is not empty" "{$core_test_uuid[$i]}" ""
done
Expand All @@ -427,7 +440,7 @@ rlJournalStart
rlRun -s "tmt run --verbose --all report --how reportportal --suite-per-plan --launch '$launch_name' --launch-rerun" 2 "" 1>/dev/null
identify_launch # >> $launch_uuid, $launch_id
identify_suite # >> $suite_uuid
identify_tests # >> $test_uuid[1..7], $test_fullname[1..7]
identify_tests # >> $test_uuid[1..10], $test_fullname[1..10]
rlAssertGrep "suite: $suite_name" $rlRun_LOG

# REST API | [GET] launch-controller | uuid
Expand Down Expand Up @@ -456,7 +469,7 @@ rlJournalStart
# TMT RUN [0]
rlLogInfo "Initial run that creates a launch"
rlRun -s "tmt run --verbose --all report --how reportportal --suite-per-plan --launch '$launch_name'" 2 "" 1>/dev/null
identify_tests # >> $test_uuid[1..7], $test_fullname[1..7]
identify_tests # >> $test_uuid[1..10], $test_fullname[1..10]
echo ""

# TMT RE-RUN [1]
Expand Down Expand Up @@ -505,7 +518,7 @@ rlJournalStart
rlLogInfo "Initial run that only creates an empty launch (with empty suite and empty test items within) with defect type 'Idle' (pre-defined in the project within 'To Investigate' category)"
rlRun -s "tmt run discover report --verbose --how reportportal --suite-per-plan --launch '$launch_name' --defect-type 'IDLE'" 3 "" 1>/dev/null
identify_launch # >> $launch_uuid, $launch_id
identify_tests # >> $test_uuid[1..7], $test_fullname[1..7]
identify_tests # >> $test_uuid[1..10], $test_fullname[1..10]

# REST API | [GET] test-item-controller | launch_id
rlLogInfo "Get info about all launch items"
Expand Down Expand Up @@ -545,6 +558,7 @@ rlJournalStart
suite_name=$PLAN_PREFIX

# TMT RUN [0]
# FIXME: 7 tests within? - spis ne - opravit
rlLogInfo "Initial run that creates a launch (with suite item and 7 test items within)"
rlRun -s "tmt run --all report --verbose --how reportportal --suite-per-plan --launch '$launch_name'" 2 "" 1>/dev/null
identify_launch # >> $launch_uuid, $launch_id
Expand All @@ -557,6 +571,7 @@ rlJournalStart
echo ""

# TMT RUN [1]
# FIXME: 7 tests - spis ne - opravit
rlLogInfo "Additional run for an upload (of a suite item with 7 test items) to the launch"
rlRun -s "tmt run --all report --verbose --how reportportal --suite-per-plan --upload-to-launch '$launch_id'" 2 "" 1>/dev/null
identify_launch # >> $launch_uuid, $launch_id
Expand Down

0 comments on commit f4427d2

Please sign in to comment.