Skip to content

Commit f8ab089

Browse files
rahul003rahul003
authored
Improving CI and integration tests (aws#113)
* fix variable for log to chime * fix s3 url * fix split of pr id * Delete README.md * fix link * change logs paths * fix local trial dir * Cleanup * fix aws s3 sync * change logs dir * separate logs for each framework * upload pytest and wheels * fix current commit path * change name of framework integration tests log file * create new on_end script * rename script * change logging to use tornasole logger * comment tests and try to run upload_on_end * Try to fix path for script * uncomment tests * cleanup tests.sh * refactor * change chime message
1 parent 63feda9 commit f8ab089

File tree

6 files changed

+178
-159
lines changed

6 files changed

+178
-159
lines changed

‎config/buildspec.yml

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ env:
99
phases:
1010
install:
1111
commands:
12-
- . config/set_env.sh
12+
- . config/change_branch.sh
1313
- su && apt-get update
1414
- apt-get install sudo
1515
- sudo apt-get update
@@ -28,7 +28,5 @@ phases:
2828
#if [ "$CODEBUILD_GIT_BRANCH" = "master" ] && [ "$CODEBUILD_WEBHOOK_EVENT" = "PUSH" ] ; then
2929
post_build:
3030
commands:
31-
- cat $CODEBUILD_SRC_DIR/upload/$CURRENT_COMMIT_PATH/reports/*.html >> $CODEBUILD_SRC_DIR/upload/$CURRENT_COMMIT_PATH/reports/all_tests.html
32-
- aws s3 cp $CODEBUILD_SRC_DIR/upload s3://tornasolecodebuildtest/ --recursive
33-
- if [ "$CODEBUILD_BUILD_SUCCEEDING" -eq 0 ]; then echo "ERROR BUILD FAILED , ACCESS BUILD LOGS THROUGH GITHUB OR TROUGH THE LINK PR:$GITHUB_PR_URL , CODEBUILD:$CODEBUILD_CURRENT_BUILD_URL , Test logs are on S3 here:$S3_TEST_REPORT_URL" ; fi
34-
- if [ "$CODEBUILD_BUILD_SUCCEEDING" -eq 1 ]; then echo "INFO BUILD SUCCEEDED !!! , ACCESS BUILD LOGS THROUGH GITHUB OR TROUGH THE LINK PR:$GITHUB_PR_URL , CODEBUILD:$CODEBUILD_CURRENT_BUILD_URL , Test logs are on S3 here:$S3_TEST_REPORT_URL" ; fi
31+
- . $CODEBUILD_SRC_DIR/config/upload_on_end.sh
32+
- rm -rf $CODEBUILD_SRC_DIR/upload/$CURRENT_COMMIT_PATH

‎config/set_env.sh renamed to ‎config/change_branch.sh

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,8 @@ echo "INFO =============================BUILD STARTED===========================
2222
echo "INFO =============================Build details========================== ::"
2323
echo "INFO CODEBUILD_CURRENT_BUILD_URL = $CODEBUILD_BUILD_URL"
2424
#echo "INFO CURRENT_REPO_NAME = $CURRENT_REPO_NAME"
25-
echo "INFO CURRENT_COMMIT_DATE = $CURRENT_COMMIT_DATE"
25+
#echo "INFO CURRENT_COMMIT_DATE = $CURRENT_COMMIT_DATE"
2626
echo "INFO CODEBUILD_ACCOUNT_ID = $CODEBUILD_ACCOUNT_ID"
2727
echo "INFO CURRENT_GIT_BRANCH = $CODEBUILD_GIT_BRANCH"
28-
echo "INFO CURRENT_GIT_COMMIT = $CODEBUILD_GIT_COMMIT"
28+
echo "INFO CURRENT_GIT_COMMIT = $CURRENT_COMMIT_HASH"
2929
echo "INFO CODEBUILD_PROJECT = $CODEBUILD_PROJECT"
30-
31-
PR_ID=$(echo $CODEBUILD_WEBHOOK_TRIGGER | cut -d ';' -f 2-)
32-
export GITHUB_PR_URL=https://github.com/awslabs/$CURRENT_REPO_NAME/pull/$PR_ID
33-
#https://s3.console.aws.amazon.com/s3/object/tornasolecodebuildtest/20190817_215022/c24a121/reports/all_tests.html?region=us-east-1&tab=overview
34-
export S3_TEST_REPORT_URL=https://s3.console.aws.amazon.com/s3/object/tornasolecodebuildtest/$CURRENT_COMMIT_PATH/reports/all_tests.html

‎config/tests.sh

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#!/usr/bin/env bash
22

33
set -ex
4+
45
check_logs() {
56
if grep -e "AssertionError" $1;
67
then
@@ -11,32 +12,30 @@ check_logs() {
1112
fi
1213
}
1314

15+
run_for_framework() {
16+
python -m pytest --html=$REPORT_DIR/report_$1.html --self-contained-html tests/$1
17+
python -m pytest --html=$REPORT_DIR/test_rules_$1.html --self-contained-html -s tests/analysis/integration_testing_rules.py::test_test_rules --mode $1 --path_to_config ./tests/analysis/config.yaml --out_dir $OUT_DIR 2>&1 | tee $REPORT_DIR/test_rules_$1.log
18+
}
19+
1420
export TORNASOLE_LOG_LEVEL=debug
1521
export BLOCK_STDOUT=TRUE
1622
export BLOCK_STDERR=FALSE
17-
python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/report_analysis.html --self-contained-html tests/analysis
18-
python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/report_core.html --self-contained-html tests/core
23+
24+
export OUT_DIR=upload/$CURRENT_COMMIT_PATH
25+
export REPORT_DIR=$OUT_DIR/pytest_reports
26+
python -m pytest --html=$REPORT_DIR/report_analysis.html --self-contained-html tests/analysis
27+
python -m pytest --html=$REPORT_DIR/report_core.html --self-contained-html tests/core
1928

2029
if [ "$run_pytest_tensorflow" = "enable" ] ; then
21-
python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/report_tensorflow.html --self-contained-html tests/tensorflow
22-
python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/test_rules_tensorflow.html --self-contained-html -s tests/analysis/integration_testing_rules.py::test_test_rules --mode tensorflow --path_to_config ./tests/analysis/config.yaml 2>&1 | tee upload/$CURRENT_COMMIT_PATH/reports/test_rules_tensorflow.log
30+
run_for_framework tensorflow
2331
fi
2432

2533
if [ "$run_pytest_mxnet" = "enable" ] ; then
26-
python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/report_mxnet.html --self-contained-html tests/mxnet
27-
python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/test_rules_mxnet.html --self-contained-html -s tests/analysis/integration_testing_rules.py::test_test_rules --mode mxnet --path_to_config ./tests/analysis/config.yaml 2>&1 | tee upload/$CURRENT_COMMIT_PATH/reports/test_rules_mxnet.log
34+
run_for_framework mxnet
2835
fi
2936

3037
if [ "$run_pytest_pytorch" = "enable" ] ; then
31-
python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/report_pytorch.html --self-contained-html tests/pytorch
32-
python -m pytest --html=upload/$CURRENT_COMMIT_PATH/reports/test_rules_pytorch.html --self-contained-html -s tests/analysis/integration_testing_rules.py::test_test_rules --mode pytorch --path_to_config ./tests/analysis/config.yaml 2>&1 | tee upload/$CURRENT_COMMIT_PATH/reports/test_rules_pytorch.log
38+
run_for_framework pytorch
3339
fi
3440

35-
check_logs upload/$CURRENT_COMMIT_PATH/reports/report_analysis.html
36-
check_logs upload/$CURRENT_COMMIT_PATH/reports/report_core.html
37-
check_logs upload/$CURRENT_COMMIT_PATH/reports/report_tensorflow.html
38-
check_logs upload/$CURRENT_COMMIT_PATH/reports/report_mxnet.html
39-
check_logs upload/$CURRENT_COMMIT_PATH/reports/report_pytorch.html
40-
check_logs upload/$CURRENT_COMMIT_PATH/reports/test_rules_pytorch.log
41-
check_logs upload/$CURRENT_COMMIT_PATH/reports/test_rules_mxnet.log
42-
check_logs upload/$CURRENT_COMMIT_PATH/reports/test_rules_pytorch.log
41+
check_logs $REPORT_DIR/*

‎config/upload_on_end.sh

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
cat $CODEBUILD_SRC_DIR/upload/$CURRENT_COMMIT_PATH/pytest_reports/*.html >> $CODEBUILD_SRC_DIR/upload/$CURRENT_COMMIT_PATH/pytest_reports/all_tests.html
2+
3+
4+
upload_dirs() {
5+
for var in "$@"
6+
do
7+
aws s3 sync $CODEBUILD_SRC_DIR/upload/$CURRENT_COMMIT_PATH/$var s3://tornasolecodebuildtest/$CURRENT_COMMIT_PATH/$var
8+
done
9+
}
10+
11+
del_dirs() {
12+
for var in "$@"
13+
do
14+
aws s3 rm --recursive s3://tornasolecodebuildtest/$CURRENT_COMMIT_PATH/$var
15+
done
16+
}
17+
18+
PR_ID=$(echo $CODEBUILD_WEBHOOK_TRIGGER | cut -d '/' -f 2-)
19+
export GITHUB_PR_URL=https://github.com/awslabs/$CURRENT_REPO_NAME/pull/$PR_ID
20+
export S3_TEST_REPORT_URL=https://s3.console.aws.amazon.com/s3/object/tornasolecodebuildtest/$CURRENT_COMMIT_PATH/pytest_reports/all_tests.html?region=us-west-1
21+
22+
if [ $CODEBUILD_BUILD_SUCCEEDING -eq 0 ]
23+
then
24+
upload_dirs local_trials integration_tests_logs pytest_reports
25+
echo "ERROR BUILD FAILED , ACCESS BUILD LOGS THROUGH GITHUB OR TROUGH THE LINK PR:$GITHUB_PR_URL . CODEBUILD:$CODEBUILD_BUILD_URL . Test logs are on S3 here:$S3_TEST_REPORT_URL"
26+
else
27+
del_dirs s3_trials
28+
upload_dirs integration_tests_logs pytest_reports wheels
29+
echo "INFO BUILD SUCCEEDED !!! , ACCESS BUILD LOGS THROUGH GITHUB OR TROUGH THE LINK PR:$GITHUB_PR_URL . CODEBUILD:$CODEBUILD_BUILD_URL. Test logs are on S3 here:$S3_TEST_REPORT_URL"
30+
fi
31+

‎tests/analysis/conftest.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
def pytest_addoption(parser):
22
parser.addoption('--mode', dest='mode', default=None)
33
parser.addoption('--path_to_config', action='store', dest='path_to_config', default=None)
4+
parser.addoption('--out_dir', action='store', dest='out_dir', default='./')
5+
parser.addoption("--test_case", action="append", dest='test_case', default=[])
6+
parser.addoption("--test_case_regex", action="store", dest='test_case_regex', default=None)
7+
48
parser.addoption('--tf_path', action='store', dest='tf_path', default=None)
59
parser.addoption('--pytorch_path', action='store', dest='pytorch_path', default=None)
610
parser.addoption('--mxnet_path', action='store', dest='mxnet_path', default=None)
711
parser.addoption('--rules_path', action='store', dest='rules_path', default=None)
812
parser.addoption('--core_path', action='store', dest='core_path', default=None)
9-
parser.addoption("--test_case", action="append", dest='test_case', default=[])
10-
parser.addoption("--test_case_regex", action="store", dest='test_case_regex', default=None)
1113
parser.addoption("--CI_OR_LOCAL", action="store", dest='CI_OR_LOCAL', default=None)
1214
parser.addoption("--CODEBUILD_SRC_DIR", action="store", dest='CODEBUILD_SRC_DIR', default=None)

0 commit comments

Comments
 (0)