Skip to content

Commit e7732a7

Browse files
pjh5pjh5
authored
Merge pull request pytorch#193 from pjh5/upload_credentials_fix
Fixes to credentials during uploading
2 parents a4d8353 + 5f017a1 commit e7732a7

File tree

4 files changed

+60
-37
lines changed

4 files changed

+60
-37
lines changed

‎cron/build_cron.sh

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ fi
153153
# days
154154
if [[ "$NIGHTLIES_DATE" == "$(date +%Y_%m_%d)" ]]; then
155155
156-
# Upload successful binaries
156+
# Upload successful binaries and all of the logs
157157
succeeded_jobs=($(ls $SUCCEEDED_LOG_DIR))
158158
echo "Uploading all of these succesful jobs\n: $succeeded_jobs"
159159
"${NIGHTLIES_BUILDER_ROOT}/cron/upload.sh" ${succeeded_jobs[@]} > "${log_root}/upload.log" 2>&1
@@ -163,13 +163,8 @@ if [[ "$NIGHTLIES_DATE" == "$(date +%Y_%m_%d)" ]]; then
163163
first_ret="$ret"
164164
fi
165165
166-
# Upload the logs
167-
"${NIGHTLIES_BUILDER_ROOT}/cron/upload_logs.sh" > "${log_root}/upload_logs.log" 2>&1
168-
ret="$?"
169-
if [[ "$ret" != 0 && "$first_ret" == 0 ]]; then
170-
echo "FAILED upload_logs.sh"
171-
first_ret="$ret"
172-
fi
166+
# Update the HUD. N.B. if there was a problem during upload then this won't
167+
# catch it
173168
"${NIGHTLIES_BUILDER_ROOT}/cron/update_hud.sh" > "${log_root}/update_hud.log" 2>&1
174169
ret="$?"
175170
if [[ "$ret" != 0 && "$first_ret" == 0 ]]; then

‎cron/nightly_defaults.sh

Lines changed: 29 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -7,28 +7,6 @@ set -ex
77

88
echo "nightly_defaults.sh at $(pwd) starting at $(date) on $(uname -a) with pid $$"
99

10-
# List of people to email when things go wrong. This is passed directly to
11-
# `mail -t`
12-
export NIGHTLIES_EMAIL_LIST='[email protected]'
13-
14-
# PYTORCH_CREDENTIALS_FILE
15-
# A bash file that exports credentials needed to upload to aws and anaconda.
16-
# Needed variables are PYTORCH_ANACONDA_USERNAME, PYTORCH_ANACONDA_PASSWORD,
17-
# AWS_ACCESS_KEY_ID, and AWS_SECRET_ACCESS_KEY. Or it can just export the AWS
18-
# keys and then prepend a logged-in conda installation to the path.
19-
if [[ -z "$PYTORCH_CREDENTIALS_FILE" ]]; then
20-
if [[ "$(uname)" == 'Darwin' ]]; then
21-
export PYTORCH_CREDENTIALS_FILE='/Users/administrator/nightlies/credentials.sh'
22-
else
23-
export PYTORCH_CREDENTIALS_FILE='/private/home/hellemn/nightly_credentials.sh'
24-
fi
25-
fi
26-
27-
# Location of the temporary miniconda that is downloaded to install conda-build
28-
# and aws to upload finished packages TODO this is messy to install this in
29-
# upload.sh and later use it in upload_logs.sh
30-
CONDA_UPLOADER_INSTALLATION="${today}/miniconda"
31-
3210
# NIGHTLIES_FOLDER
3311
# N.B. this is also defined in cron_start.sh
3412
# An arbitrary root folder to store all nightlies folders, each of which is a
@@ -70,6 +48,35 @@ fi
7048
export today="$NIGHTLIES_FOLDER/$NIGHTLIES_DATE"
7149
mkdir -p "$today" || true
7250

51+
52+
##############################################################################
53+
# Add new configuration variables below this line. 'today' should always be
54+
# defined ASAP to avoid weird errors
55+
##############################################################################
56+
57+
58+
# List of people to email when things go wrong. This is passed directly to
59+
# `mail -t`
60+
export NIGHTLIES_EMAIL_LIST='[email protected]'
61+
62+
# PYTORCH_CREDENTIALS_FILE
63+
# A bash file that exports credentials needed to upload to aws and anaconda.
64+
# Needed variables are PYTORCH_ANACONDA_USERNAME, PYTORCH_ANACONDA_PASSWORD,
65+
# AWS_ACCESS_KEY_ID, and AWS_SECRET_ACCESS_KEY. Or it can just export the AWS
66+
# keys and then prepend a logged-in conda installation to the path.
67+
if [[ -z "$PYTORCH_CREDENTIALS_FILE" ]]; then
68+
if [[ "$(uname)" == 'Darwin' ]]; then
69+
export PYTORCH_CREDENTIALS_FILE='/Users/administrator/nightlies/credentials.sh'
70+
else
71+
export PYTORCH_CREDENTIALS_FILE='/private/home/hellemn/nightly_credentials.sh'
72+
fi
73+
fi
74+
75+
# Location of the temporary miniconda that is downloaded to install conda-build
76+
# and aws to upload finished packages TODO this is messy to install this in
77+
# upload.sh and later use it in upload_logs.sh
78+
CONDA_UPLOADER_INSTALLATION="${today}/miniconda"
79+
7380
# N.B. BUILDER_REPO and BUILDER_BRANCH are both set in cron_start.sh, as that
7481
# is the script that actually clones the builder repo that /this/ script is
7582
# running from.

‎cron/upload.sh

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@ source "${SOURCE_DIR}/nightly_defaults.sh"
1212
# ./cron/upload.sh conda_2.7_cpu.log manywheel_2.7mu_cu80.log ...
1313
# and only those corresponding packages are uploaded.
1414
# Otherwise, if given no parameters this will upload all packages.
15+
#
16+
# In both use cases, this will upload all of the logs. There is no flag to
17+
# control this. If you are manually calling this function then you are probably
18+
# overwriting some binaries in the cloud, so the corresponding logs should be
19+
# updated to reflect the new visible binaries.
1520

1621
upload_it () {
1722
pkg_type="$1"
@@ -63,12 +68,22 @@ if [[ "$ret" -ne 0 || "$ret1" -ne 0 || "$ret2" -ne 0 ]]; then
6368
rm "$miniconda_sh"
6469
export PATH="$CONDA_UPLOADER_INSTALLATION/bin:$PATH"
6570

71+
# Create an env to ensure that a Python exists
72+
conda create -yn upload_eng python=3.6
73+
source activate upload_eng
74+
6675
# Install aws and anaconda client
6776
pip install awscli
6877
conda install -y anaconda-client
6978
yes | anaconda login --username "$PYTORCH_ANACONDA_USERNAME" --password "$PYTORCH_ANACONDA_PASSWORD"
7079
fi
7180

81+
82+
# Upload all of the logs
83+
##############################################################################
84+
"${NIGHTLIES_BUILDER_ROOT}/cron/upload_logs.sh"
85+
86+
7287
# Loop through all packages to upload
7388
##############################################################################
7489
# If given package types, then only upload those package types

‎cron/upload_logs.sh

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,24 @@
11
#!/bin/bash
22

3+
# Uploads all logs to s3, takes no arguments.
4+
# The log location is configured in cron/nightly_defaults.sh
5+
#
6+
# This is called by upload.sh, since upload.sh contains all the logic to set up
7+
# conda and pip with credentials. This exists as a separate file so that you
8+
# can manually just update the logs if you wish by just calling this file.
9+
#
10+
# If you call this yourself then you are responsible for your own credentials.
11+
# N.B. PLEASE BE CAREFUL. In the past, it seems like manually logging in to
12+
# anaconda has caused problems to cron's ability to login to Anaconda and
13+
# prevented uploading.
14+
315
set -ex
416
echo "upload_logs.sh at $(pwd) starting at $(date) on $(uname -a) with pid $$"
517
SOURCE_DIR=$(cd $(dirname $0) && pwd)
618
source "${SOURCE_DIR}/nightly_defaults.sh"
719

8-
# Source the credentials if given
9-
if [[ -x "$PYTORCH_CREDENTIALS_FILE" ]]; then
10-
source "$PYTORCH_CREDENTIALS_FILE"
11-
fi
12-
export PATH="$CONDA_UPLOADER_INSTALLATION/bin:$PATH"
13-
1420
# Uploads all of the logs
15-
# N.B. do NOT include the master logs, as there may be secrets in those
21+
# N.B. do NOT include the master logs, as there are secrets in those
1622

1723
pushd "$today/logs"
1824
all_logs=($(find . -name '*.log' -not -path '*master*'))

0 commit comments

Comments
 (0)