Skip to content

Commit 76dee71

Browse files
mayeutmayeut
committed
Rework scripts to be more docker cache friendly
1 parent 2cc1824 commit 76dee71

17 files changed

+679
-525
lines changed

‎docker/Dockerfile

Lines changed: 94 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,16 @@ ARG DEVTOOLSET_ROOTPATH=
66
ARG LD_LIBRARY_PATH_ARG=
77
ARG PREPEND_PATH=
88

9-
FROM $BASEIMAGE
9+
10+
FROM $BASEIMAGE AS runtime_base
1011
ARG POLICY
1112
ARG PLATFORM
1213
ARG DEVTOOLSET_ROOTPATH
1314
ARG LD_LIBRARY_PATH_ARG
1415
ARG PREPEND_PATH
1516
LABEL maintainer="The ManyLinux project"
1617

17-
ENV AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM}
18+
ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM}
1819
ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8
1920
ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}
2021
ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG}
@@ -23,13 +24,98 @@ ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
2324

2425
# setup entrypoint, this will wrap commands with `linux32` with i686 images
2526
COPY build_scripts/install-entrypoint.sh /build_scripts/install-entrypoint.sh
26-
RUN bash /build_scripts/install-entrypoint.sh && rm -rf build_scripts
27-
COPY entrypoint.sh /opt/manylinux/entrypoint.sh
28-
ENTRYPOINT ["/opt/manylinux/entrypoint.sh"]
27+
RUN /build_scripts/install-entrypoint.sh && rm -rf build_scripts
28+
COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint
29+
ENTRYPOINT ["manylinux-entrypoint"]
2930

30-
COPY build_scripts /build_scripts
31-
RUN /opt/manylinux/entrypoint.sh bash build_scripts/build.sh && rm -rf build_scripts
31+
COPY build_scripts/install-runtime-packages.sh /build_scripts/install-runtime-packages.sh
32+
RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts
3233

33-
ENV SSL_CERT_FILE=/opt/_internal/certs.pem
34+
COPY build_scripts/build_utils.sh /build_scripts/build_utils.sh
35+
36+
COPY build_scripts/install-automake.sh /build_scripts/install-automake.sh
37+
RUN manylinux-entrypoint /build_scripts/install-automake.sh
38+
39+
COPY build_scripts/install-libtool.sh /build_scripts/install-libtool.sh
40+
RUN manylinux-entrypoint /build_scripts/install-libtool.sh
41+
42+
COPY build_scripts/install-patchelf.sh /build_scripts/install-patchelf.sh
43+
RUN manylinux-entrypoint /build_scripts/install-patchelf.sh
44+
45+
COPY build_scripts/install-libxcrypt.sh /build_scripts/install-libxcrypt.sh
46+
RUN manylinux-entrypoint /build_scripts/install-libxcrypt.sh
3447

3548
CMD ["/bin/bash"]
49+
50+
51+
FROM runtime_base AS build_base
52+
COPY build_scripts/install-build-packages.sh /build_scripts/install-build-packages.sh
53+
RUN manylinux-entrypoint /build_scripts/install-build-packages.sh
54+
55+
56+
FROM build_base AS build_git
57+
COPY build_scripts/build-git.sh /build_scripts/build-git.sh
58+
RUN manylinux-entrypoint /build_scripts/build-git.sh
59+
60+
61+
FROM build_base AS build_cmake
62+
COPY build_scripts/build-cmake.sh /build_scripts/build-cmake.sh
63+
RUN manylinux-entrypoint /build_scripts/build-cmake.sh
64+
65+
66+
FROM build_base AS build_cpython
67+
COPY build_scripts/build-sqlite3.sh /build_scripts/build-sqlite3.sh
68+
RUN manylinux-entrypoint /build_scripts/build-sqlite3.sh
69+
70+
COPY build_scripts/build-cpython.sh /build_scripts/build-cpython.sh
71+
72+
73+
FROM build_cpython AS build_cpython35
74+
COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt
75+
RUN manylinux-entrypoint gpg --import /build_scripts/cpython-pubkeys.txt
76+
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.5.10
77+
78+
FROM build_cpython AS build_cpython36
79+
COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt
80+
RUN manylinux-entrypoint gpg --import /build_scripts/cpython-pubkeys.txt
81+
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.6.12
82+
83+
FROM build_cpython AS build_cpython37
84+
COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt
85+
RUN manylinux-entrypoint gpg --import /build_scripts/cpython-pubkeys.txt
86+
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.7.9
87+
88+
FROM build_cpython AS build_cpython38
89+
COPY build_scripts/ambv-pubkey.txt /build_scripts/ambv-pubkey.txt
90+
RUN manylinux-entrypoint gpg --import /build_scripts/ambv-pubkey.txt
91+
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.6
92+
93+
FROM build_cpython AS build_cpython39
94+
COPY build_scripts/ambv-pubkey.txt /build_scripts/ambv-pubkey.txt
95+
RUN manylinux-entrypoint gpg --import /build_scripts/ambv-pubkey.txt
96+
RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.1
97+
98+
FROM runtime_base AS all_cpython
99+
COPY --from=build_cpython35 /opt/_internal /opt/_internal/
100+
COPY --from=build_cpython36 /opt/_internal /opt/_internal/
101+
COPY --from=build_cpython37 /opt/_internal /opt/_internal/
102+
COPY --from=build_cpython38 /opt/_internal /opt/_internal/
103+
COPY --from=build_cpython39 /opt/_internal /opt/_internal/
104+
COPY build_scripts/install-requirements.sh /build_scripts/install-requirements.sh
105+
COPY build_scripts/requirements.txt /build_scripts/requirements.txt
106+
RUN manylinux-entrypoint /build_scripts/install-requirements.sh
107+
108+
109+
FROM runtime_base
110+
COPY --from=build_git /manylinux/git /
111+
COPY --from=build_cmake /manylinux/cmake /
112+
COPY --from=build_cpython /manylinux/sqlite3 /
113+
COPY --from=all_cpython /opt/_internal /opt/_internal/
114+
COPY build_scripts/finalize.sh /build_scripts/finalize.sh
115+
COPY build_scripts/python-tag-abi-tag.py /build_scripts/python-tag-abi-tag.py
116+
COPY build_scripts/ssl-check.py /build_scripts/ssl-check.py
117+
COPY build_scripts/manylinux-check.py /build_scripts/manylinux-check.py
118+
COPY build_scripts/requirements.txt /build_scripts/requirements.txt
119+
COPY build_scripts/requirements-tools.txt /build_scripts/requirements-tools.txt
120+
RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts
121+
ENV SSL_CERT_FILE=/opt/_internal/certs.pem

‎docker/build_scripts/build-cmake.sh

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
# Top-level build script called from Dockerfile
3+
4+
# Stop at any error, show all commands
5+
set -exuo pipefail
6+
7+
# Get script directory
8+
MY_DIR=$(dirname "${BASH_SOURCE[0]}")
9+
10+
# Get build utilities
11+
source $MY_DIR/build_utils.sh
12+
13+
# Install newest cmake
14+
CMAKE_VERSION=3.18.4
15+
CMAKE_HASH=597c61358e6a92ecbfad42a9b5321ddd801fc7e7eca08441307c9138382d4f77
16+
CMAKE_DOWNLOAD_URL=https://github.com/Kitware/CMake/releases/download
17+
18+
19+
fetch_source cmake-${CMAKE_VERSION}.tar.gz ${CMAKE_DOWNLOAD_URL}/v${CMAKE_VERSION}
20+
check_sha256sum cmake-${CMAKE_VERSION}.tar.gz ${CMAKE_HASH}
21+
tar -xzf cmake-${CMAKE_VERSION}.tar.gz
22+
pushd cmake-${CMAKE_VERSION}
23+
./bootstrap --system-curl --parallel=$(nproc)
24+
make -j$(nproc)
25+
make install DESTDIR=/manylinux/cmake
26+
popd
27+
rm -rf cmake-${CMAKE_VERSION}
28+
29+
# remove help
30+
rm -rf /manylinux/cmake/usr/local/share/cmake-*/Help
31+
32+
# Strip what we can
33+
strip_ /manylinux/cmake
34+
35+
# Install
36+
cp -rf /manylinux/cmake/* /
37+
38+
39+
cmake --version

‎docker/build_scripts/build-cpython.sh

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/bin/bash
2+
# Top-level build script called from Dockerfile
3+
4+
# Stop at any error, show all commands
5+
set -exuo pipefail
6+
7+
# Get script directory
8+
MY_DIR=$(dirname "${BASH_SOURCE[0]}")
9+
10+
# Get build utilities
11+
source $MY_DIR/build_utils.sh
12+
13+
14+
CPYTHON_VERSION=$1
15+
CPYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python
16+
17+
18+
function pyver_dist_dir {
19+
# Echoes the dist directory name of given pyver, removing alpha/beta prerelease
20+
# Thus:
21+
# 3.2.1 -> 3.2.1
22+
# 3.7.0b4 -> 3.7.0
23+
echo $1 | awk -F "." '{printf "%d.%d.%d", $1, $2, $3}'
24+
}
25+
26+
27+
CPYTHON_DIST_DIR=$(pyver_dist_dir ${CPYTHON_VERSION})
28+
fetch_source Python-${CPYTHON_VERSION}.tgz ${CPYTHON_DOWNLOAD_URL}/${CPYTHON_DIST_DIR}
29+
fetch_source Python-${CPYTHON_VERSION}.tgz.asc ${CPYTHON_DOWNLOAD_URL}/${CPYTHON_DIST_DIR}
30+
gpg --verify Python-${CPYTHON_VERSION}.tgz.asc
31+
tar -xzf Python-${CPYTHON_VERSION}.tgz
32+
pushd Python-${CPYTHON_VERSION}
33+
PREFIX="/opt/_internal/cpython-${CPYTHON_VERSION}"
34+
mkdir -p ${PREFIX}/lib
35+
./configure --prefix=${PREFIX} --disable-shared > /dev/null
36+
make -j$(nproc) > /dev/null
37+
make -j$(nproc) install > /dev/null
38+
popd
39+
rm -rf Python-${CPYTHON_VERSION} Python-${CPYTHON_VERSION}.tgz Python-${CPYTHON_VERSION}.tgz.asc
40+
41+
# we don't need libpython*.a, and they're many megabytes
42+
find ${PREFIX} -name '*.a' -print0 | xargs -0 rm -f
43+
44+
# We do not need the Python test suites
45+
find ${PREFIX} -depth \( -type d -a -name test -o -name tests \) | xargs rm -rf
46+
47+
# Strip ELF files found in ${PREFIX}
48+
strip_ ${PREFIX}
49+
50+
# Some python's install as bin/python3. Make them available as bin/python.
51+
if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then
52+
ln -s python3 ${PREFIX}/bin/python
53+
fi
54+
${PREFIX}/bin/python -m ensurepip
55+
if [ -e ${PREFIX}/bin/pip3 ] && [ ! -e ${PREFIX}/bin/pip ]; then
56+
ln -s pip3 ${PREFIX}/bin/pip
57+
fi
58+
59+
# We do not need precompiled .pyc and .pyo files.
60+
clean_pyc ${PREFIX}

‎docker/build_scripts/build-git.sh

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/bin/bash
2+
# Top-level build script called from Dockerfile
3+
4+
# Stop at any error, show all commands
5+
set -exuo pipefail
6+
7+
# Get script directory
8+
MY_DIR=$(dirname "${BASH_SOURCE[0]}")
9+
10+
# Get build utilities
11+
source $MY_DIR/build_utils.sh
12+
13+
# Install newest libtool
14+
GIT_ROOT=git-2.29.1
15+
GIT_HASH=728845a66103d8d1572a656123c2c09d7aa4c0ab8f4c3dc3911e14e18c85ee67
16+
GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git
17+
18+
19+
fetch_source ${GIT_ROOT}.tar.gz ${GIT_DOWNLOAD_URL}
20+
check_sha256sum ${GIT_ROOT}.tar.gz ${GIT_HASH}
21+
tar -xzf ${GIT_ROOT}.tar.gz
22+
pushd ${GIT_ROOT}
23+
make -j$(nproc) install prefix=/usr/local NO_GETTEXT=1 NO_TCLTK=1 DESTDIR=/manylinux/git
24+
popd
25+
rm -rf ${GIT_ROOT} ${GIT_ROOT}.tar.gz
26+
27+
28+
# Strip what we can
29+
strip_ /manylinux/git
30+
31+
# Install
32+
cp -rf /manylinux/git/* /
33+
34+
git version

‎docker/build_scripts/build-sqlite3.sh

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/bin/bash
2+
# Top-level build script called from Dockerfile
3+
4+
# Stop at any error, show all commands
5+
set -exuo pipefail
6+
7+
# Get script directory
8+
MY_DIR=$(dirname "${BASH_SOURCE[0]}")
9+
10+
# Get build utilities
11+
source $MY_DIR/build_utils.sh
12+
13+
# Install a more recent SQLite3
14+
SQLITE_AUTOCONF_VERSION=sqlite-autoconf-3330000
15+
SQLITE_AUTOCONF_HASH=106a2c48c7f75a298a7557bcc0d5f4f454e5b43811cc738b7ca294d6956bbb15
16+
SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2020
17+
18+
19+
fetch_source ${SQLITE_AUTOCONF_VERSION}.tar.gz ${SQLITE_AUTOCONF_DOWNLOAD_URL}
20+
check_sha256sum ${SQLITE_AUTOCONF_VERSION}.tar.gz ${SQLITE_AUTOCONF_HASH}
21+
tar xfz ${SQLITE_AUTOCONF_VERSION}.tar.gz
22+
pushd ${SQLITE_AUTOCONF_VERSION}
23+
DESTDIR=/manylinux/sqlite3 do_standard_install
24+
popd
25+
rm -rf ${SQLITE_AUTOCONF_VERSION} ${SQLITE_AUTOCONF_VERSION}.tar.gz
26+
27+
# static library is unused, remove it
28+
rm /manylinux/sqlite3/usr/local/lib/libsqlite3.a
29+
30+
# Install
31+
cp -rf /manylinux/sqlite3/* /
32+
33+
# Clean-up for runtime
34+
rm -rf /manylinux/sqlite3/usr/local/bin /manylinux/sqlite3/usr/local/include /manylinux/sqlite3/usr/local/lib/pkg-config /manylinux/sqlite3/usr/local/share
35+
find -L manylinux/sqlite3 -type f -a -not -name 'libsqlite3.so.*' -delete

0 commit comments

Comments
 (0)