tests/docker: use prebuilt toolchain for debian-hexagon-cross
The current docker image for cross compiling hexagon guests
is manually built since it takes >2 hours to build from source.
This patch:
1. Solves the above issue by using the prebuilt clang
toolchain hosted on CodeLinaro [1] and maintained by QUIC [2].
2. The dockerfile is also switched from multi-stage to single stage
build to allow the CI docker engine to reuse the layer cache.
3. Re-enables the hexagon-cross-container job to be always run in
CI and makes it a non-optional dependency for the
build-user-hexagon job.
The changes for 1 & 2 together bring down the build time to
~3 minutes in GitLab CI when cache is reused and ~9 minutes
when cache cannot be reused.
[1]: https://github.com/CodeLinaro/hexagon-builder
[2]: https://github.com/quic/toolchain_for_hexagon/releases/
Signed-off-by: Mukilan Thiyagarajan <quic_mthiyaga@quicinc.com>
[AJB: also tweak MAINTAINERS, remove QEMU_JOB_ONLY_FORKS and comment]
Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
Message-Id: <20221219144354.11659-1-quic_mthiyaga@quicinc.com>
Message-Id: <20221221090411.1995037-6-alex.bennee@linaro.org>
This commit is contained in:
Mukilan Thiyagarajan
Alex Bennée
parent
2bc6c79417
commit
b9052d3634
|
|
@ -275,14 +275,10 @@ build-user-static:
|
|||
CONFIGURE_ARGS: --disable-tools --disable-system --static
|
||||
MAKE_CHECK_ARGS: check-tcg
|
||||
|
||||
# Because the hexagon cross-compiler takes so long to build we don't rely
|
||||
# on the CI system to build it and hence this job has an optional dependency
|
||||
# declared. The image is manually uploaded.
|
||||
build-user-hexagon:
|
||||
extends: .native_build_job_template
|
||||
needs:
|
||||
job: hexagon-cross-container
|
||||
optional: true
|
||||
variables:
|
||||
IMAGE: debian-hexagon-cross
|
||||
TARGETS: hexagon-linux-user
|
||||
|
|
|
|||
|
|
@ -34,31 +34,11 @@ armhf-debian-cross-container:
|
|||
variables:
|
||||
NAME: debian-armhf-cross
|
||||
|
||||
# We never want to build hexagon in the CI system and by default we
|
||||
# always want to refer to the master registry where it lives.
|
||||
hexagon-cross-container:
|
||||
extends: .base_job_template
|
||||
image: docker:stable
|
||||
extends: .container_job_template
|
||||
stage: containers
|
||||
variables:
|
||||
NAME: debian-hexagon-cross
|
||||
GIT_DEPTH: 1
|
||||
QEMU_JOB_ONLY_FORKS: 1
|
||||
services:
|
||||
- docker:dind
|
||||
before_script:
|
||||
- export TAG="$CI_REGISTRY_IMAGE/qemu/$NAME:latest"
|
||||
- export COMMON_TAG="$CI_REGISTRY/qemu-project/qemu/qemu/$NAME:latest"
|
||||
- docker info
|
||||
- docker login $CI_REGISTRY -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD"
|
||||
script:
|
||||
- echo "TAG:$TAG"
|
||||
- echo "COMMON_TAG:$COMMON_TAG"
|
||||
- docker pull $COMMON_TAG
|
||||
- docker tag $COMMON_TAG $TAG
|
||||
- docker push "$TAG"
|
||||
after_script:
|
||||
- docker logout
|
||||
|
||||
hppa-debian-cross-container:
|
||||
extends: .container_job_template
|
||||
|
|
|
|||
|
|
@ -204,7 +204,6 @@ F: tests/tcg/hexagon/
|
|||
F: disas/hexagon.c
|
||||
F: configs/targets/hexagon-linux-user/default.mak
|
||||
F: docker/dockerfiles/debian-hexagon-cross.docker
|
||||
F: docker/dockerfiles/debian-hexagon-cross.docker.d/build-toolchain.sh
|
||||
|
||||
Hexagon idef-parser
|
||||
M: Alessandro Di Federico <ale@rev.ng>
|
||||
|
|
|
|||
|
|
@ -109,10 +109,6 @@ debian-toolchain-run = \
|
|||
"PREPARE", $1))
|
||||
debian-toolchain = $(call debian-toolchain-run,$(patsubst docker-image-%,%,$1))
|
||||
|
||||
docker-image-debian-hexagon-cross: $(DOCKER_FILES_DIR)/debian-hexagon-cross.docker \
|
||||
$(DOCKER_FILES_DIR)/debian-hexagon-cross.d/build-toolchain.sh
|
||||
$(call debian-toolchain, $@)
|
||||
|
||||
docker-image-debian-microblaze-cross: $(DOCKER_FILES_DIR)/debian-toolchain.docker \
|
||||
$(DOCKER_FILES_DIR)/debian-microblaze-cross.d/build-toolchain.sh
|
||||
$(call debian-toolchain, $@)
|
||||
|
|
|
|||
|
|
@ -1,141 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
BASE=$(readlink -f ${PWD})
|
||||
|
||||
TOOLCHAIN_INSTALL=$(readlink -f "$TOOLCHAIN_INSTALL")
|
||||
ROOTFS=$(readlink -f "$ROOTFS")
|
||||
|
||||
TOOLCHAIN_BIN=${TOOLCHAIN_INSTALL}/bin
|
||||
HEX_SYSROOT=${TOOLCHAIN_INSTALL}/hexagon-unknown-linux-musl
|
||||
HEX_TOOLS_TARGET_BASE=${HEX_SYSROOT}/usr
|
||||
|
||||
function cdp() {
|
||||
DIR="$1"
|
||||
mkdir -p "$DIR"
|
||||
cd "$DIR"
|
||||
}
|
||||
|
||||
function fetch() {
|
||||
DIR="$1"
|
||||
URL="$2"
|
||||
TEMP="$(readlink -f "$PWD/tmp.tar.gz")"
|
||||
wget --quiet "$URL" -O "$TEMP"
|
||||
cdp "$DIR"
|
||||
tar xaf "$TEMP" --strip-components=1
|
||||
rm "$TEMP"
|
||||
cd -
|
||||
}
|
||||
|
||||
build_llvm_clang() {
|
||||
fetch "$BASE/llvm-project" "$LLVM_URL"
|
||||
cdp "$BASE/build-llvm"
|
||||
|
||||
cmake -G Ninja \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_INSTALL_PREFIX=${TOOLCHAIN_INSTALL} \
|
||||
-DLLVM_ENABLE_LLD=ON \
|
||||
-DLLVM_TARGETS_TO_BUILD="Hexagon" \
|
||||
-DLLVM_ENABLE_PROJECTS="clang;lld" \
|
||||
"$BASE/llvm-project/llvm"
|
||||
ninja all install
|
||||
cd ${TOOLCHAIN_BIN}
|
||||
ln -sf clang hexagon-unknown-linux-musl-clang
|
||||
ln -sf clang++ hexagon-unknown-linux-musl-clang++
|
||||
ln -sf llvm-ar hexagon-unknown-linux-musl-ar
|
||||
ln -sf llvm-objdump hexagon-unknown-linux-musl-objdump
|
||||
ln -sf llvm-objcopy hexagon-unknown-linux-musl-objcopy
|
||||
ln -sf llvm-readelf hexagon-unknown-linux-musl-readelf
|
||||
ln -sf llvm-ranlib hexagon-unknown-linux-musl-ranlib
|
||||
|
||||
# workaround for now:
|
||||
cat <<EOF > hexagon-unknown-linux-musl.cfg
|
||||
-G0 --sysroot=${HEX_SYSROOT}
|
||||
EOF
|
||||
}
|
||||
|
||||
build_clang_rt() {
|
||||
cdp "$BASE/build-clang_rt"
|
||||
cmake -G Ninja \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DLLVM_CONFIG_PATH="$BASE/build-llvm/bin/llvm-config" \
|
||||
-DCMAKE_ASM_FLAGS="-G0 -mlong-calls -fno-pic --target=hexagon-unknown-linux-musl " \
|
||||
-DCMAKE_SYSTEM_NAME=Linux \
|
||||
-DCMAKE_C_COMPILER="${TOOLCHAIN_BIN}/hexagon-unknown-linux-musl-clang" \
|
||||
-DCMAKE_ASM_COMPILER="${TOOLCHAIN_BIN}/hexagon-unknown-linux-musl-clang" \
|
||||
-DCMAKE_INSTALL_PREFIX=${HEX_TOOLS_TARGET_BASE} \
|
||||
-DCMAKE_CROSSCOMPILING=ON \
|
||||
-DCMAKE_C_COMPILER_FORCED=ON \
|
||||
-DCMAKE_CXX_COMPILER_FORCED=ON \
|
||||
-DCOMPILER_RT_BUILD_BUILTINS=ON \
|
||||
-DCOMPILER_RT_BUILTINS_ENABLE_PIC=OFF \
|
||||
-DCMAKE_SIZEOF_VOID_P=4 \
|
||||
-DCOMPILER_RT_OS_DIR= \
|
||||
-DCAN_TARGET_hexagon=1 \
|
||||
-DCAN_TARGET_x86_64=0 \
|
||||
-DCOMPILER_RT_SUPPORTED_ARCH=hexagon \
|
||||
-DLLVM_ENABLE_PROJECTS="compiler-rt" \
|
||||
"$BASE/llvm-project/compiler-rt"
|
||||
ninja install-compiler-rt
|
||||
}
|
||||
|
||||
build_musl_headers() {
|
||||
fetch "$BASE/musl" "$MUSL_URL"
|
||||
cd "$BASE/musl"
|
||||
make clean
|
||||
CC=${TOOLCHAIN_BIN}/hexagon-unknown-linux-musl-clang \
|
||||
CROSS_COMPILE=hexagon-unknown-linux-musl \
|
||||
LIBCC=${HEX_TOOLS_TARGET_BASE}/lib/libclang_rt.builtins-hexagon.a \
|
||||
CROSS_CFLAGS="-G0 -O0 -mv65 -fno-builtin -fno-rounding-math --target=hexagon-unknown-linux-musl" \
|
||||
./configure --target=hexagon --prefix=${HEX_TOOLS_TARGET_BASE}
|
||||
PATH=${TOOLCHAIN_BIN}:$PATH make CROSS_COMPILE= install-headers
|
||||
|
||||
cd ${HEX_SYSROOT}/..
|
||||
ln -sf hexagon-unknown-linux-musl hexagon
|
||||
}
|
||||
|
||||
build_kernel_headers() {
|
||||
fetch "$BASE/linux" "$LINUX_URL"
|
||||
mkdir -p "$BASE/build-linux"
|
||||
cd "$BASE/linux"
|
||||
make O=../build-linux ARCH=hexagon \
|
||||
KBUILD_CFLAGS_KERNEL="-mlong-calls" \
|
||||
CC=${TOOLCHAIN_BIN}/hexagon-unknown-linux-musl-clang \
|
||||
LD=${TOOLCHAIN_BIN}/ld.lld \
|
||||
KBUILD_VERBOSE=1 comet_defconfig
|
||||
make mrproper
|
||||
|
||||
cd "$BASE/build-linux"
|
||||
make \
|
||||
ARCH=hexagon \
|
||||
CC=${TOOLCHAIN_BIN}/clang \
|
||||
INSTALL_HDR_PATH=${HEX_TOOLS_TARGET_BASE} \
|
||||
V=1 \
|
||||
headers_install
|
||||
}
|
||||
|
||||
build_musl() {
|
||||
cd "$BASE/musl"
|
||||
make clean
|
||||
CROSS_COMPILE=hexagon-unknown-linux-musl- \
|
||||
AR=llvm-ar \
|
||||
RANLIB=llvm-ranlib \
|
||||
STRIP=llvm-strip \
|
||||
CC=clang \
|
||||
LIBCC=${HEX_TOOLS_TARGET_BASE}/lib/libclang_rt.builtins-hexagon.a \
|
||||
CFLAGS="-G0 -O0 -mv65 -fno-builtin -fno-rounding-math --target=hexagon-unknown-linux-musl" \
|
||||
./configure --target=hexagon --prefix=${HEX_TOOLS_TARGET_BASE}
|
||||
PATH=${TOOLCHAIN_BIN}/:$PATH make CROSS_COMPILE= install
|
||||
cd ${HEX_TOOLS_TARGET_BASE}/lib
|
||||
ln -sf libc.so ld-musl-hexagon.so
|
||||
ln -sf ld-musl-hexagon.so ld-musl-hexagon.so.1
|
||||
cdp ${HEX_TOOLS_TARGET_BASE}/../lib
|
||||
ln -sf ../usr/lib/ld-musl-hexagon.so.1
|
||||
}
|
||||
|
||||
build_llvm_clang
|
||||
build_kernel_headers
|
||||
build_musl_headers
|
||||
build_clang_rt
|
||||
build_musl
|
||||
|
|
@ -7,44 +7,29 @@
|
|||
#
|
||||
FROM docker.io/library/debian:11-slim
|
||||
|
||||
# Install common build utilities
|
||||
RUN apt update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata && \
|
||||
DEBIAN_FRONTEND=noninteractive eatmydata \
|
||||
apt install -y --no-install-recommends \
|
||||
bison \
|
||||
ca-certificates \
|
||||
clang \
|
||||
cmake \
|
||||
flex \
|
||||
gcc \
|
||||
lld \
|
||||
make \
|
||||
ninja-build \
|
||||
python3 \
|
||||
rsync \
|
||||
wget \
|
||||
xz-utils
|
||||
|
||||
ENV TOOLCHAIN_INSTALL /usr/local
|
||||
ENV ROOTFS /usr/local
|
||||
|
||||
ENV LLVM_URL https://github.com/llvm/llvm-project/archive/bfcd21876adc3498065e4da92799f613e730d475.tar.gz
|
||||
ENV MUSL_URL https://github.com/quic/musl/archive/aff74b395fbf59cd7e93b3691905aa1af6c0778c.tar.gz
|
||||
ENV LINUX_URL https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.6.18.tar.xz
|
||||
|
||||
ADD build-toolchain.sh /root/hexagon-toolchain/build-toolchain.sh
|
||||
|
||||
RUN cd /root/hexagon-toolchain && ./build-toolchain.sh
|
||||
|
||||
FROM docker.io/library/debian:11-slim
|
||||
# Duplicate deb line as deb-src
|
||||
RUN cat /etc/apt/sources.list | sed "s/^deb\ /deb-src /" >> /etc/apt/sources.list
|
||||
# Install QEMU build deps for use in CI
|
||||
RUN apt update && \
|
||||
RUN apt-get update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata && \
|
||||
DEBIAN_FRONTEND=noninteractive eatmydata apt install -yy bison flex git ninja-build && \
|
||||
DEBIAN_FRONTEND=noninteractive eatmydata \
|
||||
# Install common build utilities
|
||||
apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
xz-utils \
|
||||
ca-certificates \
|
||||
bison \
|
||||
flex \
|
||||
git \
|
||||
ninja-build && \
|
||||
# Install QEMU build deps for use in CI
|
||||
DEBIAN_FRONTEND=noninteractive eatmydata \
|
||||
apt build-dep -yy --arch-only qemu
|
||||
COPY --from=0 /usr/local /usr/local
|
||||
ENV PATH $PATH:/usr/local/bin/
|
||||
|
||||
|
||||
ENV TOOLCHAIN_INSTALL /opt
|
||||
ENV TOOLCHAIN_RELEASE 15.0.3
|
||||
ENV TOOLCHAIN_BASENAME "clang+llvm-${TOOLCHAIN_RELEASE}-cross-hexagon-unknown-linux-musl"
|
||||
ENV TOOLCHAIN_URL https://codelinaro.jfrog.io/artifactory/codelinaro-toolchain-for-hexagon/v${TOOLCHAIN_RELEASE}/${TOOLCHAIN_BASENAME}.tar.xz
|
||||
|
||||
RUN curl -#SL "$TOOLCHAIN_URL" | tar -xJC "$TOOLCHAIN_INSTALL"
|
||||
ENV PATH $PATH:${TOOLCHAIN_INSTALL}/${TOOLCHAIN_BASENAME}/x86_64-linux-gnu/bin
|
||||
|
|
|
|||
Loading…
Reference in a new issue