Implement XBPS_DISTFILES_MIRROR (second try)

The idea is to remove some burden from the original download
locations for distfiles by specifying one or more mirror locations.

+ Works with http:// or ftp:// mirrors for all $XBPS_CHROOT_CMD variants
+ Works with file:// mirrors together with uchroot and proot
This commit is contained in:
Jürgen Buchmüller 2015-08-19 14:45:47 +02:00
parent 686b7bff71
commit 793a156a0f
3 changed files with 100 additions and 5 deletions

View file

@ -74,6 +74,27 @@ To enable it:
$ cd void-packages
$ echo XBPS_CHROOT_CMD=proot >> etc/conf
### Distfiles mirror(s)
In etc/conf you may optionally define a mirror or a list of mirrors to search for distfiles.
$ echo 'XBPS_DISTFILES_MIRROR="ftp://192.168.100.5/gentoo/distfiles"' >> etc/conf
If more than one mirror is to be searched, you can either specify multiple URLs separated
with blanks, or add to the variable like this
$ echo 'XBPS_DISTFILES_MIRROR+=" http://repo.voidlinux.de/distfiles"' >> etc/conf
Make sure to put the blank after the first double quote in this case.
The mirrors are searched in order for the distfiles to build a package until the
checksum of the downloaded file matches the one specified in the template.
Ultimately, if no mirror carries the distfile, or in case all downloads failed the
checksum verification, the original download location is used.
If you use `proot` or `uchroot` for your XBPS_CHROOT_CMD, you may also specify a local path
using the `file://` prefix or simply an absolute path on your build host (e.g. /mnt/distfiles).
Mirror locations specified this way are bind mounted inside the chroot environment
under $XBPS_MASTERDIR and searched for distfiles just the same as remote locations.
### Quick setup in Void
Clone the `void-packages` git repository, install the bootstrap packages:

View file

@ -51,6 +51,47 @@ link_cksum() {
fi
}
try_mirrors() {
local curfile="$1" distfile="$2" dfcount="$3" subdir="$4" f="$5"
local filesum cksum basefile mirror path scheme
[ -z "$XBPS_DISTFILES_MIRROR" ] && return
basefile="$(basename $f)"
cksum=$(get_cksum $curfile $dfcount)
for mirror in $XBPS_DISTFILES_MIRROR; do
scheme="file"
if [[ $mirror == *://* ]]; then
scheme="${mirror%%:/*}"
path="${mirror#${scheme}://}"
else
path="$mirror"
fi
if [ "$scheme" == "file" ]; then
# Skip file:// mirror locations (/some/where or file:///some/where)
# where the specified directory does not exist
if [ ! -d "$path" ]; then
msg_warn "$pkgver: mount point $path does not exist...\n"
continue
fi
fi
if [[ "$mirror" == *voidlinux* ]]; then
# For distfiles.voidlinux.* append the subdirectory
mirror="$mirror/$subdir"
fi
msg_normal "$pkgver: fetching distfile '$curfile' from '$mirror'...\n"
$XBPS_FETCH_CMD "$mirror/$basefile"
# If basefile was not found, but a curfile file may exist, try to fetch it
if [ ! -f "$distfile" -a "$basefile" != "$curfile" ]; then
$XBPS_FETCH_CMD "$mirror/$curfile"
fi
[ ! -f "$distfile" ] && continue
flock -n ${distfile}.part rm -f ${distfile}.part
filesum=$(${XBPS_DIGEST_CMD} "$distfile")
[ "$cksum" == "$filesum" ] && break
msg_normal "$pkgver: checksum failed - removing '$curfile'...\n"
rm -f ${distfile}
done
}
hook() {
local srcdir="$XBPS_SRCDISTDIR/$pkgname-$version"
local dfcount=0 errors=0
@ -79,13 +120,17 @@ hook() {
if [ ! -f "$distfile" ]; then
link_cksum $curfile $distfile $dfcount
fi
# If distfile does not exist, download it.
# If distfile does not exist, download it from a mirror location.
if [ ! -f "$distfile" ]; then
try_mirrors $curfile $distfile $dfcount $pkgname-$version $f
fi
# If distfile does not exist, download it from the original location.
if [ ! -f "$distfile" ]; then
msg_normal "$pkgver: fetching distfile '$curfile'...\n"
flock "${distfile}.part" $XBPS_FETCH_CMD "$f"
if [ ! -f "$distfile" ]; then
msg_error "$pkgver: failed to fetch $curfile.\n"
fi
fi
if [ ! -f "$distfile" ]; then
msg_error "$pkgver: failed to fetch $curfile.\n"
fi
# distfile downloaded, verify sha256 hash.
flock -n ${distfile}.part rm -f ${distfile}.part

View file

@ -351,6 +351,32 @@ read_pkg() {
setup_pkg $XBPS_TARGET_PKG $XBPS_CROSS_BUILD
}
setup_distfiles_mirror() {
local mirror scheme path
# Scheme file:// mirror locations only work with uchroot or proot
for mirror in $XBPS_DISTFILES_MIRROR; do
scheme="file"
if [[ "$mirror" == *://* ]]; then
scheme="${mirror%%://*}"
path="${mirror#${scheme}://}"
else
path="$mirror"
fi
[ "$scheme" != "file" ] && continue
if [ "$XBPS_CHROOT_CMD" == "uchroot" -o "$XBPS_CHROOT_CMD" == "proot" ]; then
if [ ! -d "$path" ]; then
msg_warn "Invalid path in XBPS_DISTFILES_MIRROR ($mirror)\n"
continue
fi
mkdir -p "$XBPS_MASTERDIR/$path"
XBPS_CHROOT_CMD_ARGS+=" -b $path:$path"
else
msg_warn "File URLs ($mirror) don't work with '$XBPS_CHROOT_CMD'\n"
fi
done
}
readonly XBPS_VERSION_REQ="0.46"
readonly XBPS_VERSION=$(xbps-uhelper -V|awk '{print $2}')
readonly XBPS_SRC_VERSION="113"
@ -516,7 +542,7 @@ export XBPS_SHUTILSDIR XBPS_CROSSPFDIR XBPS_TRIGGERSDIR \
XBPS_SKIP_REMOTEREPOS XBPS_CROSS_BUILD XBPS_PKG_OPTIONS \
XBPS_CONFIG_FILE XBPS_KEEP_ALL XBPS_HOSTDIR XBPS_MASTERDIR \
XBPS_SRC_VERSION XBPS_DESTDIR XBPS_MACHINE XBPS_TEMP_MASTERDIR \
XBPS_BINPKG_EXISTS XBPS_LIBEXECDIR XBPS_DISTDIR
XBPS_BINPKG_EXISTS XBPS_LIBEXECDIR XBPS_DISTDIR XBPS_DISTFILES_MIRROR
for i in REPOSITORY DESTDIR BUILDDIR SRCDISTDIR; do
eval val="\$XBPS_$i"
@ -571,6 +597,9 @@ check_build_requirements
if [ -z "$IN_CHROOT" ]; then
trap 'exit_func' INT TERM
if [ -n "$XBPS_DISTFILES_MIRROR" ]; then
setup_distfiles_mirror
fi
fi
#