Merge pull request #1956 from pullmoll/hashcache

xbps-src: implement a by_sha256 hash cache

common/hooks/do-fetch/00-distfiles.sh

@@ -2,22 +2,29 @@
 # the $distfiles variable and then verifies its sha256 checksum comparing
 # its value with the one stored in the $checksum variable.
 
-verify_cksum() {
-	local curfile="$1" distfile="$2" dfcount="$3" filesum ckcount cksum found i
+# Get the checksum for $curfile at index $dfcount
+get_cksum() {
+	local curfile="$1" dfcount="$2" ckcount cksum i
 
 	ckcount=0
+	cksum=0
 	for i in ${checksum}; do
 		if [ $dfcount -eq $ckcount -a -n "$i" ]; then
 			cksum=$i
-			found=yes
-			break
 		fi
-		ckcount=$(($ckcount + 1))
+		ckcount=$((ckcount + 1))
 	done
-	if [ -z $found ]; then
+	if [ -z "$cksum" ]; then
 		msg_error "$pkgver: cannot find checksum for $curfile.\n"
 	fi
+	echo "$cksum"
+}
 
+# Verify the checksum for $curfile stored at $distfile and index $dfcount
+verify_cksum() {
+	local curfile="$1" distfile="$2" dfcount="$3" filesum cksum
+
+	cksum=$(get_cksum $curfile $dfcount)
 	msg_normal "$pkgver: verifying checksum for distfile '$curfile'... "
 	filesum=$(${XBPS_DIGEST_CMD} $distfile)
 	if [ "$cksum" != "$filesum" ]; then
@@ -25,10 +32,25 @@ verify_cksum() {
 		msg_red "SHA256 mismatch for '$curfile:'\n$filesum\n"
 		errors=$(($errors + 1))
 	else
+		if [ ! -f "$XBPS_SRCDISTDIR/by_sha256/$cksum" ]; then
+			mkdir -p "$XBPS_SRCDISTDIR/by_sha256"
+			ln -f "$distfile" "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}"
+		fi
 		msg_normal_append "OK.\n"
 	fi
 }
 
+# Link an existing cksum $distfile for $curfile at index $dfcount
+link_cksum() {
+	local curfile="$1" distfile="$2" dfcount="$3" filesum cksum
+
+	cksum=$(get_cksum $curfile $dfcount)
+	if [ -n "$cksum" -a -f "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}" ]; then
+		ln -f "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}" "$distfile"
+		msg_normal "$pkgver: using known distfile $curfile.\n"
+	fi
+}
+
 hook() {
 	local srcdir="$XBPS_SRCDISTDIR/$pkgname-$version"
 	local dfcount=0 errors=0
@@ -53,7 +75,11 @@ hook() {
 			[ $? -eq 0 ] && break
 			msg_warn "$pkgver: ${curfile} is being already downloaded, waiting for 1s ...\n"
 		done
-		# If distfile does not exist download it.
+		# If distfile does not exist, try to link to it.
+		if [ ! -f "$distfile" ]; then
+			link_cksum $curfile $distfile $dfcount
+		fi
+		# If distfile does not exist, download it.
 		if [ ! -f "$distfile" ]; then
 			msg_normal "$pkgver: fetching distfile '$curfile'...\n"
 			flock "${distfile}.part" $XBPS_FETCH_CMD "$f"

common/xbps-src/shutils/update_hash_cache.sh

@@ -0,0 +1,12 @@
+# vim: set ts=4 sw=4 et:
+
+update_hash_cache() {
+    local cache="$XBPS_SRCDISTDIR/by_sha256"
+    local distfile curfile
+    mkdir -p "$cache"
+    find "$XBPS_SRCDISTDIR" -type f | grep -v by_sha256 | while read -r distfile; do
+        cksum=$(cat "$distfile" | sha256sum | cut -d " " -f 1)
+        curfile="$(basename ${distfile})"
+        ln -vf "$distfile" "${cache}/${cksum}_${curfile}"
+    done
+}

xbps-src

@@ -102,6 +102,9 @@ update-sys
 update-check <pkgname>
     Check upstream site of <pkgname> for new releases.
 
+update-hash-cache
+    Update the hash cache with existing source distfiles.
+
 zap
     Removes a masterdir but preserving ccache, distcc and host directories.
 
@@ -697,6 +700,9 @@ case "$XBPS_TARGET" in
         read_pkg
         update_check
         ;;
+    update-hash-cache)
+        update_hash_cache
+        ;;
     zap)
         masterdir_zap
         ;;