Merge emerge-webrsync from trunk for bugs #210945 and #130039.

svn path=/main/branches/2.1.2/; revision=9539

1 files changed, 340 insertions, 114 deletions

diff --git a/bin/emerge-webrsync b/bin/emerge-webrsync
index 8a68a4402..0dd791f3a 100755
--- a/bin/emerge-webrsync
+++ b/bin/emerge-webrsync
@@ -1,14 +1,41 @@
 #!/bin/bash
-# Copyright 1999-2006 Gentoo Foundation
+# Copyright 1999-2008 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 # $Id$
 # Author: Karl Trygve Kalleberg <karltk@gentoo.org>
 # Rewritten from the old, Perl-based emerge-webrsync script
+# Author: Alon Bar-Lev <alon.barlev@gmail.com>
+# Major rewrite from Karl's scripts.
 
-type portageq > /dev/null || exit $?
+# TODO:
+#  - all output should prob be converted to e* funcs
+#  - add support for ROOT
+
+#
+# gpg key import
+# KEY_ID=0x7DDAD20D
+# gpg --homedir /etc/portage/gnupg --keyserver subkeys.pgp.net --recv-keys $KEY_ID
+# gpg --homedir /etc/portage/gnupg --edit-key $KEY_ID trust
+#
+
+# Only echo if in verbose mode
+vvecho() { [[ ${do_verbose} -eq 1 ]] && echo "$@" ; }
+# Only echo if not in verbose mode
+nvecho() { [[ ${do_verbose} -eq 0 ]] && echo "$@" ; }
+# warning echos
+wecho() { echo "${argv0}: warning: $*" 1>&2 ; }
+# error echos
+eecho() { echo "${argv0}: error: $*" 1>&2 ; }
+
+argv0=$0
+if ! type portageq > /dev/null ; then
+	eecho "could not find 'portageq'; aborting"
+	exit 1
+fi
 eval $(portageq envvar -v FEATURES FETCHCOMMAND GENTOO_MIRRORS \
-	PORTAGE_BIN_PATH PORTAGE_INST_UID PORTAGE_INST_GID PORTAGE_NICENESS \
-	PORTAGE_TMPDIR PORTDIR PORTAGE_RSYNC_EXTRA_OPTS http_proxy ftp_proxy)
+	PORTAGE_BIN_PATH PORTAGE_GPG_DIR \
+	PORTAGE_NICENESS PORTAGE_RSYNC_EXTRA_OPTS PORTAGE_TMPDIR PORTDIR \
+	http_proxy ftp_proxy)
 DISTDIR="${PORTAGE_TMPDIR}/emerge-webrsync"
 export http_proxy ftp_proxy
 
@@ -20,147 +47,346 @@ fi
 
 source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1
 
-if [ ! -d $DISTDIR ] ; then
-	mkdir -p $DISTDIR
+do_verbose=0
+do_debug=0
+
+if hasq webrsync-gpg ${FEATURES} ; then
+	WEBSYNC_VERIFY_SIGNATURE=1
+else
+	WEBSYNC_VERIFY_SIGNATURE=0
+fi
+if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 -a -z "${PORTAGE_GPG_DIR}" ]; then
+	eecho "please set PORTAGE_GPG_DIR in make.conf"
+	exit 1
 fi
 
-cd "$DISTDIR"
-
-found=0
-if [ "$1" == "-v" ] ; then
-	wgetops=
-else	
-	#this sucks.  probably better to do 1> /dev/null
-	#that said, waiting on the refactoring.
-	if [ "${FETCHCOMMAND/wget}" != "${FETCHCOMMAND}" ]; then
-		wgetops="-q"
-	elif [ "${FETCHCOMMAND/curl}" != "${FETCHCOMMAND}" ]; then
-		wgetops="-s -f"
+do_tar() {
+	local file=$1; shift
+	local decompressor
+	case ${file} in
+		*.lzma) decompressor="lzcat" ;;
+		*.bz2)  decompressor="bzcat" ;;
+		*.gz)   decompressor="zcat"  ;;
+		*)      decompressor="cat"   ;;
+	esac
+	${decompressor} "${file}" | tar "$@"
+	_pipestatus=${PIPESTATUS[*]}
+	[[ ${_pipestatus// /} -eq 0 ]]
+}
+
+get_utc_date_in_seconds() {
+	date -u +"%s"
+}
+
+get_date_part() {
+	local utc_time_in_secs="$1"
+	local part="$2"
+
+	if	[[ ${USERLAND} == BSD ]] ; then
+		date -r ${utc_time_in_secs} -u +"${part}"
+	else
+		date -d @${utc_time_in_secs} -u +"${part}"
+	fi
+}
+
+get_utc_second_from_string() {
+	local s="$1"
+	date -d "${s:0:4}-${s:4:2}-${s:6:2}" -u +"%s"
+}
+
+get_portage_timestamp() {
+	local portage_current_timestamp=0
+
+	if [ -f "${PORTDIR}/metadata/timestamp.x" ]; then
+		portage_current_timestamp=$(cut -f 1 -d " " "${PORTDIR}/metadata/timestamp.x" )
 	fi
-fi
 
-if type -P md5sum > /dev/null; then
-	md5_com='md5sum -c "${FILE}.md5sum"'
-elif type -P md5 > /dev/null; then
-	md5_com='[ "$(md5 -q ${FILE})" == "$(cut -d \  -f 1 ${FILE}.md5sum)" ]'
-else
-	echo "warning, unable to do md5 verification of the snapshot!"
-	echo "no suitable md5/md5sum binary was found!"
-	md5_com='true'
-fi
+	echo "${portage_current_timestamp}"
+}
+
+fetch_file() {
+	local URI="$1"
+	local FILE="$2"
+	local opts
+
+	if [ "${FETCHCOMMAND/wget/}" != "${FETCHCOMMAND}" ]; then
+		opts="--continue $(nvecho -q)"
+	elif [ "${FETCHCOMMAND/curl/}" != "${FETCHCOMMAND}" ]; then
+		opts="--continue-at - $(nvecho -s -f)"
+	else
+		rm -f "${FILE}"
+	fi
+
+	vecho "Fetching file ${FILE} ..."
+	# already set DISTDIR=
+	eval "${FETCHCOMMAND}" ${opts}
+	[ -s "${FILE}" ]
+}
+
+check_file_digest() {
+	local digest="$1"
+	local file="$2"
+	local r=1
+
+	vecho "Checking digest ..."
+
+	if type -P md5sum > /dev/null; then
+		md5sum -c $digest && r=0
+	elif type -P md5 > /dev/null; then
+		[ "$(md5 -q $file)" == "$(cut -d \ -f 1 \"$digest\")" ] && r=0
+	else
+		eecho "cannot check digest: no suitable md5/md5sum binaries found"
+	fi
+
+	return "${r}"
+}
+
+check_file_signature() {
+	local signature="$1"
+	local file="$2"
+	local r=1
+
+	if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 ]; then
+
+		vecho "Checking signature ..."
+
+		if type -p gpg > /dev/null; then
+			gpg --homedir "${PORTAGE_GPG_DIR}" --verify "$signature" "$file" && r=0
+		else
+			eecho "cannot check signature: gpg binary not found"
+		fi
+	else
+		r=0
+	fi
+
+	return "${r}"
+}
+
+get_snapshot_timestamp() {
+	local file="$1"
+
+	do_tar "${file}" --to-stdout -xf - portage/metadata/timestamp.x | cut -f 1 -d " "
+}
 
 sync_local() {
-	echo Syncing local tree...
-	if type -P tarsync &> /dev/null; then
-		# tarsync doesn't take numeric uid/gid so we need to convert them.
-		local inst_user="$(python -c "import pwd; print pwd.getpwuid(int('${PORTAGE_INST_UID:-0}'))[0]")"
-		local inst_group="$(python -c "import grp; print grp.getgrgid(int('${PORTAGE_INST_GID:-0}'))[0]")"
-		if ! tarsync "${FILE}" "${PORTDIR}" -v -s 1 -o ${inst_user} -g ${inst_group} -e /distfiles -e /packages -e /local; then
-			echo "tarsync failed; tarball is corrupt?"
-			exit 1;
+	local file="$1"
+
+	vecho "Syncing local tree ..."
+
+	# tarsync-0.2.1 doesn't seem to support lzma compression.
+	if [ "${file##*.}" != "lzma" ] && type -P tarsync &> /dev/null; then
+		if ! tarsync $(vvecho -v) -s 1 -o portage -g portage -e /distfiles -e /packages -e /local "${file}" "${PORTDIR}"; then
+			eecho "tarsync failed; tarball is corrupt? (${file})"
+			return 1
 		fi
-		rm "${FILE}"
 	else
-		if ! tar jxf $FILE; then
-			echo "Tar failed to extract the image. Please review the output."
-			echo "Executed command: tar jxf $FILE"
-			exit 1
+		if ! do_tar "${file}" xf -; then
+			eecho "tar failed to extract the image. tarball is corrupt? (${file})"
+			rm -fr portage
+			return 1
 		fi
-		rm -f $FILE
-		# Make sure user and group file ownership is ${PORTAGE_INST_UID}:${PORTAGE_INST_GID}
-		chown -R ${PORTAGE_INST_UID:-0}:${PORTAGE_INST_GID:-0} portage
+
+		# Free disk space
+		rm -f "${file}"
+
+		chown portage:portage portage &> /dev/null && \
+			chown -R portage:portage portage
 		cd portage
 		rsync -av --progress --stats --delete --delete-after \
-		--exclude='/distfiles' --exclude='/packages' \
-		--exclude='/local' ${PORTAGE_RSYNC_EXTRA_OPTS} . "${PORTDIR%%/}"
+			--exclude='/distfiles' --exclude='/packages' \
+			--exclude='/local' ${PORTAGE_RSYNC_EXTRA_OPTS} . "${PORTDIR%%/}"
 		cd ..
-		echo "cleaning up"
-		rm -rf portage
+
+		vecho "Cleaning up ..."
+		rm -fr portage 
 	fi
+
 	if hasq metadata-transfer ${FEATURES} ; then
-		echo "transferring metadata/cache"
+		vecho "Updating cache ..."
 		emerge --metadata
 	fi
 	[ -x /etc/portage/bin/post_sync ] && /etc/portage/bin/post_sync
+	return 0
 }
 
-echo "Fetching most recent snapshot"
+do_snapshot() {
+	local ignore_timestamp="$1"
+	local date="$2"
 
-declare -i attempts=0
-while (( $attempts <  40 )) ; do
-	attempts=$(( attempts + 1 ))
+	local r=1
 
-	# The snapshot for a given day is generated at 01:45 UTC on the following
-	# day, so the current day's snapshot (going by UTC time) hasn't been
-	# generated yet.  Therefore, always start by looking for the previous day's
-	# snapshot (for attempts=1, subtract 1 day from the current UTC time).
-	daysbefore=$(expr $(date -u +"%s") - 86400 \* ${attempts})
-	if [ "${USERLAND}" = "BSD" ]; then
-		DATE_ARGS="-r ${daysbefore}"
-	else
-		DATE_ARGS="-d @${daysbefore}"
+	local base_file="portage-${date}.tar"
+
+	local have_files=0
+	local mirror
+
+	local compressions=""
+	type lzcat > /dev/null && compressions="${compressions} lzma"
+	type bzcat > /dev/null && compressions="${compressions} bz2"
+	type  zcat > /dev/null && compressions="${compressions} gz"
+	if [[ -z ${compressions} ]] ; then
+		eecho "unable to locate any decompressors (lzcat or bzcat or zcat)"
+		exit 1
 	fi
-	day=$(date ${DATE_ARGS} -u +"%d")
-	month=$(date ${DATE_ARGS} -u +"%m")
-	year=$(date ${DATE_ARGS} -u +"%Y")
-
-	FILE_ORIG="portage-${year}${month}${day}.tar.bz2"
-
-	echo "Attempting to fetch file dated: ${year}${month}${day}"
-		
-	got_md5=0
-
-	if [  ! -e "${FILE_ORIG}.md5sum" ]; then
-		FILE="${FILE_ORIG}.md5sum"
-		for i in $GENTOO_MIRRORS ; do 
-			URI="${i}/snapshots/${FILE}"
-			if (eval "$FETCHCOMMAND $wgetops") && [ -s "${FILE}" ]; then
-				got_md5=1
+
+	for mirror in ${GENTOO_MIRRORS} ; do 
+
+		vecho "Trying to retrieve ${date} snapshot from ${mirror} ..."
+
+		for compression in ${compressions} ; do
+			local file="portage-${date}.tar.${compression}"
+			local digest="${file}.md5sum"
+			local signature="${file}.gpgsig"
+
+			if [ -s "${file}" -a -s "${digest}" -a -s "${signature}" ] ; then
+				check_file_digest "${digest}" "${file}" && \
+				check_file_signature "${signature}" "${file}" && \
+				have_files=1
+			fi
+
+			if [ ${have_files} -eq 0 ] ; then
+				fetch_file "${mirror}/snapshots/${digest}" "${digest}" && \
+				fetch_file "${mirror}/snapshots/${signature}" "${signature}" && \
+				fetch_file "${mirror}/snapshots/${file}" "${file}" && \
+				check_file_digest "${digest}" "${file}" && \
+				check_file_signature "${signature}" "${file}" && \
+				have_files=1
+			fi
+
+			#
+			# If timestamp is invalid
+			# we want to try and retrieve
+			# from a different mirror
+			#
+			if [ ${have_files} -eq 1 ]; then
+
+				vecho "Getting snapshot timetasmp ..."
+				local snapshot_timestamp=$(get_snapshot_timestamp "${file}")
+
+				if [ ${ignore_timestamp} == 0 ]; then
+					if [ ${snapshot_timestamp} -lt $(get_portage_timestamp) ]; then
+						wecho "portage is newer than snapshot"
+						have_files=0
+					fi
+				else
+					local utc_seconds=$(get_utc_second_from_string "${date}")
+
+					#
+					# Check that this snapshot
+					# is what it claims to be ...
+					#
+					if [ ${snapshot_timestamp} -lt ${utc_seconds} ] || \
+						[ ${snapshot_timestamp} -gt $((${utc_seconds}+ 2*86400)) ]; then
+
+						wecho "snapshot timestamp is not in acceptable period"
+						have_files=0
+					fi
+				fi
+			fi
+
+			if [ ${have_files} -eq 1 ]; then
 				break
+			else
+				#
+				# Remove files and use a different mirror
+				#
+				rm -f "${file}" "${digest}" "${signature}"
 			fi
 		done
+
+		[ ${have_files} -eq 1 ] && break
+	done
+
+	if [ ${have_files} -eq 1 ]; then
+		sync_local "${file}" && r=0
 	else
-		got_md5=1
+		vecho "${date} snapshot was not found"
 	fi
-	FILE="${FILE_ORIG}"
-
-	if (($got_md5 == 0 )); then
-		echo " --- No md5sum present on the mirror. (Not yet available.)"
-		continue
-	elif [ -s "${FILE}" ]; then
-		if eval "$md5_com"; then
-			echo " === snapshot $FILE is correct, using it"
-			sync_local
-			echo
-			echo " === Snapshot has been sync'd"
-			echo
-			exit 0
-		else
-			rm $FILE
+	
+	rm -f "${file}" "${digest}" "${signature}"
+	return "${r}"
+}
+
+do_latest_snapshot() {
+	local attempts=-1
+	local r=1
+
+	vecho "Fetching most recent snapshot ..."
+
+	while (( ${attempts} <  40 )) ; do
+		local day
+		local month
+		local year
+		local seconds
+
+		attempts=$(( ${attempts} + 1 ))
+
+		utc_attempt=$(expr $(get_utc_date_in_seconds) - 86400 \* ${attempts})
+
+		day=$(get_date_part ${utc_attempt} "%d")
+		month=$(get_date_part ${utc_attempt} "%m")
+		year=$(get_date_part ${utc_attempt} "%Y")
+		utc_midnight=$(get_date_part $(expr ${utc_attempt} - ${utc_attempt} % 86400) "%s")
+
+		if [ ${utc_midnight} -lt $(($(get_portage_timestamp)-86400)) ]; then
+			wecho "portage content is newer than available snapshots (use --revert option to overide)"
+			r=0
+			break
+		fi
+
+		if do_snapshot 0 "${year}${month}${day}"; then
+			r=0
+			break;
 		fi
+	done
+
+	return "${r}"
+}
+
+usage() {
+	cat <<-EOF
+	Usage: $0 [options]
+	
+	Options:
+	  --revert=yyyymmdd   Revert to snapshot
+	  -q, --quiet         Only output errors
+	  -v, --verbose       Enable verbose output
+	  -x, --debug         Enable debug output
+	  -h, --help          This help screen (duh!)
+	EOF
+	if [[ -n $* ]] ; then
+		printf "\nError: %s\n" "$*" 1>&2
+		exit 1
+	else
+		exit 0
 	fi
+}
+
+main() {
+	local arg
+	local revert_date
 	
-	for i in $GENTOO_MIRRORS ; do
-		URI="${i}/snapshots/$FILE"
-		rm -f "$FILE"
-		if (eval "$FETCHCOMMAND $wgetops") && [ -s "$FILE" ]; then
-			if ! eval "$md5_com"; then
-				echo "md5 failed on $FILE"
-				rm ${FILE}
-				continue
-			else
-				sync_local
-				echo
-				echo " *** Completed websync, please now perform a normal rsync if possible."
-				echo "     Update is current as of the of YYYYMMDD: ${year}${month}${day}"
-				echo
-				exit 0
-			fi
-		fi
+	[ ! -d "${DISTDIR}" ] && mkdir -p "${DISTDIR}"
+	cd "${DISTDIR}"
 
+	for arg in "$@" ; do
+		local v=${arg#*=}
+		case ${arg} in
+			-h|--help)    usage ;;
+			-q|--quiet)   PORTAGE_QUIET=1 ;;
+			-v|--verbose) do_verbose=1 ;;
+			-x|--debug)   do_debug=1 ;;
+			--revert=*)   revert_date=${v} ;;
+			*)            usage "Invalid option '${arg}'" ;;
+		esac
 	done
-done
+	[[ ${do_debug} -eq 1 ]] && set -x
 
-rm -rf portage
+	if [[ -n ${revert_date} ]] ; then
+		do_snapshot 1 "${revert_date}"
+	else
+		do_latest_snapshot
+	fi
+}
 
-exit 1
+main "$@"