Merge rewrite by Alon Bar-Lev from bug #130039:

 - Verify gpg signature of snapshot if FEATURES="webrsync-gpg" is enabled.
 - Add a --revert=yyyymmdd option that alows a specific snapshot to be used.

svn path=/main/trunk/; revision=7579

1 files changed, 303 insertions, 113 deletions

diff --git a/bin/emerge-webrsync b/bin/emerge-webrsync
index 1e5f60ba1..bb6e6627c 100755
--- a/bin/emerge-webrsync
+++ b/bin/emerge-webrsync
@@ -4,11 +4,21 @@
 # $Id$
 # Author: Karl Trygve Kalleberg <karltk@gentoo.org>
 # Rewritten from the old, Perl-based emerge-webrsync script
+# Author: Alon Bar-Lev <alon.barlev@gmail.com>
+# Major rewrite from Karl's scripts.
+
+#
+# gpg key import
+# KEY_ID=0x7DDAD20D
+# gpg --homedir /etc/portage/gnupg --keyserver subkeys.pgp.net --recv-keys $KEY_ID
+# gpg --homedir /etc/portage/gnupg --edit-key $KEY_ID trust
+#
 
 type portageq > /dev/null || exit $?
 eval $(portageq envvar -v FEATURES FETCHCOMMAND GENTOO_MIRRORS \
-	PORTAGE_BIN_PATH PORTAGE_INST_UID PORTAGE_INST_GID PORTAGE_NICENESS \
-	PORTAGE_TMPDIR PORTDIR PORTAGE_RSYNC_EXTRA_OPTS http_proxy ftp_proxy)
+	PORTAGE_BIN_PATH PORTAGE_GPG_DIR PORTAGE_INST_UID PORTAGE_INST_GID \
+	PORTAGE_NICENESS PORTAGE_RSYNC_EXTRA_OPTS PORTAGE_TMPDIR PORTDIR \
+	http_proxy ftp_proxy)
 DISTDIR="${PORTAGE_TMPDIR}/emerge-webrsync"
 export http_proxy ftp_proxy
 
@@ -20,146 +30,326 @@ fi
 
 source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1
 
-if [ ! -d $DISTDIR ] ; then
-	mkdir -p $DISTDIR
-fi
+# ${USERLAND} is unreliable since the portage tree might be empty, so test
+# success of the -r option to distinguish between gnu and bsd date.
+date -r 0 >&/dev/null && DATE_ARGS="BSD" || DATE_ARGS="GNU"
 
-cd "$DISTDIR"
-
-found=0
-if [ "$1" == "-v" ] ; then
-	wgetops=
-else	
-	#this sucks.  probably better to do 1> /dev/null
-	#that said, waiting on the refactoring.
-	if [ "${FETCHCOMMAND/wget}" != "${FETCHCOMMAND}" ]; then
-		wgetops="-q"
-	elif [ "${FETCHCOMMAND/curl}" != "${FETCHCOMMAND}" ]; then
-		wgetops="-s -f"
-	fi
-fi
+do_verbose=0
 
-if type -P md5sum > /dev/null; then
-	md5_com='md5sum -c "${FILE}.md5sum"'
-elif type -P md5 > /dev/null; then
-	md5_com='[ "$(md5 -q ${FILE})" == "$(cut -d \  -f 1 ${FILE}.md5sum)" ]'
+if hasq webrsync-gpg ${FEATURES} ; then
+	WEBSYNC_VERIFY_SIGNATURE=1
 else
-	echo "warning, unable to do md5 verification of the snapshot!"
-	echo "no suitable md5/md5sum binary was found!"
-	md5_com='true'
+	WEBSYNC_VERIFY_SIGNATURE=0
 fi
+if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 -a -z "${PORTAGE_GPG_DIR}" ]; then
+	echo "Error: Please set PORTAGE_GPG_DIR in make.conf"
+	exit 1
+fi
+
+get_utc_date_in_seconds() {
+	date -u +"%s"
+}
+
+get_date_part() {
+	local utc_time_in_secs="$1"
+	local part="$2"
+
+	if [ "${DATE_ARGS}" = "BSD" ]; then
+		date -r ${utc_time_in_secs} -u +"${part}"
+	else
+		date -d @${utc_time_in_secs} -u +"${part}"
+	fi
+}
+
+get_utc_from_string() {
+	local s="$1"
+
+	seconds=$(date -d "${s:0:4}-${s:4:2}-${s:6:2}" -u +"%s")
+}
+
+get_portage_timestamp() {
+	local portage_current_timestamp=0
+
+	if [ -f "${PORTDIR}/metadata/timestamp.x" ]; then
+		portage_current_timestamp=$(cut -f 1 -d " " "${PORTDIR}/metadata/timestamp.x" )
+	fi
+
+	echo "${portage_current_timestamp}"
+}
+
+fetch_file() {
+	local URI="$1"
+	local FILE="$2"
+	local opts
+
+	if [ "${FETCHCOMMAND/wget/}" != "${FETCHCOMMAND}" ]; then
+		opts="--continue"
+
+		[ "${do_verbose}" == 0 ] && opts="$opts -q" 
+	elif [ "${FETCHCOMMAND/curl/}" != "${FETCHCOMMAND}" ]; then
+		opts="--continue-at -"
+
+		[ "${do_verbose}" == 0 ] && opts="$opts -s -f" 
+	else
+		rm -f "${FILE}"
+	fi
+
+	echo "Fetching file ${FILE}..."
+
+	#already set DISTDIR=
+	if [ "${do_verbose}" == 0 ] ; then
+		eval "${FETCHCOMMAND}" ${opts} > /dev/null && [ -s "${FILE}" ]
+	else
+		eval "${FETCHCOMMAND}" ${opts} && [ -s "${FILE}" ]
+	fi
+}
+
+check_file_digest() {
+	local digest="$1"
+	local file="$2"
+	local r=1
+
+	echo "Checking digest..."
+
+	if type -P md5sum > /dev/null; then
+		md5sum -c $digest && r=0
+	elif type -P md5 > /dev/null; then
+		[ "$(md5 -q $file)" == "$(cut -d \ -f 1 \"$digest\")" ] && r=0
+	else
+		echo "Error: Cannot check digest"
+		echo "No suitable md5/md5sum binaries found"
+	fi
+
+	return "${r}"
+}
+
+check_file_signature() {
+	local signature="$1"
+	local file="$2"
+	local r=1
+
+	if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 ]; then
+
+		echo "Checking signature..."
+
+		if type -p gpg > /dev/null; then
+			gpg --homedir "${PORTAGE_GPG_DIR}" --verify "$signature" "$file" && r=0
+		else
+			echo "Error: Cannot check signature"
+			echo "gpg binary not found"
+		fi
+	else
+		r=0
+	fi
+
+	return "${r}"
+}
+
+get_snapshot_timestamp() {
+	local file="$1"
+
+	tar --to-stdout -xf "${file}" portage/metadata/timestamp.x | cut -f 1 -d " "
+}
 
 sync_local() {
-	echo Syncing local tree...
+	local file="$1"
+
+	echo "Syncing local tree..."
+
 	if type -P tarsync &> /dev/null; then
-		# tarsync doesn't take numeric uid/gid so we need to convert them.
-		local inst_user="$(python -c "import pwd; print pwd.getpwuid(int('${PORTAGE_INST_UID:-0}'))[0]")"
-		local inst_group="$(python -c "import grp; print grp.getgrgid(int('${PORTAGE_INST_GID:-0}'))[0]")"
-		if ! tarsync "${FILE}" "${PORTDIR}" -v -s 1 -o ${inst_user} -g ${inst_group} -e /distfiles -e /packages -e /local; then
-			echo "tarsync failed; tarball is corrupt?"
-			exit 1;
+		if [ "${do_verbose}" != 0 ] ; then
+			tarsync_verbose=-v
+		fi
+		if ! tarsync $tarsync_verbose -s 1 -o portage -g portage -e /distfiles -e /packages -e /local "${file}" "${PORTDIR}"; then
+			echo "Error: tarsync failed; tarball is corrupt? (${file})"
+			return 1
 		fi
-		rm "${FILE}"
 	else
-		if ! tar jxf $FILE; then
-			echo "Tar failed to extract the image. Please review the output."
-			echo "Executed command: tar jxf $FILE"
-			exit 1
+		echo "Note: tarsync was not found, you may consider emerge it..."
+
+		if ! tar jxf "${file}"; then
+			echo "Error: tar failed to extract the image. tarball is corrupt? (${file})"
+			rm -fr portage
+			return 1
 		fi
-		rm -f $FILE
+
+		# Free disk space
+		rm -f "${file}"
+
 		# Make sure user and group file ownership is ${PORTAGE_INST_UID}:${PORTAGE_INST_GID}
 		chown -R ${PORTAGE_INST_UID:-0}:${PORTAGE_INST_GID:-0} portage
 		cd portage
 		rsync -av --progress --stats --delete --delete-after \
-		--exclude='/distfiles' --exclude='/packages' \
-		--exclude='/local' ${PORTAGE_RSYNC_EXTRA_OPTS} . ${PORTDIR%%/}
+			--exclude='/distfiles' --exclude='/packages' \
+			--exclude='/local' ${PORTAGE_RSYNC_EXTRA_OPTS} . "${PORTDIR%%/}"
 		cd ..
-		echo "cleaning up"
-		rm -rf portage
+
+		echo "Cleaning up..."
+		rm -fr portage 
 	fi
+
 	if hasq metadata-transfer ${FEATURES} ; then
-		echo "transferring metadata/cache"
+		echo "Updating cache..."
 		emerge --metadata
 	fi
 	[ -x /etc/portage/bin/post_sync ] && /etc/portage/bin/post_sync
 }
 
-echo "Fetching most recent snapshot"
-
-declare -i attempts=0
-while (( $attempts <  40 )) ; do
-	attempts=$(( attempts + 1 ))
-
-	# The snapshot for a given day is generated at 01:45 UTC on the following
-	# day, so the current day's snapshot (going by UTC time) hasn't been
-	# generated yet.  Therefore, always start by looking for the previous day's
-	# snapshot (for attempts=1, subtract 1 day from the current UTC time).
-	daysbefore=$(expr $(date -u +"%s") - 86400 \* ${attempts})
-	DATE_ARGS="-d @${daysbefore}"
-	# ${USERLAND} is unreliable since the portage tree might be empty, so test
-	# success of the -r option to distinguish between gnu and bsd date.
-	date -r ${daysbefore} >&/dev/null && DATE_ARGS="-r ${daysbefore}"
-	day=$(date ${DATE_ARGS} -u +"%d")
-	month=$(date ${DATE_ARGS} -u +"%m")
-	year=$(date ${DATE_ARGS} -u +"%Y")
-
-	FILE_ORIG="portage-${year}${month}${day}.tar.bz2"
-
-	echo "Attempting to fetch file dated: ${year}${month}${day}"
-		
-	got_md5=0
-
-	if [  ! -e "${FILE_ORIG}.md5sum" ]; then
-		FILE="${FILE_ORIG}.md5sum"
-		for i in $GENTOO_MIRRORS ; do 
-			URI="${i}/snapshots/${FILE}"
-			if (eval "$FETCHCOMMAND $wgetops") && [ -s "${FILE}" ]; then
-				got_md5=1
-				break
+do_snapshot() {
+	local ignore_timestamp="$1"
+	local date="$2"
+
+	local r=1
+
+	local file="portage-${date}.tar.bz2"
+	local digest="${file}.md5sum"
+	local signature="${file}.gpgsig"
+
+	local have_files=0
+	local mirror
+
+	echo "Trying to retrieve ${date} snapshot..."
+
+	for mirror in ${GENTOO_MIRRORS} ; do 
+
+		[ -s "${file}" -a -s "${digest}" -a -s "${signature}" ] && \
+			check_file_digest "${digest}" "${file}" && \
+			check_file_signature "${signature}" "${file}" && \
+			have_files=1
+
+		[ ${have_files} == 0 ] && \
+			fetch_file "${mirror}/snapshots/${digest}" "${digest}" && \
+			fetch_file "${mirror}/snapshots/${signature}" "${signature}" && \
+			fetch_file "${mirror}/snapshots/${file}" "${file}" && \
+			check_file_digest "${digest}" "${file}" && \
+			check_file_signature "${signature}" "${file}" && \
+			have_files=1
+
+		#
+		# If timestamp is invalid
+		# we want to try and retieve
+		# from a different mirror
+		#
+		if [ ${have_files} != 0 ]; then
+
+			echo "Getting snapshot timetasmp..."
+			local snapshot_timestamp=$(get_snapshot_timestamp "${file}")
+
+			if [ ${ignore_timestamp} == 0 ]; then
+				if [ ${snapshot_timestamp} -lt $(get_portage_timestamp) ]; then
+					echo "Warning: Portage is newer than snapshot"
+					have_files=0
+				fi
+			else
+				utc_date=$(get_utc_from_string "${date}")
+
+				#
+				# Check that this snapshot
+				# is what it claims to be...
+				#
+				if [ ${snapshot_timestamp} -lt ${seconds} ] || \
+					[ ${snapshot_timestamp} -gt $((${seconds}+ 2*86400)) ]; then
+
+					echo "Warning: Snapshot timestamp is not in acceptable period."
+					have_files=0
+				fi
 			fi
-		done
-	else
-		got_md5=1
-	fi
-	FILE="${FILE_ORIG}"
-
-	if (($got_md5 == 0 )); then
-		echo " --- No md5sum present on the mirror. (Not yet available.)"
-		continue
-	elif [ -s "${FILE}" ]; then
-		if eval "$md5_com"; then
-			echo " === snapshot $FILE is correct, using it"
-			sync_local
-			echo
-			echo " === Snapshot has been sync'd"
-			echo
-			exit 0
+		fi
+
+		if [ ${have_files} != 0 ]; then
+			break;
 		else
-			rm $FILE
+			#
+			# Remove files and use
+			# a diffeernt mirror
+			#
+			rm -f "${file}" "${digest}" "${signature}"
 		fi
+	done
+
+	if [ ${have_files} != 0 ]; then
+		sync_local "${file}" && r=0
+	else
+		echo "Warning: ${date} snapshot was not found."
 	fi
 	
-	for i in $GENTOO_MIRRORS ; do
-		URI="${i}/snapshots/$FILE"
-		rm -f "$FILE"
-		if (eval "$FETCHCOMMAND $wgetops") && [ -s "$FILE" ]; then
-			if ! eval "$md5_com"; then
-				echo "md5 failed on $FILE"
-				rm ${FILE}
-				continue
-			else
-				sync_local
-				echo
-				echo " *** Completed websync, please now perform a normal rsync if possible."
-				echo "     Update is current as of the of YYYYMMDD: ${year}${month}${day}"
-				echo
-				exit 0
-			fi
+	rm -f "${file}" "${digest}" "${signature}"
+	return "${r}"
+}
+
+do_latest_snapshot() {
+	local attempts=-1
+	local r=1
+
+	echo "Fetching most recent snapshot..."
+
+	while (( ${attempts} <  40 )) ; do
+		local day
+		local month
+		local year
+		local seconds
+
+		attempts=$(( ${attempts} + 1 ))
+
+		utc_attempt=$(expr $(get_utc_date_in_seconds) - 86400 \* ${attempts})
+
+		day=$(get_date_part ${utc_attempt} "%d")
+		month=$(get_date_part ${utc_attempt} "%m")
+		year=$(get_date_part ${utc_attempt} "%Y")
+		utc_midnight=$(get_date_part $(expr ${utc_attempt} - ${utc_attempt} % 86400) "%s")
+
+		if [ ${utc_midnight} -lt $(($(get_portage_timestamp)-86400)) ]; then
+			echo "Note: Portage content is newer than available snapshots"
+			echo "use --revert option to overide."
+			r=0
+			break;
+		fi
+
+		if do_snapshot 0 "${year}${month}${day}"; then
+			r=0
+			break;
 		fi
+	done
+
+	return "${r}"
+}
 
+main() {
+	local arg
+	local do_revert=0
+	local revert_date
+	
+	[ ! -d "${DISTDIR}" ] && mkdir -p "${DISTDIR}"
+	cd "${DISTDIR}"
+
+	for arg in $*; do
+		local v=${arg#*=}
+		case ${arg} in
+			--help)
+				echo "usage: $0 [options]"
+				echo "        --verbose (-v)    - verbose"
+				echo "        --revert=yyyymmdd - revert to snapshot"
+				exit 0
+			;;
+			--verbose|-v)
+				do_verbose=1
+			;;
+			--revert=*)
+				do_revert=1
+				revert_date=${v}
+			;;
+			*)
+				echo "Error: Invalid arguments"
+				exit 1
+			;;
+		esac
 	done
-done
 
-rm -rf portage
+	if [ ${do_revert} != 0 ]; then
+		do_snapshot 1 "${revert_date}"
+	else
+		do_latest_snapshot
+	fi
+}
 
-exit 1
+main $*