path: root/bin/emerge-webrsync

#!/bin/bash
# Copyright 1999-2011 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
# Author: Karl Trygve Kalleberg <karltk@gentoo.org>
# Rewritten from the old, Perl-based emerge-webrsync script
# Author: Alon Bar-Lev <alon.barlev@gmail.com>
# Major rewrite from Karl's scripts.

# TODO:
#  - all output should prob be converted to e* funcs
#  - add support for ROOT

#
# gpg key import
# KEY_ID=0x7DDAD20D
# gpg --homedir /etc/portage/gnupg --keyserver subkeys.pgp.net --recv-keys $KEY_ID
# gpg --homedir /etc/portage/gnupg --edit-key $KEY_ID trust
#

# Only echo if in verbose mode
vvecho() { [[ ${do_verbose} -eq 1 ]] && echo "$@" ; }
# Only echo if not in verbose mode
nvecho() { [[ ${do_verbose} -eq 0 ]] && echo "$@" ; }
# warning echos
wecho() { echo "${argv0}: warning: $*" 1>&2 ; }
# error echos
eecho() { echo "${argv0}: error: $*" 1>&2 ; }

argv0=$0
if ! type -P portageq > /dev/null ; then
	eecho "could not find 'portageq'; aborting"
	exit 1
fi
eval $(portageq envvar -v FEATURES FETCHCOMMAND GENTOO_MIRRORS \
	PORTAGE_BIN_PATH PORTAGE_GPG_DIR \
	PORTAGE_NICENESS PORTAGE_RSYNC_EXTRA_OPTS PORTAGE_TMPDIR PORTDIR \
	SYNC http_proxy ftp_proxy)
DISTDIR="${PORTAGE_TMPDIR}/emerge-webrsync"
export http_proxy ftp_proxy

# If PORTAGE_NICENESS is overriden via the env then it will
# still pass through the portageq call and override properly.
if [ -n "${PORTAGE_NICENESS}" ]; then
	renice $PORTAGE_NICENESS $$ > /dev/null
fi

source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1

do_verbose=0
do_debug=0

if hasq webrsync-gpg ${FEATURES} ; then
	WEBSYNC_VERIFY_SIGNATURE=1
else
	WEBSYNC_VERIFY_SIGNATURE=0
fi
if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 -a -z "${PORTAGE_GPG_DIR}" ]; then
	eecho "please set PORTAGE_GPG_DIR in make.conf"
	exit 1
fi

do_tar() {
	local file=$1; shift
	local decompressor
	case ${file} in
		*.xz)   decompressor="xzcat" ;;
		*.lzma) decompressor="lzcat" ;;
		*.bz2)  decompressor="bzcat" ;;
		*.gz)   decompressor="zcat"  ;;
		*)      decompressor="cat"   ;;
	esac
	${decompressor} "${file}" | tar "$@"
	_pipestatus=${PIPESTATUS[*]}
	[[ ${_pipestatus// /} -eq 0 ]]
}

get_utc_date_in_seconds() {
	date -u +"%s"
}

get_date_part() {
	local utc_time_in_secs="$1"
	local part="$2"

	if	[[ ${USERLAND} == BSD ]] ; then
		date -r ${utc_time_in_secs} -u +"${part}"
	else
		date -d @${utc_time_in_secs} -u +"${part}"
	fi
}

get_utc_second_from_string() {
	local s="$1"
	if [[ ${USERLAND} == BSD ]] ; then
		date -juf "%Y%m%d" "$s" +"%s"
	else
		date -d "${s:0:4}-${s:4:2}-${s:6:2}" -u +"%s"
	fi
}

get_portage_timestamp() {
	local portage_current_timestamp=0

	if [ -f "${PORTDIR}/metadata/timestamp.x" ]; then
		portage_current_timestamp=$(cut -f 1 -d " " "${PORTDIR}/metadata/timestamp.x" )
	fi

	echo "${portage_current_timestamp}"
}

fetch_file() {
	local URI="$1"
	local FILE="$2"
	local opts

	if [ "${FETCHCOMMAND/wget/}" != "${FETCHCOMMAND}" ]; then
		opts="--continue $(nvecho -q)"
	elif [ "${FETCHCOMMAND/curl/}" != "${FETCHCOMMAND}" ]; then
		opts="--continue-at - $(nvecho -s -f)"
	else
		rm -f "${FILE}"
	fi

	vecho "Fetching file ${FILE} ..."
	# already set DISTDIR=
	eval "${FETCHCOMMAND}" ${opts}
	[ -s "${FILE}" ]
}

check_file_digest() {
	local digest="$1"
	local file="$2"
	local r=1

	vecho "Checking digest ..."

	if type -P md5sum > /dev/null; then
		md5sum -c $digest && r=0
	elif type -P md5 > /dev/null; then
		[ "$(md5 -q "${file}")" == "$(cut -d ' ' -f 1 "${digest}")" ] && r=0
	else
		eecho "cannot check digest: no suitable md5/md5sum binaries found"
	fi

	return "${r}"
}

check_file_signature() {
	local signature="$1"
	local file="$2"
	local r=1

	if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 ]; then

		vecho "Checking signature ..."

		if type -P gpg > /dev/null; then
			gpg --homedir "${PORTAGE_GPG_DIR}" --verify "$signature" "$file" && r=0
		else
			eecho "cannot check signature: gpg binary not found"
		fi
	else
		r=0
	fi

	return "${r}"
}

get_snapshot_timestamp() {
	local file="$1"

	do_tar "${file}" --to-stdout -xf - portage/metadata/timestamp.x | cut -f 1 -d " "
}

sync_local() {
	local file="$1"

	vecho "Syncing local tree ..."

	if type -P tarsync > /dev/null ; then
		local chown_opts="-o portage -g portage"
		chown portage:portage portage > /dev/null 2>&1 || chown_opts=""
		if ! tarsync $(vvecho -v) -s 1 ${chown_opts} \
			-e /distfiles -e /packages -e /local "${file}" "${PORTDIR}"; then
			eecho "tarsync failed; tarball is corrupt? (${file})"
			return 1
		fi
	else
		if ! do_tar "${file}" xf -; then
			eecho "tar failed to extract the image. tarball is corrupt? (${file})"
			rm -fr portage
			return 1
		fi

		# Free disk space
		rm -f "${file}"

		chown portage:portage portage > /dev/null 2>&1 && \
			chown -R portage:portage portage
		cd portage
		rsync -av --progress --stats --delete --delete-after \
			--exclude='/distfiles' --exclude='/packages' \
			--exclude='/local' ${PORTAGE_RSYNC_EXTRA_OPTS} . "${PORTDIR%%/}"
		cd ..

		vecho "Cleaning up ..."
		rm -fr portage
	fi

	if hasq metadata-transfer ${FEATURES} ; then
		vecho "Updating cache ..."
		emerge --metadata
	fi
	[ -x /etc/portage/bin/post_sync ] && /etc/portage/bin/post_sync
	return 0
}

do_snapshot() {
	local ignore_timestamp="$1"
	local date="$2"

	local r=1

	local base_file="portage-${date}.tar"

	local have_files=0
	local mirror

	local compressions=""
	# lzma is not supported in app-arch/tarsync, so use
	# bz2 format if we have tarsync.
	if ! type -P tarsync > /dev/null ; then
		type -P xzcat > /dev/null && compressions="${compressions} xz"
		type -P lzcat > /dev/null && compressions="${compressions} lzma"
	fi
	type -P bzcat > /dev/null && compressions="${compressions} bz2"
	type -P  zcat > /dev/null && compressions="${compressions} gz"
	if [[ -z ${compressions} ]] ; then
		eecho "unable to locate any decompressors (lzcat or bzcat or zcat)"
		exit 1
	fi

	for mirror in ${GENTOO_MIRRORS} ; do

		vecho "Trying to retrieve ${date} snapshot from ${mirror} ..."

		for compression in ${compressions} ; do
			local file="portage-${date}.tar.${compression}"
			local digest="${file}.md5sum"
			local signature="${file}.gpgsig"

			if [ -s "${file}" -a -s "${digest}" -a -s "${signature}" ] ; then
				check_file_digest "${DISTDIR}/${digest}" "${DISTDIR}/${file}" && \
				check_file_signature "${DISTDIR}/${signature}" "${DISTDIR}/${file}" && \
				have_files=1
			fi

			if [ ${have_files} -eq 0 ] ; then
				fetch_file "${mirror}/snapshots/${digest}" "${digest}" && \
				fetch_file "${mirror}/snapshots/${signature}" "${signature}" && \
				fetch_file "${mirror}/snapshots/${file}" "${file}" && \
				check_file_digest "${DISTDIR}/${digest}" "${DISTDIR}/${file}" && \
				check_file_signature "${DISTDIR}/${signature}" "${DISTDIR}/${file}" && \
				have_files=1
			fi

			#
			# If timestamp is invalid
			# we want to try and retrieve
			# from a different mirror
			#
			if [ ${have_files} -eq 1 ]; then

				vecho "Getting snapshot timestamp ..."
				local snapshot_timestamp=$(get_snapshot_timestamp "${file}")

				if [ ${ignore_timestamp} == 0 ]; then
					if [ ${snapshot_timestamp} -lt $(get_portage_timestamp) ]; then
						wecho "portage is newer than snapshot"
						have_files=0
					fi
				else
					local utc_seconds=$(get_utc_second_from_string "${date}")

					#
					# Check that this snapshot
					# is what it claims to be ...
					#
					if [ ${snapshot_timestamp} -lt ${utc_seconds} ] || \
						[ ${snapshot_timestamp} -gt $((${utc_seconds}+ 2*86400)) ]; then

						wecho "snapshot timestamp is not in acceptable period"
						have_files=0
					fi
				fi
			fi

			if [ ${have_files} -eq 1 ]; then
				break
			else
				#
				# Remove files and use a different mirror
				#
				rm -f "${file}" "${digest}" "${signature}"
			fi
		done

		[ ${have_files} -eq 1 ] && break
	done

	if [ ${have_files} -eq 1 ]; then
		sync_local "${file}" && r=0
	else
		vecho "${date} snapshot was not found"
	fi
	
	rm -f "${file}" "${digest}" "${signature}"
	return "${r}"
}

do_latest_snapshot() {
	local attempts=0
	local r=1

	vecho "Fetching most recent snapshot ..."

	# The snapshot for a given day is generated at 01:45 UTC on the following
	# day, so the current day's snapshot (going by UTC time) hasn't been
	# generated yet.  Therefore, always start by looking for the previous day's
	# snapshot (for attempts=1, subtract 1 day from the current UTC time).

	# Timestamps that differ by less than 2 hours
	# are considered to be approximately equal.
	local min_time_diff=$(( 2 * 60 * 60 ))

	local existing_timestamp=$(get_portage_timestamp)
	local timestamp_difference
	local timestamp_problem
	local approx_snapshot_time
	local start_time=$(get_utc_date_in_seconds)
	local start_hour=$(get_date_part ${start_time} "%H")

	# Daily snapshots are created at 1:45 AM and are not
	# available until after 2 AM. Don't waste time trying
	# to fetch a snapshot before it's been created.
	if [ ${start_hour} -lt 2 ] ; then
		(( start_time -= 86400 ))
	fi
	local snapshot_date=$(get_date_part ${start_time} "%Y%m%d")
	local snapshot_date_seconds=$(get_utc_second_from_string ${snapshot_date})

	while (( ${attempts} <  40 )) ; do
		(( attempts++ ))
		(( snapshot_date_seconds -= 86400 ))
		# snapshots are created at 1:45 AM
		(( approx_snapshot_time = snapshot_date_seconds + 86400 + 6300 ))
		(( timestamp_difference = existing_timestamp - approx_snapshot_time ))
		[ ${timestamp_difference} -lt 0 ] && (( timestamp_difference = -1 * timestamp_difference ))
		snapshot_date=$(get_date_part ${snapshot_date_seconds} "%Y%m%d")

		timestamp_problem=""
		if [ ${timestamp_difference} -eq 0 ]; then
			timestamp_problem="is identical to"
		elif [ ${timestamp_difference} -lt ${min_time_diff} ]; then
			timestamp_problem="is possibly identical to"
		elif [ ${approx_snapshot_time} -lt ${existing_timestamp} ] ; then
			timestamp_problem="is newer than"
		fi

		if [ -n "${timestamp_problem}" ]; then
			ewarn "Latest snapshot date: ${snapshot_date}"
			ewarn
			ewarn "Approximate snapshot timestamp: ${approx_snapshot_time}"
			ewarn "       Current local timestamp: ${existing_timestamp}"
			ewarn
			echo -e "The current local timestamp" \
				"${timestamp_problem} the" \
				"timestamp of the latest" \
				"snapshot. In order to force sync," \
				"use the --revert option or remove" \
				"the timestamp file located at" \
				"'${PORTDIR}/metadata/timestamp.x'." | fmt -w 70 | \
				while read -r line ; do
					ewarn "${line}"
				done
			r=0
			break
		fi

		if do_snapshot 0 "${snapshot_date}"; then
			r=0
			break;
		fi
	done

	return "${r}"
}

usage() {
	cat <<-EOF
	Usage: $0 [options]
	
	Options:
	  --revert=yyyymmdd   Revert to snapshot
	  -q, --quiet         Only output errors
	  -v, --verbose       Enable verbose output
	  -x, --debug         Enable debug output
	  -h, --help          This help screen (duh!)
	EOF
	if [[ -n $* ]] ; then
		printf "\nError: %s\n" "$*" 1>&2
		exit 1
	else
		exit 0
	fi
}

main() {
	local arg
	local revert_date
	
	[ ! -d "${DISTDIR}" ] && mkdir -p "${DISTDIR}"
	cd "${DISTDIR}"

	for arg in "$@" ; do
		local v=${arg#*=}
		case ${arg} in
			-h|--help)    usage ;;
			-q|--quiet)   PORTAGE_QUIET=1 ;;
			-v|--verbose) do_verbose=1 ;;
			-x|--debug)   do_debug=1 ;;
			--revert=*)   revert_date=${v} ;;
			*)            usage "Invalid option '${arg}'" ;;
		esac
	done

	# This is a sanity check to help prevent people like funtoo users
	# from accidentally wiping out their git tree.
	if [[ -n $SYNC && ${SYNC#rsync:} = $SYNC ]] ; then
		echo "The current SYNC variable setting does not refer to an rsync URI:" >&2
		echo >&2
		echo "  SYNC=$SYNC" >&2
		echo >&2
		echo "If you intend to use emerge-webrsync then please" >&2
		echo "adjust SYNC to refer to an rsync URI." >&2
		echo "emerge-webrsync exiting due to abnormal SYNC setting." >&2
		exit 1
	fi

	[[ ${do_debug} -eq 1 ]] && set -x

	if [[ -n ${revert_date} ]] ; then
		do_snapshot 1 "${revert_date}"
	else
		do_latest_snapshot
	fi
}

main "$@"