From f5002b88d715957f0e31a7e866011bb72f9a973c Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Fri, 28 Mar 2008 10:21:22 +0000 Subject: Merge emerge-webrsync from trunk for bugs #210945 and #130039. svn path=/main/branches/2.1.2/; revision=9539 --- bin/emerge-webrsync | 454 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 340 insertions(+), 114 deletions(-) diff --git a/bin/emerge-webrsync b/bin/emerge-webrsync index 8a68a4402..0dd791f3a 100755 --- a/bin/emerge-webrsync +++ b/bin/emerge-webrsync @@ -1,14 +1,41 @@ #!/bin/bash -# Copyright 1999-2006 Gentoo Foundation +# Copyright 1999-2008 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Id$ # Author: Karl Trygve Kalleberg # Rewritten from the old, Perl-based emerge-webrsync script +# Author: Alon Bar-Lev +# Major rewrite from Karl's scripts. -type portageq > /dev/null || exit $? +# TODO: +# - all output should prob be converted to e* funcs +# - add support for ROOT + +# +# gpg key import +# KEY_ID=0x7DDAD20D +# gpg --homedir /etc/portage/gnupg --keyserver subkeys.pgp.net --recv-keys $KEY_ID +# gpg --homedir /etc/portage/gnupg --edit-key $KEY_ID trust +# + +# Only echo if in verbose mode +vvecho() { [[ ${do_verbose} -eq 1 ]] && echo "$@" ; } +# Only echo if not in verbose mode +nvecho() { [[ ${do_verbose} -eq 0 ]] && echo "$@" ; } +# warning echos +wecho() { echo "${argv0}: warning: $*" 1>&2 ; } +# error echos +eecho() { echo "${argv0}: error: $*" 1>&2 ; } + +argv0=$0 +if ! type portageq > /dev/null ; then + eecho "could not find 'portageq'; aborting" + exit 1 +fi eval $(portageq envvar -v FEATURES FETCHCOMMAND GENTOO_MIRRORS \ - PORTAGE_BIN_PATH PORTAGE_INST_UID PORTAGE_INST_GID PORTAGE_NICENESS \ - PORTAGE_TMPDIR PORTDIR PORTAGE_RSYNC_EXTRA_OPTS http_proxy ftp_proxy) + PORTAGE_BIN_PATH PORTAGE_GPG_DIR \ + PORTAGE_NICENESS PORTAGE_RSYNC_EXTRA_OPTS PORTAGE_TMPDIR PORTDIR \ + http_proxy ftp_proxy) DISTDIR="${PORTAGE_TMPDIR}/emerge-webrsync" export http_proxy ftp_proxy @@ -20,147 +47,346 @@ fi source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1 -if [ ! -d $DISTDIR ] ; then - mkdir -p $DISTDIR +do_verbose=0 +do_debug=0 + +if hasq webrsync-gpg ${FEATURES} ; then + WEBSYNC_VERIFY_SIGNATURE=1 +else + WEBSYNC_VERIFY_SIGNATURE=0 +fi +if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 -a -z "${PORTAGE_GPG_DIR}" ]; then + eecho "please set PORTAGE_GPG_DIR in make.conf" + exit 1 fi -cd "$DISTDIR" - -found=0 -if [ "$1" == "-v" ] ; then - wgetops= -else - #this sucks. probably better to do 1> /dev/null - #that said, waiting on the refactoring. - if [ "${FETCHCOMMAND/wget}" != "${FETCHCOMMAND}" ]; then - wgetops="-q" - elif [ "${FETCHCOMMAND/curl}" != "${FETCHCOMMAND}" ]; then - wgetops="-s -f" +do_tar() { + local file=$1; shift + local decompressor + case ${file} in + *.lzma) decompressor="lzcat" ;; + *.bz2) decompressor="bzcat" ;; + *.gz) decompressor="zcat" ;; + *) decompressor="cat" ;; + esac + ${decompressor} "${file}" | tar "$@" + _pipestatus=${PIPESTATUS[*]} + [[ ${_pipestatus// /} -eq 0 ]] +} + +get_utc_date_in_seconds() { + date -u +"%s" +} + +get_date_part() { + local utc_time_in_secs="$1" + local part="$2" + + if [[ ${USERLAND} == BSD ]] ; then + date -r ${utc_time_in_secs} -u +"${part}" + else + date -d @${utc_time_in_secs} -u +"${part}" + fi +} + +get_utc_second_from_string() { + local s="$1" + date -d "${s:0:4}-${s:4:2}-${s:6:2}" -u +"%s" +} + +get_portage_timestamp() { + local portage_current_timestamp=0 + + if [ -f "${PORTDIR}/metadata/timestamp.x" ]; then + portage_current_timestamp=$(cut -f 1 -d " " "${PORTDIR}/metadata/timestamp.x" ) fi -fi -if type -P md5sum > /dev/null; then - md5_com='md5sum -c "${FILE}.md5sum"' -elif type -P md5 > /dev/null; then - md5_com='[ "$(md5 -q ${FILE})" == "$(cut -d \ -f 1 ${FILE}.md5sum)" ]' -else - echo "warning, unable to do md5 verification of the snapshot!" - echo "no suitable md5/md5sum binary was found!" - md5_com='true' -fi + echo "${portage_current_timestamp}" +} + +fetch_file() { + local URI="$1" + local FILE="$2" + local opts + + if [ "${FETCHCOMMAND/wget/}" != "${FETCHCOMMAND}" ]; then + opts="--continue $(nvecho -q)" + elif [ "${FETCHCOMMAND/curl/}" != "${FETCHCOMMAND}" ]; then + opts="--continue-at - $(nvecho -s -f)" + else + rm -f "${FILE}" + fi + + vecho "Fetching file ${FILE} ..." + # already set DISTDIR= + eval "${FETCHCOMMAND}" ${opts} + [ -s "${FILE}" ] +} + +check_file_digest() { + local digest="$1" + local file="$2" + local r=1 + + vecho "Checking digest ..." + + if type -P md5sum > /dev/null; then + md5sum -c $digest && r=0 + elif type -P md5 > /dev/null; then + [ "$(md5 -q $file)" == "$(cut -d \ -f 1 \"$digest\")" ] && r=0 + else + eecho "cannot check digest: no suitable md5/md5sum binaries found" + fi + + return "${r}" +} + +check_file_signature() { + local signature="$1" + local file="$2" + local r=1 + + if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 ]; then + + vecho "Checking signature ..." + + if type -p gpg > /dev/null; then + gpg --homedir "${PORTAGE_GPG_DIR}" --verify "$signature" "$file" && r=0 + else + eecho "cannot check signature: gpg binary not found" + fi + else + r=0 + fi + + return "${r}" +} + +get_snapshot_timestamp() { + local file="$1" + + do_tar "${file}" --to-stdout -xf - portage/metadata/timestamp.x | cut -f 1 -d " " +} sync_local() { - echo Syncing local tree... - if type -P tarsync &> /dev/null; then - # tarsync doesn't take numeric uid/gid so we need to convert them. - local inst_user="$(python -c "import pwd; print pwd.getpwuid(int('${PORTAGE_INST_UID:-0}'))[0]")" - local inst_group="$(python -c "import grp; print grp.getgrgid(int('${PORTAGE_INST_GID:-0}'))[0]")" - if ! tarsync "${FILE}" "${PORTDIR}" -v -s 1 -o ${inst_user} -g ${inst_group} -e /distfiles -e /packages -e /local; then - echo "tarsync failed; tarball is corrupt?" - exit 1; + local file="$1" + + vecho "Syncing local tree ..." + + # tarsync-0.2.1 doesn't seem to support lzma compression. + if [ "${file##*.}" != "lzma" ] && type -P tarsync &> /dev/null; then + if ! tarsync $(vvecho -v) -s 1 -o portage -g portage -e /distfiles -e /packages -e /local "${file}" "${PORTDIR}"; then + eecho "tarsync failed; tarball is corrupt? (${file})" + return 1 fi - rm "${FILE}" else - if ! tar jxf $FILE; then - echo "Tar failed to extract the image. Please review the output." - echo "Executed command: tar jxf $FILE" - exit 1 + if ! do_tar "${file}" xf -; then + eecho "tar failed to extract the image. tarball is corrupt? (${file})" + rm -fr portage + return 1 fi - rm -f $FILE - # Make sure user and group file ownership is ${PORTAGE_INST_UID}:${PORTAGE_INST_GID} - chown -R ${PORTAGE_INST_UID:-0}:${PORTAGE_INST_GID:-0} portage + + # Free disk space + rm -f "${file}" + + chown portage:portage portage &> /dev/null && \ + chown -R portage:portage portage cd portage rsync -av --progress --stats --delete --delete-after \ - --exclude='/distfiles' --exclude='/packages' \ - --exclude='/local' ${PORTAGE_RSYNC_EXTRA_OPTS} . "${PORTDIR%%/}" + --exclude='/distfiles' --exclude='/packages' \ + --exclude='/local' ${PORTAGE_RSYNC_EXTRA_OPTS} . "${PORTDIR%%/}" cd .. - echo "cleaning up" - rm -rf portage + + vecho "Cleaning up ..." + rm -fr portage fi + if hasq metadata-transfer ${FEATURES} ; then - echo "transferring metadata/cache" + vecho "Updating cache ..." emerge --metadata fi [ -x /etc/portage/bin/post_sync ] && /etc/portage/bin/post_sync + return 0 } -echo "Fetching most recent snapshot" +do_snapshot() { + local ignore_timestamp="$1" + local date="$2" -declare -i attempts=0 -while (( $attempts < 40 )) ; do - attempts=$(( attempts + 1 )) + local r=1 - # The snapshot for a given day is generated at 01:45 UTC on the following - # day, so the current day's snapshot (going by UTC time) hasn't been - # generated yet. Therefore, always start by looking for the previous day's - # snapshot (for attempts=1, subtract 1 day from the current UTC time). - daysbefore=$(expr $(date -u +"%s") - 86400 \* ${attempts}) - if [ "${USERLAND}" = "BSD" ]; then - DATE_ARGS="-r ${daysbefore}" - else - DATE_ARGS="-d @${daysbefore}" + local base_file="portage-${date}.tar" + + local have_files=0 + local mirror + + local compressions="" + type lzcat > /dev/null && compressions="${compressions} lzma" + type bzcat > /dev/null && compressions="${compressions} bz2" + type zcat > /dev/null && compressions="${compressions} gz" + if [[ -z ${compressions} ]] ; then + eecho "unable to locate any decompressors (lzcat or bzcat or zcat)" + exit 1 fi - day=$(date ${DATE_ARGS} -u +"%d") - month=$(date ${DATE_ARGS} -u +"%m") - year=$(date ${DATE_ARGS} -u +"%Y") - - FILE_ORIG="portage-${year}${month}${day}.tar.bz2" - - echo "Attempting to fetch file dated: ${year}${month}${day}" - - got_md5=0 - - if [ ! -e "${FILE_ORIG}.md5sum" ]; then - FILE="${FILE_ORIG}.md5sum" - for i in $GENTOO_MIRRORS ; do - URI="${i}/snapshots/${FILE}" - if (eval "$FETCHCOMMAND $wgetops") && [ -s "${FILE}" ]; then - got_md5=1 + + for mirror in ${GENTOO_MIRRORS} ; do + + vecho "Trying to retrieve ${date} snapshot from ${mirror} ..." + + for compression in ${compressions} ; do + local file="portage-${date}.tar.${compression}" + local digest="${file}.md5sum" + local signature="${file}.gpgsig" + + if [ -s "${file}" -a -s "${digest}" -a -s "${signature}" ] ; then + check_file_digest "${digest}" "${file}" && \ + check_file_signature "${signature}" "${file}" && \ + have_files=1 + fi + + if [ ${have_files} -eq 0 ] ; then + fetch_file "${mirror}/snapshots/${digest}" "${digest}" && \ + fetch_file "${mirror}/snapshots/${signature}" "${signature}" && \ + fetch_file "${mirror}/snapshots/${file}" "${file}" && \ + check_file_digest "${digest}" "${file}" && \ + check_file_signature "${signature}" "${file}" && \ + have_files=1 + fi + + # + # If timestamp is invalid + # we want to try and retrieve + # from a different mirror + # + if [ ${have_files} -eq 1 ]; then + + vecho "Getting snapshot timetasmp ..." + local snapshot_timestamp=$(get_snapshot_timestamp "${file}") + + if [ ${ignore_timestamp} == 0 ]; then + if [ ${snapshot_timestamp} -lt $(get_portage_timestamp) ]; then + wecho "portage is newer than snapshot" + have_files=0 + fi + else + local utc_seconds=$(get_utc_second_from_string "${date}") + + # + # Check that this snapshot + # is what it claims to be ... + # + if [ ${snapshot_timestamp} -lt ${utc_seconds} ] || \ + [ ${snapshot_timestamp} -gt $((${utc_seconds}+ 2*86400)) ]; then + + wecho "snapshot timestamp is not in acceptable period" + have_files=0 + fi + fi + fi + + if [ ${have_files} -eq 1 ]; then break + else + # + # Remove files and use a different mirror + # + rm -f "${file}" "${digest}" "${signature}" fi done + + [ ${have_files} -eq 1 ] && break + done + + if [ ${have_files} -eq 1 ]; then + sync_local "${file}" && r=0 else - got_md5=1 + vecho "${date} snapshot was not found" fi - FILE="${FILE_ORIG}" - - if (($got_md5 == 0 )); then - echo " --- No md5sum present on the mirror. (Not yet available.)" - continue - elif [ -s "${FILE}" ]; then - if eval "$md5_com"; then - echo " === snapshot $FILE is correct, using it" - sync_local - echo - echo " === Snapshot has been sync'd" - echo - exit 0 - else - rm $FILE + + rm -f "${file}" "${digest}" "${signature}" + return "${r}" +} + +do_latest_snapshot() { + local attempts=-1 + local r=1 + + vecho "Fetching most recent snapshot ..." + + while (( ${attempts} < 40 )) ; do + local day + local month + local year + local seconds + + attempts=$(( ${attempts} + 1 )) + + utc_attempt=$(expr $(get_utc_date_in_seconds) - 86400 \* ${attempts}) + + day=$(get_date_part ${utc_attempt} "%d") + month=$(get_date_part ${utc_attempt} "%m") + year=$(get_date_part ${utc_attempt} "%Y") + utc_midnight=$(get_date_part $(expr ${utc_attempt} - ${utc_attempt} % 86400) "%s") + + if [ ${utc_midnight} -lt $(($(get_portage_timestamp)-86400)) ]; then + wecho "portage content is newer than available snapshots (use --revert option to overide)" + r=0 + break + fi + + if do_snapshot 0 "${year}${month}${day}"; then + r=0 + break; fi + done + + return "${r}" +} + +usage() { + cat <<-EOF + Usage: $0 [options] + + Options: + --revert=yyyymmdd Revert to snapshot + -q, --quiet Only output errors + -v, --verbose Enable verbose output + -x, --debug Enable debug output + -h, --help This help screen (duh!) + EOF + if [[ -n $* ]] ; then + printf "\nError: %s\n" "$*" 1>&2 + exit 1 + else + exit 0 fi +} + +main() { + local arg + local revert_date - for i in $GENTOO_MIRRORS ; do - URI="${i}/snapshots/$FILE" - rm -f "$FILE" - if (eval "$FETCHCOMMAND $wgetops") && [ -s "$FILE" ]; then - if ! eval "$md5_com"; then - echo "md5 failed on $FILE" - rm ${FILE} - continue - else - sync_local - echo - echo " *** Completed websync, please now perform a normal rsync if possible." - echo " Update is current as of the of YYYYMMDD: ${year}${month}${day}" - echo - exit 0 - fi - fi + [ ! -d "${DISTDIR}" ] && mkdir -p "${DISTDIR}" + cd "${DISTDIR}" + for arg in "$@" ; do + local v=${arg#*=} + case ${arg} in + -h|--help) usage ;; + -q|--quiet) PORTAGE_QUIET=1 ;; + -v|--verbose) do_verbose=1 ;; + -x|--debug) do_debug=1 ;; + --revert=*) revert_date=${v} ;; + *) usage "Invalid option '${arg}'" ;; + esac done -done + [[ ${do_debug} -eq 1 ]] && set -x -rm -rf portage + if [[ -n ${revert_date} ]] ; then + do_snapshot 1 "${revert_date}" + else + do_latest_snapshot + fi +} -exit 1 +main "$@" -- cgit v1.2.3-1-g7c22