diff options
| -rwxr-xr-x | bin/emerge-webrsync | 416 |
1 files changed, 303 insertions, 113 deletions
diff --git a/bin/emerge-webrsync b/bin/emerge-webrsync index 1e5f60ba1..bb6e6627c 100755 --- a/bin/emerge-webrsync +++ b/bin/emerge-webrsync @@ -4,11 +4,21 @@ # $Id$ # Author: Karl Trygve Kalleberg <karltk@gentoo.org> # Rewritten from the old, Perl-based emerge-webrsync script +# Author: Alon Bar-Lev <alon.barlev@gmail.com> +# Major rewrite from Karl's scripts. + +# +# gpg key import +# KEY_ID=0x7DDAD20D +# gpg --homedir /etc/portage/gnupg --keyserver subkeys.pgp.net --recv-keys $KEY_ID +# gpg --homedir /etc/portage/gnupg --edit-key $KEY_ID trust +# type portageq > /dev/null || exit $? eval $(portageq envvar -v FEATURES FETCHCOMMAND GENTOO_MIRRORS \ - PORTAGE_BIN_PATH PORTAGE_INST_UID PORTAGE_INST_GID PORTAGE_NICENESS \ - PORTAGE_TMPDIR PORTDIR PORTAGE_RSYNC_EXTRA_OPTS http_proxy ftp_proxy) + PORTAGE_BIN_PATH PORTAGE_GPG_DIR PORTAGE_INST_UID PORTAGE_INST_GID \ + PORTAGE_NICENESS PORTAGE_RSYNC_EXTRA_OPTS PORTAGE_TMPDIR PORTDIR \ + http_proxy ftp_proxy) DISTDIR="${PORTAGE_TMPDIR}/emerge-webrsync" export http_proxy ftp_proxy @@ -20,146 +30,326 @@ fi source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1 -if [ ! -d $DISTDIR ] ; then - mkdir -p $DISTDIR -fi +# ${USERLAND} is unreliable since the portage tree might be empty, so test +# success of the -r option to distinguish between gnu and bsd date. +date -r 0 >&/dev/null && DATE_ARGS="BSD" || DATE_ARGS="GNU" -cd "$DISTDIR" - -found=0 -if [ "$1" == "-v" ] ; then - wgetops= -else - #this sucks. probably better to do 1> /dev/null - #that said, waiting on the refactoring. - if [ "${FETCHCOMMAND/wget}" != "${FETCHCOMMAND}" ]; then - wgetops="-q" - elif [ "${FETCHCOMMAND/curl}" != "${FETCHCOMMAND}" ]; then - wgetops="-s -f" - fi -fi +do_verbose=0 -if type -P md5sum > /dev/null; then - md5_com='md5sum -c "${FILE}.md5sum"' -elif type -P md5 > /dev/null; then - md5_com='[ "$(md5 -q ${FILE})" == "$(cut -d \ -f 1 ${FILE}.md5sum)" ]' +if hasq webrsync-gpg ${FEATURES} ; then + WEBSYNC_VERIFY_SIGNATURE=1 else - echo "warning, unable to do md5 verification of the snapshot!" - echo "no suitable md5/md5sum binary was found!" - md5_com='true' + WEBSYNC_VERIFY_SIGNATURE=0 fi +if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 -a -z "${PORTAGE_GPG_DIR}" ]; then + echo "Error: Please set PORTAGE_GPG_DIR in make.conf" + exit 1 +fi + +get_utc_date_in_seconds() { + date -u +"%s" +} + +get_date_part() { + local utc_time_in_secs="$1" + local part="$2" + + if [ "${DATE_ARGS}" = "BSD" ]; then + date -r ${utc_time_in_secs} -u +"${part}" + else + date -d @${utc_time_in_secs} -u +"${part}" + fi +} + +get_utc_from_string() { + local s="$1" + + seconds=$(date -d "${s:0:4}-${s:4:2}-${s:6:2}" -u +"%s") +} + +get_portage_timestamp() { + local portage_current_timestamp=0 + + if [ -f "${PORTDIR}/metadata/timestamp.x" ]; then + portage_current_timestamp=$(cut -f 1 -d " " "${PORTDIR}/metadata/timestamp.x" ) + fi + + echo "${portage_current_timestamp}" +} + +fetch_file() { + local URI="$1" + local FILE="$2" + local opts + + if [ "${FETCHCOMMAND/wget/}" != "${FETCHCOMMAND}" ]; then + opts="--continue" + + [ "${do_verbose}" == 0 ] && opts="$opts -q" + elif [ "${FETCHCOMMAND/curl/}" != "${FETCHCOMMAND}" ]; then + opts="--continue-at -" + + [ "${do_verbose}" == 0 ] && opts="$opts -s -f" + else + rm -f "${FILE}" + fi + + echo "Fetching file ${FILE}..." + + #already set DISTDIR= + if [ "${do_verbose}" == 0 ] ; then + eval "${FETCHCOMMAND}" ${opts} > /dev/null && [ -s "${FILE}" ] + else + eval "${FETCHCOMMAND}" ${opts} && [ -s "${FILE}" ] + fi +} + +check_file_digest() { + local digest="$1" + local file="$2" + local r=1 + + echo "Checking digest..." + + if type -P md5sum > /dev/null; then + md5sum -c $digest && r=0 + elif type -P md5 > /dev/null; then + [ "$(md5 -q $file)" == "$(cut -d \ -f 1 \"$digest\")" ] && r=0 + else + echo "Error: Cannot check digest" + echo "No suitable md5/md5sum binaries found" + fi + + return "${r}" +} + +check_file_signature() { + local signature="$1" + local file="$2" + local r=1 + + if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 ]; then + + echo "Checking signature..." + + if type -p gpg > /dev/null; then + gpg --homedir "${PORTAGE_GPG_DIR}" --verify "$signature" "$file" && r=0 + else + echo "Error: Cannot check signature" + echo "gpg binary not found" + fi + else + r=0 + fi + + return "${r}" +} + +get_snapshot_timestamp() { + local file="$1" + + tar --to-stdout -xf "${file}" portage/metadata/timestamp.x | cut -f 1 -d " " +} sync_local() { - echo Syncing local tree... + local file="$1" + + echo "Syncing local tree..." + if type -P tarsync &> /dev/null; then - # tarsync doesn't take numeric uid/gid so we need to convert them. - local inst_user="$(python -c "import pwd; print pwd.getpwuid(int('${PORTAGE_INST_UID:-0}'))[0]")" - local inst_group="$(python -c "import grp; print grp.getgrgid(int('${PORTAGE_INST_GID:-0}'))[0]")" - if ! tarsync "${FILE}" "${PORTDIR}" -v -s 1 -o ${inst_user} -g ${inst_group} -e /distfiles -e /packages -e /local; then - echo "tarsync failed; tarball is corrupt?" - exit 1; + if [ "${do_verbose}" != 0 ] ; then + tarsync_verbose=-v + fi + if ! tarsync $tarsync_verbose -s 1 -o portage -g portage -e /distfiles -e /packages -e /local "${file}" "${PORTDIR}"; then + echo "Error: tarsync failed; tarball is corrupt? (${file})" + return 1 fi - rm "${FILE}" else - if ! tar jxf $FILE; then - echo "Tar failed to extract the image. Please review the output." - echo "Executed command: tar jxf $FILE" - exit 1 + echo "Note: tarsync was not found, you may consider emerge it..." + + if ! tar jxf "${file}"; then + echo "Error: tar failed to extract the image. tarball is corrupt? (${file})" + rm -fr portage + return 1 fi - rm -f $FILE + + # Free disk space + rm -f "${file}" + # Make sure user and group file ownership is ${PORTAGE_INST_UID}:${PORTAGE_INST_GID} chown -R ${PORTAGE_INST_UID:-0}:${PORTAGE_INST_GID:-0} portage cd portage rsync -av --progress --stats --delete --delete-after \ - --exclude='/distfiles' --exclude='/packages' \ - --exclude='/local' ${PORTAGE_RSYNC_EXTRA_OPTS} . ${PORTDIR%%/} + --exclude='/distfiles' --exclude='/packages' \ + --exclude='/local' ${PORTAGE_RSYNC_EXTRA_OPTS} . "${PORTDIR%%/}" cd .. - echo "cleaning up" - rm -rf portage + + echo "Cleaning up..." + rm -fr portage fi + if hasq metadata-transfer ${FEATURES} ; then - echo "transferring metadata/cache" + echo "Updating cache..." emerge --metadata fi [ -x /etc/portage/bin/post_sync ] && /etc/portage/bin/post_sync } -echo "Fetching most recent snapshot" - -declare -i attempts=0 -while (( $attempts < 40 )) ; do - attempts=$(( attempts + 1 )) - - # The snapshot for a given day is generated at 01:45 UTC on the following - # day, so the current day's snapshot (going by UTC time) hasn't been - # generated yet. Therefore, always start by looking for the previous day's - # snapshot (for attempts=1, subtract 1 day from the current UTC time). - daysbefore=$(expr $(date -u +"%s") - 86400 \* ${attempts}) - DATE_ARGS="-d @${daysbefore}" - # ${USERLAND} is unreliable since the portage tree might be empty, so test - # success of the -r option to distinguish between gnu and bsd date. - date -r ${daysbefore} >&/dev/null && DATE_ARGS="-r ${daysbefore}" - day=$(date ${DATE_ARGS} -u +"%d") - month=$(date ${DATE_ARGS} -u +"%m") - year=$(date ${DATE_ARGS} -u +"%Y") - - FILE_ORIG="portage-${year}${month}${day}.tar.bz2" - - echo "Attempting to fetch file dated: ${year}${month}${day}" - - got_md5=0 - - if [ ! -e "${FILE_ORIG}.md5sum" ]; then - FILE="${FILE_ORIG}.md5sum" - for i in $GENTOO_MIRRORS ; do - URI="${i}/snapshots/${FILE}" - if (eval "$FETCHCOMMAND $wgetops") && [ -s "${FILE}" ]; then - got_md5=1 - break +do_snapshot() { + local ignore_timestamp="$1" + local date="$2" + + local r=1 + + local file="portage-${date}.tar.bz2" + local digest="${file}.md5sum" + local signature="${file}.gpgsig" + + local have_files=0 + local mirror + + echo "Trying to retrieve ${date} snapshot..." + + for mirror in ${GENTOO_MIRRORS} ; do + + [ -s "${file}" -a -s "${digest}" -a -s "${signature}" ] && \ + check_file_digest "${digest}" "${file}" && \ + check_file_signature "${signature}" "${file}" && \ + have_files=1 + + [ ${have_files} == 0 ] && \ + fetch_file "${mirror}/snapshots/${digest}" "${digest}" && \ + fetch_file "${mirror}/snapshots/${signature}" "${signature}" && \ + fetch_file "${mirror}/snapshots/${file}" "${file}" && \ + check_file_digest "${digest}" "${file}" && \ + check_file_signature "${signature}" "${file}" && \ + have_files=1 + + # + # If timestamp is invalid + # we want to try and retieve + # from a different mirror + # + if [ ${have_files} != 0 ]; then + + echo "Getting snapshot timetasmp..." + local snapshot_timestamp=$(get_snapshot_timestamp "${file}") + + if [ ${ignore_timestamp} == 0 ]; then + if [ ${snapshot_timestamp} -lt $(get_portage_timestamp) ]; then + echo "Warning: Portage is newer than snapshot" + have_files=0 + fi + else + utc_date=$(get_utc_from_string "${date}") + + # + # Check that this snapshot + # is what it claims to be... + # + if [ ${snapshot_timestamp} -lt ${seconds} ] || \ + [ ${snapshot_timestamp} -gt $((${seconds}+ 2*86400)) ]; then + + echo "Warning: Snapshot timestamp is not in acceptable period." + have_files=0 + fi fi - done - else - got_md5=1 - fi - FILE="${FILE_ORIG}" - - if (($got_md5 == 0 )); then - echo " --- No md5sum present on the mirror. (Not yet available.)" - continue - elif [ -s "${FILE}" ]; then - if eval "$md5_com"; then - echo " === snapshot $FILE is correct, using it" - sync_local - echo - echo " === Snapshot has been sync'd" - echo - exit 0 + fi + + if [ ${have_files} != 0 ]; then + break; else - rm $FILE + # + # Remove files and use + # a diffeernt mirror + # + rm -f "${file}" "${digest}" "${signature}" fi + done + + if [ ${have_files} != 0 ]; then + sync_local "${file}" && r=0 + else + echo "Warning: ${date} snapshot was not found." fi - for i in $GENTOO_MIRRORS ; do - URI="${i}/snapshots/$FILE" - rm -f "$FILE" - if (eval "$FETCHCOMMAND $wgetops") && [ -s "$FILE" ]; then - if ! eval "$md5_com"; then - echo "md5 failed on $FILE" - rm ${FILE} - continue - else - sync_local - echo - echo " *** Completed websync, please now perform a normal rsync if possible." - echo " Update is current as of the of YYYYMMDD: ${year}${month}${day}" - echo - exit 0 - fi + rm -f "${file}" "${digest}" "${signature}" + return "${r}" +} + +do_latest_snapshot() { + local attempts=-1 + local r=1 + + echo "Fetching most recent snapshot..." + + while (( ${attempts} < 40 )) ; do + local day + local month + local year + local seconds + + attempts=$(( ${attempts} + 1 )) + + utc_attempt=$(expr $(get_utc_date_in_seconds) - 86400 \* ${attempts}) + + day=$(get_date_part ${utc_attempt} "%d") + month=$(get_date_part ${utc_attempt} "%m") + year=$(get_date_part ${utc_attempt} "%Y") + utc_midnight=$(get_date_part $(expr ${utc_attempt} - ${utc_attempt} % 86400) "%s") + + if [ ${utc_midnight} -lt $(($(get_portage_timestamp)-86400)) ]; then + echo "Note: Portage content is newer than available snapshots" + echo "use --revert option to overide." + r=0 + break; + fi + + if do_snapshot 0 "${year}${month}${day}"; then + r=0 + break; fi + done + + return "${r}" +} +main() { + local arg + local do_revert=0 + local revert_date + + [ ! -d "${DISTDIR}" ] && mkdir -p "${DISTDIR}" + cd "${DISTDIR}" + + for arg in $*; do + local v=${arg#*=} + case ${arg} in + --help) + echo "usage: $0 [options]" + echo " --verbose (-v) - verbose" + echo " --revert=yyyymmdd - revert to snapshot" + exit 0 + ;; + --verbose|-v) + do_verbose=1 + ;; + --revert=*) + do_revert=1 + revert_date=${v} + ;; + *) + echo "Error: Invalid arguments" + exit 1 + ;; + esac done -done -rm -rf portage + if [ ${do_revert} != 0 ]; then + do_snapshot 1 "${revert_date}" + else + do_latest_snapshot + fi +} -exit 1 +main $* |