1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
|
#!/bin/bash
# Copyright 1999-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
# Author: Brian Harring <ferringb@gentoo.org>, karltk@gentoo.org originally.
# Rewritten from the old, Perl-based emerge-webrsync script
#
# gpg key import
# KEY_ID=0x96D8BF6D
# gpg --homedir /etc/portage/gnupg --keyserver subkeys.pgp.net --recv-keys $KEY_ID
# gpg --homedir /etc/portage/gnupg --edit-key $KEY_ID trust
#
argv0=$0
# error echos
eecho() { echo "${argv0##*/}: error: $*" 1>&2 ; }
#-------------------
#initialization
#------------------
# Use portageq from the same directory/prefix as the current script, so
# that we don't have to rely on PATH including the current EPREFIX.
scriptpath=${BASH_SOURCE[0]}
if [ -x "${scriptpath%/*}/portageq" ]; then
portageq=${scriptpath%/*}/portageq
elif type -P portageq > /dev/null ; then
portageq=portageq
else
eecho "could not find 'portageq'; aborting"
exit 1
fi
eval $("${portageq}" envvar -v DISTDIR EPREFIX FEATURES \
FETCHCOMMAND GENTOO_MIRRORS \
PORTAGE_BIN_PATH PORTAGE_GPG_DIR \
PORTAGE_NICENESS PORTAGE_RSYNC_EXTRA_OPTS \
PORTAGE_RSYNC_OPTS PORTAGE_TMPDIR PORTDIR \
USERLAND http_proxy ftp_proxy)
export http_proxy ftp_proxy
source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1
if [ -z "$NICENESS_PULLED" ]; then
if [ -n "${PORTAGE_NICENESS}" ]; then
export NICENESS_PULLED=asdf
exec nice -n "${PORTAGE_NICENESS}" "$0" "$@"
echo "failed setting PORTAGE_NICENESS to '$PORTAGE_NICENESS', disabling"
fi
fi
STATE_DIR="/var/delta-webrsync/"
# hack. bug 92224
if [ "${FETCHCOMMAND/getdelta.sh}" != "${FETCHCOMMAND}" ]; then
# evil evil evil evil
eval "$(grep "^FETCHCOMMAND=" "${EPREFIX}/usr/share/portage/config/make.globals")"
fi
unset f
unset IFS
MUST_SYNC='1'
unset PUKE_HELP wgetops
for x in $*; do
case "${x}" in
-q|--quiet)
PORTAGE_QUIET=1
continue
;;
esac
if [[ $x == "-u" ]]; then
MUST_SYNC=''
elif [[ $x == "-k" ]]; then
KEEP_OLDIES='asdf'
elif [[ $x == "-h" ]]; then
PUKE_HELP=1
elif [[ $x == "-v" ]]; then
wgetops=
else
PUKE_HELP=1
echo "$x isn't a valid arg. bailing."
fi
if [[ -n $PUKE_HELP ]]; then
echo "-u for upgrade; sync only if new snapshots are found"
echo "-k for keep; keep old tree snapshots around"
exit -1
fi
done
if [[ ! -d $STATE_DIR ]]; then
echo "$STATE_DIR doesn't exist. don't have the ability to compensate for compressor differences without it!"
exit -2
fi
if has webrsync-gpg ${FEATURES} ; then
WEBSYNC_VERIFY_SIGNATURE=1
else
WEBSYNC_VERIFY_SIGNATURE=0
fi
if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 -a -z "${PORTAGE_GPG_DIR}" ]; then
eecho "please set PORTAGE_GPG_DIR in make.conf"
exit 1
fi
[[ -d ${PORTDIR} ]] || mkdir -p "${PORTDIR}"
if [[ ! -w ${PORTDIR} ]] ; then
eecho "PORTDIR is not writable: ${PORTDIR}"
exit 1
fi
[[ -d ${DISTDIR} ]] || mkdir -p "${DISTDIR}"
if [[ ! -w ${DISTDIR} ]] ; then
eecho "DISTDIR is not writable: ${DISTDIR}"
exit 1
fi
[[ -d ${PORTAGE_TMPDIR}/portage ]] || mkdir -p "${PORTAGE_TMPDIR}/portage"
TMPDIR=$(mktemp -d "${PORTAGE_TMPDIR}/portage/delta-webrsync-XXXXXX")
if [[ ! -w ${TMPDIR} ]] ; then
eecho "TMPDIR is not writable: ${TMPDIR}"
exit 1
fi
cd "$DISTDIR"
found=0
if [ "${wgetops-UNSET}" == "unset" ]; then
#this sucks. probably better to do 1> /dev/null
#that said, waiting on the refactoring.
if [ "${FETCHCOMMAND/wget}" != "${FETCHCOMMAND}" ]; then
wgetops="-q"
elif [ "${FETCHCOMMAND/curl}" != "${FETCHCOMMAND}" ]; then
wgetops="-s -f"
fi
fi
if type -p md5sum > /dev/null; then
md5_com='md5sum -c "${MD5_LOC}" &> /dev/null'
elif type -p md5 > /dev/null; then
md5_com='[ "$(md5 -q ${FILE})" == "$(cut -d \ -f 1 ${MD5_LOC})" ]'
else
echo "warning, unable to do md5 verification of the snapshot!"
echo "no suitable md5/md5sum binary was found!"
md5_com='true'
fi
#---------------
#funcs
#---------------
cleanse_state_dir() {
[[ ${STATE_DIR:-/} != '/' ]] && rm -f "${STATE_DIR}"/* &> /dev/null
}
full_version_attempt() {
local FILE file_exists
echo "Fetching most recent snapshot"
declare -i attempts=-1
while (( $attempts < 40 )) ; do
unset file_exists
attempts=$(( attempts + 1 ))
#this too, sucks. it works in the interim though.
if [ "$USERLAND" == "BSD" ] || [ "$USERLAND" == "Darwin" ] ; then
daysbefore=$(expr $(date +"%s") - 86400 \* $attempts)
day=$(date -r $daysbefore +"%d")
month=$(date -r $daysbefore +"%m")
year=$(date -r $daysbefore +"%Y")
else
day=$(date -d "-$attempts day" +"%d")
month=$(date -d "-$attempts day" +"%m")
year=$(date -d "-$attempts day" +"%Y")
fi
FILE="portage-${year}${month}${day}.tar.bz2"
echo "Attempting to fetch file dated: ${year}${month}${day}"
got_md5=0
if [ ! -e "${FILE}.md5sum" ]; then
fetch_from_mirrors "/snapshots/${FILE}.md5sum" "${FILE}.md5sum"
got_md5=$?
else
file_exists='asdf'
got_md5=0
fi
if [[ $got_md5 != 0 ]]; then
echo " --- No md5sum present on the mirror. (Not yet available.)"
continue
elif [ -s "${FILE}" ]; then
if verify_md5_file "$FILE"; then
echo " === snapshot $FILE is correct, using it"
if [[ -n $MUST_SYNC ]] || [[ -z file_exists ]]; then
sync_local "${FILE}"
echo
echo " === Snapshot has been sync'd"
echo
else
echo
echo "skipped sync"
echo
fi
exit 0
else
echo "md5 on ${FILE} failed, removing it and starting anew"
rm -f "${FILE}" &> /dev/null
fi
fi
if fetch_from_mirrors "/snapshots/${FILE}" "${FILE}"; then
if ! verify_md5_file "$FILE"; then
echo "md5 failed on $FILE"
rm -f "${FILE}" &> /dev/null
continue
else
sync_local "${FILE}"
cleanse_state_dir
echo
echo " *** Completed websync, please now perform a normal rsync if possible."
echo " Update is current as of YYYYMMDD: ${year}${month}${day}"
echo
exit 0
fi
fi
done
exit 1
}
sync_local() {
local FILE flags
FILE="$1"
if [ "${FILE/\/}" == "${FILE}" ]; then
FILE="${DISTDIR}/${FILE}";
fi
echo Syncing local tree...
local ownership="portage:portage"
if has usersync ${FEATURES} ; then
case "${USERLAND}" in
BSD)
ownership=$(stat -f '%Su:%Sg' "${PORTDIR}")
;;
*)
ownership=$(stat -c '%U:%G' "${PORTDIR}")
;;
esac
fi
if type -p tarsync &> /dev/null; then
echo "apparently you have tarsync installed. using it."
local chown_opts="-o ${ownership%:*} -g ${ownership#*:}"
chown ${ownership} "${PORTDIR}" > /dev/null 2>&1 || chown_opts=""
if ! tarsync "${FILE}" "${PORTDIR}" -v -s 1 ${chown_opts} -e /distfiles -e /packages -e /local; then
echo "ok, tarsync failed. that's teh suck :/"
exit 6
fi
else
cd "${TMPDIR}" || die "couldn't cd to tmpdir, ${TMPDIR}!?"
flags="xf"
if [ "${FILE%.bz2}" != "${FILE}" ]; then
flags="jxf"
fi
if ! tar ${flags} "$FILE"; then
echo "Tar failed to extract the image. Please review the output."
echo "Executed command: tar jxf $FILE"
exit 1
fi
# Make sure user and group file ownership is appropriate
chown ${ownership} portage > /dev/null 2>&1 && \
chown -R ${ownership} portage
cd portage
rsync ${PORTAGE_RSYNC_OPTS} ${PORTAGE_RSYNC_EXTRA_OPTS} . "${PORTDIR%%/}"
cd ..
echo "cleaning up"
rm -rf portage
fi
if has metadata-transfer ${FEATURES} ; then
echo "transferring metadata/cache"
emerge --metadata
fi
local post_sync=/etc/portage/bin/post_sync
[[ -x "${post_sync}" ]] && ${post_sync}
}
fetch_from_mirrors() {
local i URI FILE MIRRORS
if [[ "$#" == 3 ]]; then
MIRRORS="${3}"
else
MIRRORS=$GENTOO_MIRRORS
fi
FILE="$2"
for i in $MIRRORS ; do
URI="${i%/}/${1#/}"
if (eval "$FETCHCOMMAND $wgetops") && [ -s "${FILE}" ]; then
return 0
else
rm -f "${FILE}" &> /dev/null
fi
done
return 1
}
verify_md5_file() {
local FILE MD5_LOC CUR
FILE="$1"
if [[ $# == 2 ]]; then
MD5_LOC="$2"
else
MD5_LOC="$(pwd)/$1.md5sum"
fi
if [ "${FILE/*\/}" != "$1" ]; then
CUR="$(pwd)"
cd "$(dirname ${FILE})"
FILE="$(basename ${FILE})"
fi
if eval "$md5_com"; then
[ -n "${CUR}" ] && cd "${CUR}"
return 0
else
[ -n "${CUR}" ] && cd "${CUR}"
return 1
fi
}
check_file_signature() {
local signature="$1"
local file="$2"
local r=1
if [[ ${WEBSYNC_VERIFY_SIGNATURE} != 0 ]] ; then
vecho "Checking signature ..."
if type -P gpg > /dev/null; then
gpg --homedir "${PORTAGE_GPG_DIR}" --verify "$signature" "$file" && r=0
else
eecho "cannot check signature: gpg binary not found"
exit 1
fi
else
r=0
fi
return "${r}"
}
#--------------------
#inline actual script
#--------------------
if ! type -p patcher &> /dev/null; then
echo "!!!"
echo "!!! cannot find patcher, did you emerge dev-util/diffball?"
echo "!!! lack of patcher == have to do full fetch"
echo "!!!"
sleep 10
full_version_attempt
fi
echo "Looking for available base versions for a delta"
#note we're already in distdir
unset base_version
# portage-snapshots in reverse order.
# icky.
unset dfile
potentials="$(ls -1 portage-2[[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]].tar.bz2 ${STATE_DIR}/portage-2[[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]].tar.bz2 2> /dev/null | sed -e 's:^.*/::' | sort -r)"
for basef in ${potentials}; do
chksum=''
found="dar"
if [ -e "${STATE_DIR}/${basef}.md5sum" ]; then
chksum="${STATE_DIR}/${basef}.md5sum"
elif [ -e "${basef}.md5sum" ]; then
chksum="${DISTDIR}/${basef}.md5sum"
else
echo "attempting to get md5sum for $basef"
if ! fetch_from_mirrors "/snapshots/${basef}.md5sum" "${basef}.md5sum"; then
echo "can't get md5 for ${basef}"
continue
fi
chksum="${basef}.md5sum"
fi
if [ -e "${basef}" ]; then
dfile="${DISTDIR}/${basef}"
else
dfile="${STATE_DIR}/${basef}"
fi
if ! verify_md5_file "${dfile}" "${chksum}"; then
echo "found a stale snapshot. cleansing"
rm -f "${dfile}" &> /dev/null
rm -f "${chksum}.md5sum" &> /dev/null
dar=""
else
base_version="${basef}"
break
fi
done
#by this point, we either have a base_version, or we don't.
if [[ -z ${base_version} ]]; then
echo "no base found. resorting to pulling a full version"
full_version_attempt
fi
#we have a md5 verified base. now we get the patch.
base_date="${base_version%.tar.bz2}"
base_date="${base_date#portage-}"
# we now have yyyymmdd
unset patch_dates
if [ "$USERLAND" == "BSD" ] || [ "$USERLAND" == "Darwin" ] ; then
daysbefore=$(expr $(date +"%s") - 86400 \* $attempts)
day=$(date -r $daysbefore +"%d")
month=$(date -r $daysbefore +"%m")
year=$(date -r $daysbefore +"%Y")
else
day=$(date -d "-$attempts day" +"%d")
month=$(date -d "-$attempts day" +"%m")
year=$(date -d "-$attempts day" +"%Y")
fi
#todays_date="${year}${month}${day}"
next_date() {
local year day month
year="${1:0:4}"
month="${1:4:2}"
day="${1:6:2}"
# if [[ "${USERLAND}" == "BSD" ]] || [[ "${USERLAND}" == "Darwin" ]]; then
# else
date -d "$year/$month/$day +1 day" +"%Y%m%d"
# fi
}
patches=''
echo "fetching patches"
fetched='asdf'
while [[ -n ${fetched} ]]; do
next_day=$(next_date ${base_date})
# if we can't get a *single* patch or md5, even one missing, do full.
p="snapshot-${base_date}-${next_day}.patch.bz2"
if [[ ! -e ${p}.md5sum ]] && ! fetch_from_mirrors "/snapshots/deltas/${p}.md5sum" "${p}.md5sum"; then
echo "failed fetching ${p}.md5sum"
fetched=''
break
fi
fetch="yes"
if [[ -e ${p} ]]; then
if ! verify_md5_file "${p}"; then
rm -f "${p}" &> /dev/null
else
fetch=""
fi
fi
if [[ -n $fetch ]]; then
if ! fetch_from_mirrors "/snapshots/deltas/${p}" "${p}"; then
echo "failed fetching ${p}"
fetched=''
fi
fi
if [[ -z ${fetched} ]]; then
break
fi
if ! verify_md5_file "${p}"; then
echo "md5 failed on ${p}"
fetched=''
break
fi
patches="${patches} ${p}"
base_date="${next_day}"
done
final_date=${base_date}
if [[ -z $patches ]]; then
echo "no patches found? up to date?"
if [[ -n $MUST_SYNC ]]; then
echo "syncing with existing file"
sync_local "${dfile}"
else
:
fi
exit $?
fi
unset got_umd5
#grab the md5 for later usage.
if [[ ! -e portage-${final_date}.tar.bz2.md5sum ]] && ! fetch_from_mirrors "/snapshots/portage-${final_date}.tar.bz2.md5sum" "portage-${final_date}.tar.bz2.md5sum"; then
echo "warning... couldn't grab the md5sum for ${final_date}. which is odd"
echo "thus, bailing (sorry)"
exit 5
else
if [[ ! -e portage-${final_date}.tar.bz2.umd5sum ]] && ! fetch_from_mirrors "/snapshots/portage-${final_date}.tar.bz2.umd5sum" "portage-${final_date}.tar.bz2.umd5sum"; then
if ! fetch_from_mirrors "/snapshots/portage-${final_date}.tar.bz2.umd5sum" "portage-${final_date}.tar.bz2.umd5sum"; then
echo "couldn't grab umd5sum (uncompressed md5sum) for ${final_date}."
echo "can't compensate for bzip2 version differences iow."
else
got_umd5=1
fi
else
got_umd5=1
fi
fi
if [[ ${WEBSYNC_VERIFY_SIGNATURE} == 1 && ! -e portage-${final_date}.tar.bz2.gpgsig ]] && \
! fetch_from_mirrors "/snapshots/portage-${final_date}.tar.bz2.gpgsig" "portage-${final_date}.tar.bz2.gpgsig" ; then
echo "warning... couldn't grab the gpgsig for ${final_date}. which is odd"
echo "thus, bailing (sorry)"
exit 5
fi
# got our patches.
if ! patcher -v "${dfile}" ${patches} "${TMPDIR}/portage-${final_date}.tar"; then
echo "reconstruction failed (contact the author with the error from the reconstructor please)"
rm -f "${TMPDIR}/portage-${final_date}.tar"
full_version_attempt
fi
verified=0
if [[ -n $got_umd5 ]]; then
echo "verifying uncompressed md5"
if ! verify_md5_file "${TMPDIR}/portage-${final_date}.tar" "${DISTDIR}/portage-${final_date}.tar.bz2.umd5sum"; then
echo "uncompressed verification failed. This means either you found a bug in diffball, or something odd is going on"
echo "with upstream patch generation"
echo "trying md5sum next, which probably will fail."
else
verified="1"
fi
fi
unset need_last_sync
if [ "$verified" == "1" ]; then
echo "recompressing. (backgrounding)"
need_last_sync="dar"
if [[ ${WEBSYNC_VERIFY_SIGNATURE} == 1 ]] ; then
# BUG: Signature verification will fail if the local bzip2
# program does not produce output that is perfectly identical
# to the bzip2 program used to compress the signed tar file.
bzip2 -vk9 "${TMPDIR}/portage-${final_date}.tar"
check_file_signature "${DISTDIR}/portage-${final_date}.tar.bz2.gpgsig" "${TMPDIR}/portage-${final_date}.tar.bz2" || exit 1
else
bzip2 -vk9 "${TMPDIR}/portage-${final_date}.tar" &
fi
echo "beginning update to the tree"
sync_local "${TMPDIR}/portage-${final_date}.tar"
echo "doing final md5 stuff"
wait
# bzip2 is finished now.
rm -f "${TMPDIR}/portage-${final_date}.tar"
else
echo "recompressing."
bzip2 -v9 "${TMPDIR}/portage-${final_date}.tar.bz2"
fi
echo "verifying generated tarball"
if ! verify_md5_file "${TMPDIR}/portage-${final_date}.tar.bz2" "${DISTDIR}/portage-${final_date}.tar.bz2.md5sum"; then
if [[ -z $verified ]]; then
echo "couldn't verify the generated tarball. bug, most likely."
exit 5
fi
# hokay. md5 doesn't agree with umd5. bzip2 issue in effect.
echo "compressed md5 differs, but uncompressed md5 says it right. bzip2 version incompatability in other words"
echo "saving the md5"
if type -p md5sum &> /dev/null; then
md5sum "${TMPDIR}/portage-${final_date}.tar.bz2" | sed -e "s:${TMPDIR}/\?::" > \
"${STATE_DIR}/portage-${final_date}.tar.bz2.md5sum"
elif type -p md5 &> /dev/null; then
echo "$(md5 -q "${TMPDIR}/portage-${final_date}.tar.bz2") portage-${final_date}.tar.bz2" > \
"${STATE_DIR}/portage-${final_date}.tar.bz2.md5sum"
else
echo "couldn't find either md5 or md5sum. something is screwed... (bailing, sorry)"
exit 7
fi
mv "${DISTDIR}/portage-${final_date}.tar.bz2.umd5sum" "${TMPDIR}/portage-${final_date}.tar.bz2" "${STATE_DIR}/"
dfile="${STATE_DIR}/portage-${final_date}.tar.bz2"
else
dfile="${DISTDIR}/portage-${final_date}.tar.bz2"
mv "${TMPDIR}/portage-${final_date}.tar.bz2" "${DISTDIR}/"
fi
if [ -z "${need_last_sync}" ]; then
if [[ ${WEBSYNC_VERIFY_SIGNATURE} == 1 ]] ; then
check_file_signature "${DISTDIR}/portage-${final_date}.tar.bz2.gpgsig" "${dfile}" || exit 1
fi
echo "beginning update to the tree"
sync_local "${dfile}"
fi
if [[ -z $KEEP_OLDIES ]]; then
echo "cleansing"
for x in $potentials; do
echo "removing ${x}"
rm -f "${DISTDIR}/${x}"{,.md5sum,.umd5sum,.gpgsig} &> /dev/null
rm -f "${STATE_DIR}/${x}"{,.md5sum,.umd5sum} &> /dev/null
done
fi
rm -rf "${TMPDIR}"
echo "done."
|