From 657d1fc484fa0b21b979525a92ace5c9e6f63da2 Mon Sep 17 00:00:00 2001 From: syncer Date: Wed, 11 Jan 2012 18:17:09 +0100 Subject: inital import --- bin/cron_wrapper.sh | 74 +++++++++++++++++++++++++++++++++++++++++++ bin/sync-archlinux.sh | 15 +++++++++ bin/sync-crux.sh | 15 +++++++++ bin/sync-dhozac.sh | 15 +++++++++ bin/sync-dslinux-svn-repo.sh | 18 +++++++++++ bin/sync-gentoo-ftp-mirror.sh | 15 +++++++++ bin/sync-gentoo-portage.sh | 16 ++++++++++ bin/sync-openbsd-cvs-repo.sh | 30 ++++++++++++++++++ bin/sync-openbsd-ftp-tree.sh | 24 ++++++++++++++ bin/sync-osm-planet.sh | 64 +++++++++++++++++++++++++++++++++++++ bin/sync-siduction.sh | 16 ++++++++++ bin/sync-tor.sh | 15 +++++++++ 12 files changed, 317 insertions(+) create mode 100755 bin/cron_wrapper.sh create mode 100755 bin/sync-archlinux.sh create mode 100755 bin/sync-crux.sh create mode 100755 bin/sync-dhozac.sh create mode 100755 bin/sync-dslinux-svn-repo.sh create mode 100755 bin/sync-gentoo-ftp-mirror.sh create mode 100755 bin/sync-gentoo-portage.sh create mode 100755 bin/sync-openbsd-cvs-repo.sh create mode 100755 bin/sync-openbsd-ftp-tree.sh create mode 100755 bin/sync-osm-planet.sh create mode 100755 bin/sync-siduction.sh create mode 100755 bin/sync-tor.sh (limited to 'bin') diff --git a/bin/cron_wrapper.sh b/bin/cron_wrapper.sh new file mode 100755 index 0000000..4e25802 --- /dev/null +++ b/bin/cron_wrapper.sh @@ -0,0 +1,74 @@ +#!/bin/bash +## +## Wrapper Script for Cronscripts +## Log all errors and stdout and sends only once a day an email +## +## (c) Alexander Sulfrian, Sep 2008 +## + +echo_error() { + echo "$@" 1>&2 +} + +create_and_check_log() { + if [ ! -e "${1}" ] + then + touch "${1}" + fi + + if [ ! -w "${1}" ] + then + echo_error "Unable to open log: ${1}" + exit 2 + fi +} + +if [ -z "${1}" -o ! -x "${1}" ] +then + echo_error + echo_error " Usage:" + echo_error " ${0} filename [parameter]" + echo_error + echo_error " The filename have to exists an have to be executable." + echo_error + exit 1 +fi + +CRON_SCRIPT="${1}" +CRON_SCRIPT_BASENAME="$(basename ${CRON_SCRIPT} ".sh")" +CRON_SCRIPT_LOGNAME="${CRON_SCRIPT_BASENAME#sync-}" +ERROR_LOG="/var/log/sync/${CRON_SCRIPT_LOGNAME}_error.log" +LOG="/var/log/sync/${CRON_SCRIPT_LOGNAME}.log" + +create_and_check_log "${ERROR_LOG}" +create_and_check_log "${LOG}" + +ERROR_TMP=$(mktemp -t "${CRON_SCRIPT_BASENAME}.XXXXXXXX") + +# parameter eins nach vorne shiften +# damit der cronscript dateiname nicht mehr enthalten ist +shift 1 + +# log header +LOG_HEADER="-------------------- $(date) --------------------" +echo "${LOG_HEADER}" >> "${LOG}" + +# cron ausführen und output auffangen +${CRON_SCRIPT} $@ >> "${LOG}" 2> "${ERROR_TMP}" + +if [ $(grep -v "^\s*$" "${ERROR_TMP}" | wc -l) -gt 0 ] +then + + if [ "$(date -r${ERROR_LOG} +%F)" != "$(date +%F)" ] + then + cat "${ERROR_LOG}" + echo "${LOG_HEADER}" + cat "${ERROR_TMP}" + echo -n > "${ERROR_LOG}" + else + echo ${LOG_HEADER} >> "${ERROR_LOG}" + cat "${ERROR_TMP}" >> "${ERROR_LOG}" + fi +fi + +rm "${ERROR_TMP}" diff --git a/bin/sync-archlinux.sh b/bin/sync-archlinux.sh new file mode 100755 index 0000000..fd03d23 --- /dev/null +++ b/bin/sync-archlinux.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +RSYNC="/usr/bin/rsync" +OPTS="-rtlvH --delete-after --delay-updates --safe-links --max-delete=1000" # from wiki.archlinux.org +SRC="rsync://ftp5.gwdg.de/pub/linux/archlinux/" +DST="/mirror/ftp/pub/archlinux/" + +lockfile=/tmp/`basename $0`.lock + +if lockfile -r 5 $lockfile +then + ${RSYNC} ${OPTS} ${SRC} ${DST} + echo "End: "`date` + rm -f $lockfile +fi diff --git a/bin/sync-crux.sh b/bin/sync-crux.sh new file mode 100755 index 0000000..9b14f6b --- /dev/null +++ b/bin/sync-crux.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +RSYNC="/usr/bin/rsync" +OPTS="--quiet --recursive --links --perms --times --devices --delete --timeout=600" +SRC="rsync://rsync.crux.nu/crux/" +DST="/mirror/ftp/pub/crux/" + +lockfile=/tmp/`basename $0`.lock + +if lockfile -r 5 $lockfile +then + ${RSYNC} ${OPTS} ${SRC} ${DST} + echo "End: "`date` + rm -f $lockfile +fi diff --git a/bin/sync-dhozac.sh b/bin/sync-dhozac.sh new file mode 100755 index 0000000..7229e40 --- /dev/null +++ b/bin/sync-dhozac.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +RSYNC="/usr/bin/rsync" +OPTS="-Hax --delete --exclude=/plague/tmp" +SRC="rsync://rpm.hozac.com/dhozac/" +DST="/mirror/ftp/pub/dhozac/" + +lockfile=/tmp/`basename $0`.lock + +if lockfile -r 5 $lockfile +then + ${RSYNC} ${OPTS} ${SRC} ${DST} && date > "${DST}/lastsync" + echo "End: "`date` + rm -f $lockfile +fi diff --git a/bin/sync-dslinux-svn-repo.sh b/bin/sync-dslinux-svn-repo.sh new file mode 100755 index 0000000..c302be6 --- /dev/null +++ b/bin/sync-dslinux-svn-repo.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# update dslinux read-only svn mirror (scripted by stsp) + +#set -x + +#logger "`basename $0`: invoked `date`" + +ANONSVN_HOME=/raid/anonsvn +SVN_ROOTDIR=$ANONSVN_HOME/dslinux + +lockfile=/tmp/`basename $0`.lock + +if lockfile -r 5 $lockfile +then + (cd $SVN_ROOTDIR && rsync -q -z -a --delete rsync://pferd.in-berlin.de/dslinuxsvn/ .) + rm -f $lockfile +fi + diff --git a/bin/sync-gentoo-ftp-mirror.sh b/bin/sync-gentoo-ftp-mirror.sh new file mode 100755 index 0000000..74d9cdc --- /dev/null +++ b/bin/sync-gentoo-ftp-mirror.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +RSYNC="/usr/bin/rsync" +OPTS="--quiet --recursive --links --perms --times --devices --delete --timeout=600 --password-file=/home/syncer/etc/gentoo_distfiles" +SRC="gentoo@masterdistfiles.gentoo.org::gentoo" +DST="/raid/ftp/pub/gentoo/" + +lockfile=/tmp/`basename $0`.lock + +if lockfile -r 5 $lockfile +then + ${RSYNC} ${OPTS} ${SRC} ${DST} + echo "End: "`date` + rm -f $lockfile +fi diff --git a/bin/sync-gentoo-portage.sh b/bin/sync-gentoo-portage.sh new file mode 100755 index 0000000..3f8c552 --- /dev/null +++ b/bin/sync-gentoo-portage.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +RSYNC="/usr/bin/rsync" +OPTS="--quiet --recursive --links --perms --times --devices --compress --delete --timeout=600" +SRC="rsync://masterportage.gentoo.org/gentoo-portage" +DST="/raid/gentoo-portage/" + +lockfile=/tmp/`basename $0`.lock + +if lockfile -r 5 $lockfile +then + ${RSYNC} ${OPTS} ${SRC} ${DST} + echo "End: "`date` + rm -f $lockfile +fi + diff --git a/bin/sync-openbsd-cvs-repo.sh b/bin/sync-openbsd-cvs-repo.sh new file mode 100755 index 0000000..cc546ff --- /dev/null +++ b/bin/sync-openbsd-cvs-repo.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# update openbsd anoncvscvs mirror (scripted by stsp) + +#set -e + +#logger "`basename $0`: invoked `date`" + +CVS_ROOTDIR=/raid/anoncvs/OpenBSD/cvs +LOCKFILE=/tmp/`basename ${0}`.lock + +lockfile -r 3 ${LOCKFILE} || exit 1 +(cd $CVS_ROOTDIR && \ + rsync -q -a --recursive --times --links --hard-links --delete \ + --exclude=CVSROOT/history \ + --exclude=CVSROOT/writers \ + --exclude=CVSROOT/readers \ + --exclude=CVSROOT/passwd \ + rsync://rsync.de.openbsd.org/OpenBSD-CVS . ) + +# we are using our own readers, writers and passwd files. +cp /dev/null $CVS_ROOTDIR/CVSROOT/writers +echo "anoncvs" > $CVS_ROOTDIR/CVSROOT/readers +echo "anoncvs:AHDysQkJIubEc" > $CVS_ROOTDIR/CVSROOT/passwd +echo "" > $CVS_ROOTDIR/CVSROOT/config + +# One line in the modules file breaks checkouts, +# due to debian bug #226888. Comment it out. --stsp +sed -i -e 's/\(^www.*-i.*\)/#\1/' $CVS_ROOTDIR/CVSROOT/modules + +rm -f ${LOCKFILE} diff --git a/bin/sync-openbsd-ftp-tree.sh b/bin/sync-openbsd-ftp-tree.sh new file mode 100755 index 0000000..ed0ddcc --- /dev/null +++ b/bin/sync-openbsd-ftp-tree.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +# See http://www.openbsd.org/ftp.html for up-to-date mirror list! +REMOTE=rsync://ftp.eu.openbsd.org/OpenBSD/ +PUBDIR=/raid/ftp/pub/OpenBSD +LOCKFILE=/tmp/`basename ${0}`.lock + +if [ "$1" = "-v" ]; then + verbose="-v" +else + verbose="-q" +fi + +lockfile -r 3 ${LOCKFILE} || exit 1 +rsync $verbose -a -i --recursive --times --links --hard-links --delete \ + --exclude="/[2-3].[0-9]" \ + --exclude="/4.[0-8]" \ + --exclude="/cvs" \ + --exclude="/distfiles" \ + --exclude="/tmp" \ + --exclude="/README" \ + --exclude="/README~" \ + ${REMOTE} ${PUBDIR} +rm -f ${LOCKFILE} diff --git a/bin/sync-osm-planet.sh b/bin/sync-osm-planet.sh new file mode 100755 index 0000000..878ad5b --- /dev/null +++ b/bin/sync-osm-planet.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +RSYNC="/usr/bin/rsync" +SRC="rsync://planet.openstreetmap.org/planet/" +DST="/mirror/ftp/pub/openstreetmap/" +RSYNC_OPTS=( + "-Hax" + "--delete" + "--delete-excluded" + "--include=*-$(date +%y%m)*" + "--include=*-$(date -d '-1month' +%y%m)*" + "--include=*-$(date -d '-2month' +%y%m)*" + "--include=*-$(date -d '-3month' +%y%m)*" + "--include=*-latest*" + "--exclude=*" + "--filter=protect torrents/" ) + +MKTORRENT="/usr/bin/mktorrent" +TRACKER=( + "http://tracker.ipv6tracker.org:80/announce,udp://tracker.ipv6tracker.org:80/announce" + "udp://tracker.publicbt.com:80/announce,http://tracker.publicbt.com:80/announce" + "udp://tracker.openbittorrent.com:80/announce" ) +MKTORRENT_OPTS=( "-l 22" ) + +lockfile=/tmp/`basename $0`.lock + +if lockfile -r 5 $lockfile +then + ${RSYNC} "${RSYNC_OPTS[@]}" ${SRC} ${DST} + + # create torrents + for file in ${DST}/*.bz2 + do + archiv="$(basename $file)" + if [ ! -L ${file} ] + then + torrent="${DST}/torrents/${archiv}.torrent" + torrent_tmp="/tmp/${archiv}.torrent" + + if [ ! -e "${torrent}" ] + then + rm -f "${torrent_tmp}" + + TRACKER_OPT="" + for t in "${TRACKER[@]}" + do + TRACKER_OPT="${TRACKER_OPT} -a ${t}" + done + + ${MKTORRENT} ${TRACKER_OPT} "${MKTORRENT_OPTS[@]}" \ + -w "http://ftp.spline.inf.fu-berlin.de/pub/openstreetmap/${archiv}" \ + -w "http://planet.osm.org/${archiv}" \ + "${file}" -o "${torrent_tmp}" > /dev/null && \ + mv "${torrent_tmp}" "${torrent}" + fi + else + target="$(readlink ${file})" + ln -sf "$(basename ${target}).torrent" "${DST}/torrents/${archiv}.torrent" + fi + done + + echo "End: $(date)" + rm -f $lockfile +fi diff --git a/bin/sync-siduction.sh b/bin/sync-siduction.sh new file mode 100755 index 0000000..997e0ec --- /dev/null +++ b/bin/sync-siduction.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +RSYNC="/usr/bin/rsync" +#-a = -prtlgoD +OPTS="-prtlvH --delete-after --delay-updates --safe-links" +SRC="rsync://sync.siduction.org:/siduction" +DST="/mirror/ftp/pub/siduction/" + +lockfile=/tmp/`basename $0`.lock + +if lockfile -r 5 $lockfile +then + ${RSYNC} ${OPTS} ${SRC} ${DST} + echo "End: "`date` + rm -f $lockfile +fi diff --git a/bin/sync-tor.sh b/bin/sync-tor.sh new file mode 100755 index 0000000..d1df555 --- /dev/null +++ b/bin/sync-tor.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +RSYNC="/usr/bin/rsync" +OPTS="-a --quiet --delete --timeout=600" +SRC="rsync://rsync.torproject.org/tor" +DST="/mirror/ftp/pub/tor" + +lockfile=/tmp/`basename $0`.lock + +if lockfile -r 5 $lockfile +then + ${RSYNC} ${OPTS} ${SRC} ${DST} + echo "End: "`date` + rm -f $lockfile +fi -- cgit v1.2.3-1-g7c22