script that generates the rsync tree (calls egencache and more)

rsync-gen.sh (text/plain), 4.09 KB, created by Zac Medico on 2010-10-05 23:20:07 UTC

#!/bin/bash
echo "---------------------------------------------------" >> ~/logs/times.log
export CVS_RSH=ssh
umask 022
cd ~/gentoo-x86
source ~/.keychain/*-sh

# NOTE: The egencache --tolerant option is non-standard, so
# misc/egencache_tolerant.patch must be applied.
# TODO: Implement a monitoring system that guarantees low-latency human
# intervention when necessary, and stop using --tolerant (bug #239266).
#EGENCACHE_OPTS="--jobs=1 --load-average=1.5 --tolerant"
EGENCACHE_OPTS="--jobs=2 --load-average=1.9 --tolerant"
HOME="/home/gmirror"
PORTDIR="$HOME/gentoo-x86"
PORTAGE_DEPCACHEDIR="$HOME/depcache"
PORTAGE_CONFIGROOT="$HOME/scripts/misc/config_root"
PORTAGE_BASE_PATH="$HOME/scripts/misc/portage"
export PYTHONPATH=$PORTAGE_BASE_PATH/pym${PYTHONPATH:+:}$PYTHONPATH

#echo ">>> Updating cvs..."
echo "START CVS   $(date -u)" >> ~/logs/times.log
cvs -q update -dP >> ~/logs/cvs_up.log 2>&1 || exit 1
echo "END   CVS   $(date -u)" >> ~/logs/times.log
[ ! -d ~/gentoo-x86/metadata/cache ] && install -d ~/gentoo-x86/metadata/cache

#
# no.  this thing doesn't even fricking work.
#~/scripts/cleancache.sh
#

echo "START REGEN $(date -u)" >> ~/logs/times.log
LOG_TIMESTAMP=$(date -u '+%Y%m%d-%H%M')
REGEN_LOG_FILE=regen-run-${LOG_TIMESTAMP}.log

"$PORTAGE_BASE_PATH/bin/egencache" --update --rsync \
	--config-root="$PORTAGE_CONFIGROOT" \
	--cache-dir="$PORTAGE_DEPCACHEDIR" \
	$EGENCACHE_OPTS \
	>> ~/logs/regen/$REGEN_LOG_FILE 2>&1

rval=$?

for x in $REGEN_LOG_FILE ; do
	rm ~/logs/${x/-${LOG_TIMESTAMP}} # remove previous symlink or empty log
	if [[ -s ~/logs/regen/${x} ]] ; then
		ln -s regen/${x} ~/logs/${x/-${LOG_TIMESTAMP}} # symlink log (not empty)
	else
		mv ~/logs/regen/${x} ~/logs/${x/-${LOG_TIMESTAMP}} # move empty log
	fi
done

[ $rval = 0 ] || exit 5

find ~/logs/regen -type f -mtime +7 -print0 | xargs -0r rm
echo "END   REGEN $(date -u)" >> ~/logs/times.log
date -u > ~/gentoo-x86/metadata/timestamp
#date -R -u > ~/gentoo-x86/metadata/timestamp.chk

echo "START DTD   $(date -u)" >> ~/logs/times.log
cd ~/gentoo-x86/metadata/dtd && cvs up -dP >> ~/logs/glsa-cvs.dtd 2>&1
date -R -u > ~/gentoo-x86/metadata/dtd/timestamp.chk
echo "END   DTD   $(date -u)" >> ~/logs/times.log
echo "START GLSA  $(date -u)" >> ~/logs/times.log
cd ~/gentoo-x86/metadata/glsa && cvs up -dP >> ~/logs/glsa-cvs.log 2>&1
date -R -u > ~/gentoo-x86/metadata/glsa/timestamp.chk
echo "END   GLSA  $(date -u)" >> ~/logs/times.log
echo "START NEWS  $(date -u)" >> ~/logs/times.log
cd ~/svn/gentoo-news/ && svn up >> ~/logs/news-svn.log 2>&1
# Rebuild from scratch, in case any news items have been removed from svn.
rm -rf ~/gentoo-x86/metadata/news && mkdir ~/gentoo-x86/metadata/news
# As specified by GLEP 42, copy yyyy/mm/* out of subdirectories.
find ~/svn/gentoo-news -mindepth 2 -maxdepth 2 -type d ! -name .svn ! -path '*/.svn/*' -print | \
while read news_item ; do
	rsync -Wqa --exclude .svn --delete "${news_item}" ~/gentoo-x86/metadata/news/
done
date -R -u > ~/gentoo-x86/metadata/news/timestamp.chk
echo "END   NEWS  $(date -u)" >> ~/logs/times.log
echo "START HERDS $(date -u)" >> ~/logs/times.log
#cd ~/cvs/gentoo/misc && cvs up herds.xml >> ~/logs/herds-cvs.log 2>&1
#rsync -a ~/cvs/gentoo/misc/herds.xml ~/gentoo-x86/metadata/herds.xml > /dev/null 2>&1 || echo "rsync failed to complete for herds.xml" >&2
cd ~/cvs/gentoo/xml/htdocs/proj/en/metastructure/herds && \
	cvs up herds.xml >> ~/logs/herds-cvs.log 2>&1
rsync -a ~/cvs/gentoo/xml/htdocs/proj/en/metastructure/herds/herds.xml \
	~/gentoo-x86/metadata/herds.xml > /dev/null 2>&1 || \
	echo "rsync failed to complete for herds.xml" >&2
echo "END   HERDS $(date -u)" >> ~/logs/times.log

#echo ">>> Syncing to rsync directory..."
echo "START RSYNC $(date -u)" >> ~/logs/times.log
rsync -Wqa --exclude=CVS --exclude=/metadata/timestamp.chk --delete ~/gentoo-x86/ ~/rsync/ >& /dev/null || \
	echo "rsync failed to complete from cvs to rsynctree" >&2
echo "END   RSYNC $(date -u)" >> ~/logs/times.log

echo "START PERMS $(date -u)" >> ~/logs/times.log
chmod -R u-s,g-s ~/rsync/metadata
echo "END   PERMS $(date -u)" >> ~/logs/times.log

echo "---------------------------------------------------" >> ~/logs/times.log