bash script which creates snapshots and deltas

snapshots-create.sh (text/plain), 4.28 KB, created by Zac Medico on 2010-10-05 23:01:16 UTC

#!/bin/bash


###########################################
#
# DON'T CHANGE THIS FILE WITHOUT TALKING TO
# KLIEBER FIRST!!!!!!!!!!!!!!!!!!!!!!!!!!!!
#
###########################################

#
# This file creates snapshots of Portage and moves them to the correct location so they get uploaded to OSU's mirror
#
# Author: Kurt Lieber
# Version: 0.2
#
# Based on the script by Martin Holzer
#
#
# CHANGELOG
#
# 0.3 - delta integration, and better error handling.
# 0.2 - added features to create symlinks to latest version of the snapshot
# 0.1 - original version


#
# Define locations for stuff
#

MASTER="/home/gmirror/rsync" #where the master repository lives
TEMP="/home/gmirror/snapshots-tmp/" #working directory
#UPLOAD="/home/gmirror/upload/" #temp location for testing
UPLOAD="/mnt/distfiles/snapshots/" #real location
HISTORY="7" #number in dsays of previous snapshots to keep
DELTA_UPLOAD="${UPLOAD}/deltas/"

#
# used to name the file
#

DELTA_BASE=`/bin/date -d '-2 day' +%Y%m%d`
DELTA_FILENAME="portage-${DELTA_BASE}.tar.bz2"
YESTERDAY=`/bin/date -d yesterday +%Y%m%d`
FILENAME="portage-${YESTERDAY}.tar.bz2"

#
# GPG info
#

SIGNKEYID="7DDAD20D"

if [ ! -e "${UPLOAD}/${DELTA_FILENAME}" ]; then
	echo "Previous snapshot does not exist: '${UPLOAD}/${DELTA_FILENAME}'"
	exit 1
fi

#
# copy gmirror's copy of the gentoo-x86 repository to a temp location
#

#/usr/bin/sudo /bin/rm -rf ${TEMP}/portage
#/bin/mkdir -p ${TEMP}/portage
#/bin/cp -a -R ${MASTER}/* ${TEMP}/portage/ || { echo "cp failed, $?"; exit 1; }
#/usr/bin/sudo /bin/chown -R 250:250 ${TEMP}/portage
mkdir -p ${TEMP} || { echo "mkdir failed, $?"; exit 1; }
cd ${TEMP} || { echo "cd failed, $?"; exit 1; }


#
# create the tarball and move it to the right location
#

set -e
rm -rf portage
mkdir portage
DIRECTORY_TIMESTAMP_CHANGES="metadata"
for x in $(ls "${MASTER}") ; do
	[ "${x}" == "${DIRECTORY_TIMESTAMP_CHANGES}" ] && continue
	ln -s "${MASTER}"/${x} portage/${x} || { echo "ln failed, $?"; exit 1; }
done
# gnu tar-1.16 will fail with '/bin/tar: portage/metadata: file changed as we read it'
# unless we make a copy because datestamp.sh runs every minute.
for x in ${DIRECTORY_TIMESTAMP_CHANGES} ; do
	mkdir portage/${x}
	for y in $(ls "${MASTER}"/${x}) ; do
		ln -s "${MASTER}"/${x}/${y} portage/${x}/${y}
	done
done
set +e
/bin/tar --owner=portage --group=portage -h -cf ${FILENAME%.bz2} portage || { echo "tar failed, $?"; exit 1; }
rm -rf portage
#/bin/tar --exclude=CVS -cf ${FILENAME%.bz2} portage
# Disk space is limited, so it is freed ASAP.
#/usr/bin/sudo /bin/rm -rf ${TEMP}/portage
/usr/bin/md5sum ${FILENAME%.bz2} > ${FILENAME}.umd5sum
bzip2 -k9 ${FILENAME%.bz2}

# Sanity check the tarball size and bail out if it appears abnormal.
current_size=$(stat -c '%s' "${FILENAME}")
previous_size=$(stat -c '%s' "${UPLOAD}/${DELTA_FILENAME}")
if [ ${current_size} -lt ${previous_size} ]; then
	size_difference=$(expr ${previous_size} - ${current_size})
	difference_ratio=$(expr ${previous_size} / ${size_difference})
	if [ ${difference_ratio} -lt 5 ]; then
		echo "Snapshot size has decreased by more than 20% in one day!!!"
		echo "${FILENAME} ${current_size} bytes"
		echo "${DELTA_FILENAME} ${previous_size} bytes"
		exit 1
	fi
fi

/usr/bin/md5sum ${FILENAME} > ${FILENAME}.md5sum
/usr/bin/gpg --batch -u "${SIGNKEYID}" --armor --detach-sign --output ${FILENAME}.gpgsig ${FILENAME}
/bin/mv ${FILENAME} ${FILENAME}.umd5sum ${FILENAME}.md5sum ${FILENAME}.gpgsig ${UPLOAD}/

bzip2 -dkc ${UPLOAD}/${DELTA_FILENAME} > orig
PATCH=snapshot-${DELTA_BASE}-${YESTERDAY}.patch.bz2
/usr/bin/differ -f bdelta orig ${FILENAME%.bz2} ${PATCH%.bz2}
bzip2 -9 ${PATCH%.bz2}
/usr/bin/md5sum ${PATCH} > ${PATCH}.md5sum
/bin/chmod 644 ${PATCH}{,.md5sum}
/bin/mv ${PATCH}{,.md5sum} ${DELTA_UPLOAD}
/bin/rm orig ${FILENAME%.bz2}

#
# create some symlinks
#

cd ${UPLOAD}
/usr/bin/ln -sf ${FILENAME} ${UPLOAD}portage-latest.tar.bz2
/bin/rm -f ${UPLOAD}portage-latest.tar.bz2.md5sum
/bin/sed "s/${FILENAME}\$/portage-latest.tar.bz2/" ${UPLOAD}${FILENAME}.md5sum > ${UPLOAD}portage-latest.tar.bz2.md5sum
/usr/bin/ln -sf ${FILENAME}.gpgsig ${UPLOAD}portage-latest.tar.bz2.gpgsig

#
# tidy up
#

/usr/bin/find ${UPLOAD} -maxdepth 1 -mtime +${HISTORY} | /usr/bin/xargs /bin/rm -f

#clean up old snapshots.
~gmirror/scripts/clean-old-deltas.py "${DELTA_UPLOAD}" "${YESTERDAY}" $(stat -c '%s' "${UPLOAD}/${FILENAME}")