diff --git a/snapshots-create.sh b/snapshots-create.sh index f29dec6..d90c829 100755 --- a/snapshots-create.sh +++ b/snapshots-create.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2011-2017 Gentoo Foundation; Distributed under the GPL v2 +# Copyright 2011-2019 Gentoo Authors; Distributed under the GPL v2 # might be earlier copyright, no history available # 1) Create the tarball @@ -33,6 +33,8 @@ DELTA_FILENAME="portage-${DELTA_BASE}.tar.bz2" YESTERDAY=`/bin/date -d yesterday +%Y%m%d` FILENAME="portage-${YESTERDAY}.tar.bz2" +FILENAME_NEW="gentoo-${YESTERDAY}.tar.xz" + # Parallel compressors can use a LOT of cpu, be nice about it NICE="nice -n 10" @@ -148,6 +150,62 @@ fi write_time_log "END TARBALL $(date -u)" # end 1) +# 1b) Also create new-style tarball +write_time_log "START TARBALL_NEW $(date -u)" +if [ ! -f "${FILENAME_NEW%.xz}" ]; then + # Build exclusion list + EXCLUSION_LIST="$(mktemp -p ${TEMP} snapshot-exclude.XXXXXXXXXX)" + /usr/local/bin/mastermirror/print-exclusion-list.sh "${MASTER}" >"${EXCLUSION_LIST}" + + TAR_OPTIONS=( + # Force a small block size + --blocking-factor=1 + --record-size=512 + # GNU tar format saves approximately 1K per file in the tarball over POSIX + # format. Multiply ~170k files, and the savings are large. + --format=gnu + # Sorting by name produces consistent ordering and helps compression of + # related content. Custom ordering might further improve ordering in future + # (eg all metadata.xml first) + --sort=name + # Force ownership of content: + --owner=portage + --group=portage + # Excluded content: + --no-wildcards + --exclude-from "${EXCLUSION_LIST}" + # Do not capture any xattr/acl info at all. + --no-acls + --no-xattrs + --no-selinux + # Include a volume ID for tracing + # volume header is not supported by: + # - Docker https://bugs.gentoo.org/631644 + # - tarsync https://bugs.gentoo.org/631616 + # -V "${FILENAME%.bz2}" + # do everything relative to the destination + -C "${MASTER}" + # The . needs to match the file argument + --transform="s,^\.,${FILENAME_NEW%%.*},g" + # The operation, destination, source arguments + --create + --file ${FILENAME_NEW%.xz} + . + ) + + tar "${TAR_OPTIONS[@]}" + rc=$? + if [ $rc -ne 0 ]; then + echo "Tar run failed!" + exit 1 + fi + rm -f "${EXCLUSION_LIST}" +fi +[ ! -f "${FILENAME_NEW}.umd5sum" ] && md5sum ${FILENAME_NEW%.xz} > ${FILENAME_NEW}.umd5sum +[ ! -f "${FILENAME_NEW}" ] && ${NICE} ${XZ_PROG} -9 -e <"${FILENAME_NEW%.*}" >"${FILENAME_NEW%.*}.xz" +write_time_log "END TARBALL_NEW $(date -u)" +# end 1b) + # 2) Sanity check the tarball size and bail out if it appears abnormal. write_time_log "START SIZE SANITY $(date -u)" current_size=$(stat -c '%s' "${FILENAME}") @@ -187,7 +245,7 @@ write_time_log "END XZ $(date -u)" # 4) sign write_time_log "START SIGN $(date -u)" -for f in "${FILENAME}" "${FILENAME%.*}".xz ; do +for f in "${FILENAME}" "${FILENAME%.*}".xz "${FILENAME_NEW}"; do if [ ! -f "${UPLOAD}${f}".umd5sum ]; then cp "${FILENAME}".umd5sum "${UPLOAD}${f}".umd5sum || exit $? md5sum "$f" > "$f".md5sum || exit $?