|
Lines 25-30
Link Here
|
| 25 |
VERSION="0.7" |
25 |
VERSION="0.7" |
| 26 |
|
26 |
|
| 27 |
# Changelog |
27 |
# Changelog |
|
|
28 |
# version 0.7.9 2008/10/08 |
| 29 |
# - add second parameter for destination name (EAPI=2) |
| 30 |
# - quote more carefully |
| 28 |
# version 0.7.8 2007/07/13 |
31 |
# version 0.7.8 2007/07/13 |
| 29 |
# - added option -O to call of emerge when finding mirrors |
32 |
# - added option -O to call of emerge when finding mirrors |
| 30 |
# - replaced some calls of coreutils with bash-internals |
33 |
# - replaced some calls of coreutils with bash-internals |
|
Lines 326-338
Link Here
|
| 326 |
|
329 |
|
| 327 |
remove() { |
330 |
remove() { |
| 328 |
output "${GREEN}You have chosen to remove ${CYAN}$1\n" |
331 |
output "${GREEN}You have chosen to remove ${CYAN}$1\n" |
| 329 |
pushd ${DISTDIR} >/dev/null 2>&1 |
332 |
pushd -- "${DISTDIR}" >/dev/null 2>&1 |
| 330 |
removeme=true |
333 |
removeme=true |
| 331 |
for n in $(grep -v "^#" ${DO_NOT_REMOVE}) |
334 |
for n in $(grep -v "^#" ${DO_NOT_REMOVE}) |
| 332 |
do |
335 |
do |
| 333 |
grep -q $n <<< "$1" && removeme=false && output "${CYAN}${1}${RED} is not deleted, since it matches ${n} in ${DO_NOT_REMOVE}" |
336 |
grep -q $n <<< "$1" && removeme=false && output "${CYAN}${1}${RED} is not deleted, since it matches ${n} in ${DO_NOT_REMOVE}" |
| 334 |
done |
337 |
done |
| 335 |
$removeme && rm -f $1 |
338 |
$removeme && rm -f -- "$1" |
| 336 |
popd >/dev/null 2>&1 |
339 |
popd >/dev/null 2>&1 |
| 337 |
} |
340 |
} |
| 338 |
|
341 |
|
|
Lines 349-355
Link Here
|
| 349 |
-e "s/src4/srcDDD/g" \ |
352 |
-e "s/src4/srcDDD/g" \ |
| 350 |
-e "s/src5/srcEEE/g" \ |
353 |
-e "s/src5/srcEEE/g" \ |
| 351 |
-e "s/src6/srcFFF/g" \ |
354 |
-e "s/src6/srcFFF/g" \ |
| 352 |
-e "s/src7/srcGGG/g" <<< $MASK_FILENAME) |
355 |
-e "s/src7/srcGGG/g" <<< "$MASK_FILENAME") |
| 353 |
fi |
356 |
fi |
| 354 |
|
357 |
|
| 355 |
# ignore PR for src-files of firefox |
358 |
# ignore PR for src-files of firefox |
|
Lines 375-381
Link Here
|
| 375 |
-e "s/rc//g" \ |
378 |
-e "s/rc//g" \ |
| 376 |
-e "s/[\._-]//g" \ |
379 |
-e "s/[\._-]//g" \ |
| 377 |
-e "s/\+//g" \ |
380 |
-e "s/\+//g" \ |
| 378 |
-e "s/ //g" <<< $MASK_FILENAME) |
381 |
-e "s/ //g" <<< "$MASK_FILENAME") |
| 379 |
} |
382 |
} |
| 380 |
|
383 |
|
| 381 |
# create or update a config-file |
384 |
# create or update a config-file |
|
Lines 384-389
Link Here
|
| 384 |
add_to_configfile LOCAL_MIRROR "" "set this to one or more (space separated) URI ending with '/' if you want to check one or more local mirror(s) first\n# most people just leave it empty." |
387 |
add_to_configfile LOCAL_MIRROR "" "set this to one or more (space separated) URI ending with '/' if you want to check one or more local mirror(s) first\n# most people just leave it empty." |
| 385 |
add_to_configfile DELTUP_SERVER "http://linux01.gwdg.de/~nlissne/deltup.php" "deltup-server to use" |
388 |
add_to_configfile DELTUP_SERVER "http://linux01.gwdg.de/~nlissne/deltup.php" "deltup-server to use" |
| 386 |
add_to_configfile FETCH "/usr/bin/wget -t 1 --passive-ftp" "command to use for downloading" |
389 |
add_to_configfile FETCH "/usr/bin/wget -t 1 --passive-ftp" "command to use for downloading" |
|
|
390 |
add_to_configfile FETCHNAME "\$FETCH -O" "command to use for downloading; first parameter is the filename, second the url" |
| 387 |
add_to_configfile QUEUERETRY 15 "number of seconds to wait before a queued request is retried" |
391 |
add_to_configfile QUEUERETRY 15 "number of seconds to wait before a queued request is retried" |
| 388 |
add_to_configfile MAXIMUM_ACCEPTABLE_QUEUEPOS "15" "the maximum queuepos you would accept (if higher download full archive instead)" |
392 |
add_to_configfile MAXIMUM_ACCEPTABLE_QUEUEPOS "15" "the maximum queuepos you would accept (if higher download full archive instead)" |
| 389 |
add_to_configfile QUEUETIMEOUT 900 "when a dtu-request is queued - how long should we wait max. before downloading the original archive instead (in seconds)" |
393 |
add_to_configfile QUEUETIMEOUT 900 "when a dtu-request is queued - how long should we wait max. before downloading the original archive instead (in seconds)" |
|
Lines 424-437
Link Here
|
| 424 |
add_to_donotremove "^rp-pppoe" |
428 |
add_to_donotremove "^rp-pppoe" |
| 425 |
|
429 |
|
| 426 |
|
430 |
|
| 427 |
if [ -z $1 ] |
431 |
if [ -z "$1" ] |
| 428 |
then |
432 |
then |
| 429 |
COLOR=true |
433 |
COLOR=true |
| 430 |
echo -e "${YELLOW}getdelta.sh version ${VERSION}" |
434 |
echo -e "${YELLOW}getdelta.sh version ${VERSION}" |
| 431 |
echo "This script has to be called like this:" |
435 |
echo "This script has to be called like this:" |
| 432 |
echo -e "${CYAN}$0 <URI>" |
436 |
echo -e "${CYAN}$0 <URI>" |
| 433 |
echo -e "\n${YELLOW}To use it, you should just put the following line into your /etc/make.conf" |
437 |
echo -e "\n${YELLOW}To use it, you should just put the following line into your /etc/make.conf" |
| 434 |
echo -e "${GREEN}FETCHCOMMAND=\"$0 \\\${URI}\"" |
438 |
echo -e "${GREEN}FETCHCOMMAND=\"$0 \\\"\\\${URI}\\\" \\\"\\\${FILE}\\\"\"" |
| 435 |
echo -e "\n${YELLOW}There is a config-file ${CYAN}${GETDELTA_CONFIGFILE}${YELLOW} with some variables to control the behaviour of this script." |
439 |
echo -e "\n${YELLOW}There is a config-file ${CYAN}${GETDELTA_CONFIGFILE}${YELLOW} with some variables to control the behaviour of this script." |
| 436 |
echo -e "Edit it to your needs.${NORMAL}" |
440 |
echo -e "Edit it to your needs.${NORMAL}" |
| 437 |
exit 1 |
441 |
exit 1 |
|
Lines 464-473
Link Here
|
| 464 |
output "use getdelta.sh for your FETCHCOMMAND, only." && |
468 |
output "use getdelta.sh for your FETCHCOMMAND, only." && |
| 465 |
sleep 5 && exit 1 |
469 |
sleep 5 && exit 1 |
| 466 |
|
470 |
|
| 467 |
pushd $DISTDIR >/dev/null 2>/dev/null |
471 |
pushd -- "$DISTDIR" >/dev/null 2>/dev/null |
| 468 |
ORIG_URI=$1 |
472 |
ORIG_URI=$1 |
| 469 |
NEW_FILE=$(basename $ORIG_URI) |
473 |
NEW_FILE=$(basename $ORIG_URI) |
| 470 |
|
474 |
|
|
|
475 |
[ -z "${NEW_FILE}" ] && NEW_FILE=${ORIG_URI##*/} |
| 476 |
|
| 471 |
# repoman downloads metadata.dtd with FETCHCOMMAND |
477 |
# repoman downloads metadata.dtd with FETCHCOMMAND |
| 472 |
# this should not be done with getdelta - so just fetch the file and exit |
478 |
# this should not be done with getdelta - so just fetch the file and exit |
| 473 |
|
479 |
|
|
Lines 475-481
Link Here
|
| 475 |
# dtu files. |
481 |
# dtu files. |
| 476 |
if [ "${NEW_FILE}" = "metadata.dtd" ] || [ "$GETDELTA" = "0" ] |
482 |
if [ "${NEW_FILE}" = "metadata.dtd" ] || [ "$GETDELTA" = "0" ] |
| 477 |
then |
483 |
then |
| 478 |
$FETCH $@ |
484 |
$FETCHNAME "$NEW_FILE" "$ORIG_URI" |
| 479 |
exit $? |
485 |
exit $? |
| 480 |
fi |
486 |
fi |
| 481 |
|
487 |
|
|
Lines 516-522
Link Here
|
| 516 |
# |
522 |
# |
| 517 |
output "${GREEN}Searching for a previously downloaded file in ${YELLOW}${DISTDIR}\n" |
523 |
output "${GREEN}Searching for a previously downloaded file in ${YELLOW}${DISTDIR}\n" |
| 518 |
|
524 |
|
| 519 |
first_chars=$(sed 's/[[:digit:]][[:print:]]*$//' <<< $NEW_FILE) |
525 |
first_chars=$(sed 's/[[:digit:]][[:print:]]*$//' <<< "$NEW_FILE") |
| 520 |
length_first_chars=${#first_chars} |
526 |
length_first_chars=${#first_chars} |
| 521 |
[ $length_first_chars -lt 3 ] && first_chars="${NEW_FILE:0:2}" |
527 |
[ $length_first_chars -lt 3 ] && first_chars="${NEW_FILE:0:2}" |
| 522 |
|
528 |
|
|
Lines 527-533
Link Here
|
| 527 |
let len1=${#mask} |
533 |
let len1=${#mask} |
| 528 |
filelist="" |
534 |
filelist="" |
| 529 |
|
535 |
|
| 530 |
for name in $( ls ${first_chars}* 2>/dev/null ) |
536 |
for name in $( ls -- "${first_chars}"* 2>/dev/null ) |
| 531 |
do |
537 |
do |
| 532 |
mask2=$(mask_name "${name}") |
538 |
mask2=$(mask_name "${name}") |
| 533 |
# add any file, that results in the same mask or differ not more than two letters |
539 |
# add any file, that results in the same mask or differ not more than two letters |
|
Lines 582-588
Link Here
|
| 582 |
|
588 |
|
| 583 |
# find matching part of filename - first: frontmatch |
589 |
# find matching part of filename - first: frontmatch |
| 584 |
x=0; |
590 |
x=0; |
| 585 |
a=($NEW_FILE $filelist) |
591 |
a=("$NEW_FILE" $filelist) |
| 586 |
match="" |
592 |
match="" |
| 587 |
while [ -z "$match" ] |
593 |
while [ -z "$match" ] |
| 588 |
do |
594 |
do |
|
Lines 657-663
Link Here
|
| 657 |
FILE_IS_CORRUPT=false |
663 |
FILE_IS_CORRUPT=false |
| 658 |
if $CHECK_OLD_FILE |
664 |
if $CHECK_OLD_FILE |
| 659 |
then |
665 |
then |
| 660 |
file_digest=$(grep -h ${best_candidate} ${FILESDIR}/digest-* | sed -n 1p) |
666 |
file_digest=$(grep -h ${best_candidate} "${FILESDIR}"/digest-* | sed -n 1p) |
| 661 |
if [ "$file_digest" ] |
667 |
if [ "$file_digest" ] |
| 662 |
then |
668 |
then |
| 663 |
file_md5=$(cut -d ' ' -f2 <<< "$file_digest") |
669 |
file_md5=$(cut -d ' ' -f2 <<< "$file_digest") |
|
Lines 669-675
Link Here
|
| 669 |
FILE_IS_CORRUPT=true |
675 |
FILE_IS_CORRUPT=true |
| 670 |
fi |
676 |
fi |
| 671 |
else |
677 |
else |
| 672 |
if [ $(rev <<< ${best_candidate} | cut -d. -f2 | rev) = "tar" ] |
678 |
if [ $(rev <<< "${best_candidate}" | cut -d. -f2 | rev) = "tar" ] |
| 673 |
then |
679 |
then |
| 674 |
output "${YELLOW}Could not find a digest-file for ${CYAN}${best_candidate}${YELLOW}. Testing file integrity with tar.\n" |
680 |
output "${YELLOW}Could not find a digest-file for ${CYAN}${best_candidate}${YELLOW}. Testing file integrity with tar.\n" |
| 675 |
if ! tar -tf ${best_candidate} >/dev/null |
681 |
if ! tar -tf ${best_candidate} >/dev/null |
|
Lines 698-714
Link Here
|
| 698 |
|
704 |
|
| 699 |
# Remember where we are, and go to a new dir there we can work |
705 |
# Remember where we are, and go to a new dir there we can work |
| 700 |
tmp_dwn_dest="${DISTDIR}/.getdelta-`date +%N`-tmp" |
706 |
tmp_dwn_dest="${DISTDIR}/.getdelta-`date +%N`-tmp" |
| 701 |
mkdir ${tmp_dwn_dest} |
707 |
mkdir -- "${tmp_dwn_dest}" |
| 702 |
# If user abort Ctrl+C (signal 2), remove tmp-dir; enabable trap again and send it again to stop wget |
708 |
# If user abort Ctrl+C (signal 2), remove tmp-dir; enabable trap again and send it again to stop wget |
| 703 |
trap "rm -r ${tmp_dwn_dest}; trap 2; kill -2 $$" 2 |
709 |
trap 'rm -r -- "${tmp_dwn_dest}"; trap 2; kill -2 '$$ 2 |
| 704 |
pushd ${tmp_dwn_dest} >/dev/null 2>&1 |
710 |
pushd -- "${tmp_dwn_dest}" >/dev/null 2>&1 |
| 705 |
|
711 |
|
| 706 |
# thanks to MATSUI Fe2+ Tetsushi for idea and patch |
712 |
# thanks to MATSUI Fe2+ Tetsushi for idea and patch |
| 707 |
FILESIZE=$(stat -c %s "${DISTDIR}/${best_candidate}") |
713 |
FILESIZE=$(stat -c %s "${DISTDIR}/${best_candidate}") |
| 708 |
let TIMELIMIT=${FILESIZE}/${BANDWIDTH} |
714 |
let TIMELIMIT=${FILESIZE}/${BANDWIDTH} |
| 709 |
[[ $TIMELIMIT -lt $QUEUETIMEOUT ]] && QUEUETIMEOUT=$TIMELIMIT |
715 |
[[ $TIMELIMIT -lt $QUEUETIMEOUT ]] && QUEUETIMEOUT=$TIMELIMIT |
| 710 |
|
716 |
if $FETCH "${DELTUP_SERVER}${query}" -O ${best_candidate}-${NEW_FILE}.dtu |
| 711 |
if $FETCH "${DELTUP_SERVER}${query}" |
|
|
| 712 |
then |
717 |
then |
| 713 |
# thanks to deelkar for this much more elegant solution to the "broken pipe" problem with "head -n1" |
718 |
# thanks to deelkar for this much more elegant solution to the "broken pipe" problem with "head -n1" |
| 714 |
GOTFILE=$(ls -c | sed -n 1p) |
719 |
GOTFILE=$(ls -c | sed -n 1p) |
|
Lines 718-732
Link Here
|
| 718 |
# may have sento to us. |
723 |
# may have sento to us. |
| 719 |
|
724 |
|
| 720 |
# first: the request have been queued |
725 |
# first: the request have been queued |
| 721 |
if [ "${GOTFILE}" = "deltup-queued" ] |
726 |
# if [ "${GOTFILE}" = "deltup-queued" ] |
|
|
727 |
if [[ "$(file -bi ${GOTFILE}|cut -d ";" -f1)" != "application/x-bzip2" && "$(cat ${GOTFILE}|cut -d "," -f1)" != "sorry" ]] |
| 722 |
then |
728 |
then |
| 723 |
let QTMOUT=$(date +%s)+QUEUETIMEOUT |
729 |
let QTMOUT=$(date +%s)+QUEUETIMEOUT |
| 724 |
while [ -f deltup-queued ] |
730 |
while [[ "$(file -bi ${GOTFILE}|cut -d ";" -f1)" != "application/x-bzip2" && "$(cat ${GOTFILE}|cut -d "," -f1)" != "sorry" ]] |
| 725 |
do |
731 |
do |
| 726 |
output "${GREEN}destination file: ${CYAN}${NEW_FILE}\n" |
732 |
output "${GREEN}destination file: ${CYAN}${NEW_FILE}\n" |
| 727 |
output "${YELLOW}$(cat deltup-queued)" |
733 |
output "${YELLOW}$(cat ${GOTFILE})" |
| 728 |
QUEUEPOS=$(grep "has been queued" deltup-queued | cut -d. -f2 | cut -d")" -f1) |
734 |
QUEUEPOS=$(grep "has been queued" ${GOTFILE} | cut -d. -f2 | cut -d")" -f1) |
| 729 |
rm -f deltup-queued |
|
|
| 730 |
TSTAMP=$(date +%s) |
735 |
TSTAMP=$(date +%s) |
| 731 |
if ((TSTAMP<QTMOUT)) && ((QUEUEPOS<=MAXIMUM_ACCEPTABLE_QUEUEPOS)) |
736 |
if ((TSTAMP<QTMOUT)) && ((QUEUEPOS<=MAXIMUM_ACCEPTABLE_QUEUEPOS)) |
| 732 |
then |
737 |
then |
|
Lines 741-747
Link Here
|
| 741 |
sleep 1 |
746 |
sleep 1 |
| 742 |
done |
747 |
done |
| 743 |
echo -n -e "${NORMAL}" |
748 |
echo -n -e "${NORMAL}" |
| 744 |
$FETCH "${DELTUP_SERVER}${query}" |
749 |
$FETCH "${DELTUP_SERVER}${query}" -O ${best_candidate}-${NEW_FILE}.dtu |
| 745 |
GOTFILE=$(ls -c | sed -n 1p) |
750 |
GOTFILE=$(ls -c | sed -n 1p) |
| 746 |
else |
751 |
else |
| 747 |
if ((TSTAMP>=QTMOUT)) |
752 |
if ((TSTAMP>=QTMOUT)) |
|
Lines 761-780
Link Here
|
| 761 |
done |
766 |
done |
| 762 |
fi |
767 |
fi |
| 763 |
|
768 |
|
| 764 |
if [ -f ${best_candidate}-${NEW_FILE}.failed ] |
769 |
if [ "$(cat ${GOTFILE}|cut -d "," -f1)" = "sorry" ] |
| 765 |
then |
770 |
then |
| 766 |
output "\n${RED}The server could not build the dtu-file for ${NEW_FILE}\n" |
771 |
output "\n${RED}The server could not build the dtu-file for ${NEW_FILE}\n" |
| 767 |
output "${YELLOW}reason:\n${RED}$(cat ${best_candidate}-${NEW_FILE}.failed)\n" |
772 |
output "${YELLOW}reason:\n${RED}$(cat ${GOTFILE})\n" |
| 768 |
rm -rf ${best_candidate}-${NEW_FILE}.failed |
773 |
rm -rf -- "${GOTFILE}" |
| 769 |
fi |
774 |
fi |
| 770 |
|
775 |
|
| 771 |
if [ -f ${best_candidate}-${NEW_FILE}.dtu ] |
776 |
if [ -f "${best_candidate}-${NEW_FILE}.dtu" ] |
| 772 |
then |
777 |
then |
| 773 |
output "${GREEN}Successfully fetched the dtu-file - let's build ${NEW_FILE}...\n" |
778 |
output "${GREEN}Successfully fetched the dtu-file - let's build ${NEW_FILE}...\n" |
| 774 |
downloadsize=$(stat -c %s ${best_candidate}-${NEW_FILE}.dtu) |
779 |
downloadsize=$(stat -c %s "${best_candidate}-${NEW_FILE}.dtu") |
| 775 |
if deltup -p -v -D ${DISTDIR} ${best_candidate}-${NEW_FILE}.dtu |
780 |
if deltup -p -v -D "${DISTDIR}" "${best_candidate}-${NEW_FILE}.dtu" |
| 776 |
then |
781 |
then |
| 777 |
newsize=$(stat -c %s ${NEW_FILE}) |
782 |
newsize=$(stat -c %s "${NEW_FILE}") |
| 778 |
let savedsize=${newsize}-${downloadsize} |
783 |
let savedsize=${newsize}-${downloadsize} |
| 779 |
let percent=${savedsize}*100/${newsize} |
784 |
let percent=${savedsize}*100/${newsize} |
| 780 |
unit="bytes" |
785 |
unit="bytes" |
|
Lines 791-798
Link Here
|
| 791 |
esac |
796 |
esac |
| 792 |
output "${YELLOW}This dtu-file saved ${UCOLOR}${savedsize} ${unit} (${percent}%)${YELLOW} download size.\n" |
797 |
output "${YELLOW}This dtu-file saved ${UCOLOR}${savedsize} ${unit} (${percent}%)${YELLOW} download size.\n" |
| 793 |
fi |
798 |
fi |
| 794 |
mv -f ${NEW_FILE} ${DISTDIR} && |
799 |
mv -f -- "${NEW_FILE}" "${DISTDIR}" && |
| 795 |
${REMOVE_OLD} && remove "${best_candidate}" |
800 |
${REMOVE_OLD} && remove ${best_candidate} |
| 796 |
fi |
801 |
fi |
| 797 |
|
802 |
|
| 798 |
fi # if $FETCH "${DELTUP_SERVER}${query}" |
803 |
fi # if $FETCH "${DELTUP_SERVER}${query}" |
|
Lines 812-818
Link Here
|
| 812 |
read |
817 |
read |
| 813 |
fi |
818 |
fi |
| 814 |
popd >/dev/null 2>&1 |
819 |
popd >/dev/null 2>&1 |
| 815 |
rm -rf ${tmp_dwn_dest} |
820 |
rm -rf -- "${tmp_dwn_dest}" |
| 816 |
#stop respond to trap2 |
821 |
#stop respond to trap2 |
| 817 |
trap 2 |
822 |
trap 2 |
| 818 |
fi # if ! FILE_IS_CORRUPT |
823 |
fi # if ! FILE_IS_CORRUPT |
|
Lines 824-833
Link Here
|
| 824 |
|
829 |
|
| 825 |
# Ok, once we are here, we should have got the delta (and used it) |
830 |
# Ok, once we are here, we should have got the delta (and used it) |
| 826 |
# or we still have to download the full file |
831 |
# or we still have to download the full file |
| 827 |
if ! [ -f ${DISTDIR}/${NEW_FILE} ] |
832 |
if ! [ -f "${DISTDIR}/${NEW_FILE}" ] |
| 828 |
then |
833 |
then |
| 829 |
output "${RED}The dtu could not be fetched,${YELLOW} downloading full file from original URL\n" |
834 |
output "${RED}The dtu could not be fetched,${YELLOW} downloading full file from original URL\n" |
| 830 |
$FETCH $ORIG_URI |
835 |
$FETCHNAME "$NEW_FILE" "$ORIG_URI" |
| 831 |
# remember we had a fallback to use correct exitcode for portage |
836 |
# remember we had a fallback to use correct exitcode for portage |
| 832 |
FALLBACK=$? |
837 |
FALLBACK=$? |
| 833 |
fi |
838 |
fi |
