Test version of mirrorselect

mirrorselect. (text/plain), 10.59 KB, created by Fred Van Andel (RETIRED) on 2003-12-01 14:40:51 UTC

#!/bin/bash
# Gentoo Mirror Selection Tool
# Copyright 2003 Mark Guertin <[e-mail]>
# Distributed under the GPL v2
# Jan 30, 2003

# Completely Re-Written May 9, 2003
# John Mylchreest <[e-mail]>

SERVERS="1"
VERSION="0.82-r2"

CHECK_VHOSTS="$(which host 2> /dev/null)"
DIALOG="$(which dialog 2>/dev/null)"

RSYNC_CONTINENTS="asia(Asia) europe(Europe) namerica(NorthAmerica) samerica(SouthAmerica)"
RSYNC_COUNTRIES="au(Australia) be(Belgium) ca(Canada) de(Germany) es(Spain) fr(France) nl(Netherlands) ro(Romania) uk(UK) us(USA)"
RSYNC_LIST="${RSYNC_CONTINENTS} ${RSYNC_COUNTRIES}"

MIRROR_LIST=""

print_usage() {
	echo "Mirrorselect Usage:"
	echo "options:"
	echo "	-a	: Automatic Mode, this will automatically"
	echo "		  select the best mirrors."
	echo "	-i	: Interactive Mode, this will present a list"
	echo "		  to make it possible to select mirrors"
	echo "		  you wish to use."
	echo "	-D	: Deep mode. This is used to give a more accurate"
	echo "		  speed test. It will download a 100k file from"
	echo "		  each server. Because of this you should only use"
	echo "		  this option if you have a good connection"
	echo "	-sX	: Specify Number of servers for Automatic Mode"
	echo "		  to select. this is only valid for download mirrors"
	echo "		  If this is not specified, a default of 1 is used."
	echo "	-r	: Rsync Mode, This will set the default type to"
	echo "		  rsync mirrors, instead of download mirrors."
	echo "		  These are the mirrors portage will sync to."
	echo "	-o	: Output Only Mode, this is especially useful"
	echo "		  when being used during installation, to redirect"
	echo "		  output to a file other than /etc/make.conf"
	echo "	-bX	: This is to be used in automatic mode"
	echo "		  and will split the hosts into blocks of X for use"
	echo "		  with netselect. This is required for certain routers"
	echo "		  which block 40+ requests at any given time."
	echo "		  Recommended parameters to pass for -b are:"
	echo "		  -a -s3 -b10"
	echo "	-c	: Prevent \$DISPLAY detection, this will force the use"
	echo "		  of dialog instead of Xdialog (if available) in Interactive"
	echo "		  mode"
	echo "	-v	: Display Mirrorselect version"
	echo
	echo "Automatic Example:"
	echo "	# mirrorselect -a -s5"
	echo "	# mirrorselect -a -s3 -b10 -o >> /mnt/gentoo/etc/make.conf"
	echo "	# mirrorselect -a -D -s4"
	echo
	echo "Interactive Example:"
	echo "	# mirrorselect -i -r"
	echo
}

sanity_check() {
	if [ -z "${MIRROR_LIST}" ] ; then
    		echo "Unable to fetch a valid list of mirrors, Update Failed." >&2
    		echo "Quitting" >&2
    		exit 1
	fi
}

grab_list() {
	local i

	echo "Downloading a list of mirrors..." >&2
	if [ -n "${RSYNC}" ] ; then
		for i in ${RSYNC_LIST}
       		do
            		i="$(echo ${i} | sed 's:(: (:' | awk '{ print $1 }')"
            		MIRROR_LIST="${MIRROR_LIST} rsync://rsync.${i}.gentoo.org/gentoo-portage"
        	done
    	else
        	MIRROR_LIST=$(wget -O - "http://www.gentoo.org/main/en/mirrors.xml" 2>/dev/null | \
		awk '/<a href=\".*tp:.*\".*\(.*\)/ { print $0 }' | grep -v "MIRRORS" | \
		tr -d '"\t' | sed -e 's:<a href=::g' -e 's:</a><br>::g' -e 's:/ftp):):' )
    	fi
}

print_percent() {
	echo -n -e "                                                                                \r" >&2
	echo -n -e "       Testing: ${2}\r" >&2
	echo -n -e "[${1}%]\r" >&2
}

strip_ipv6() {
	local i
	local IPV6_ONLY

	echo "Removing IPV6 Only Servers..." >&2
	IPV6_ONLY="$(echo $MIRROR_LIST | sed -e 's:http\::\n&:g' -e 's:ftp\::\n&:g' | grep ipv6 | cut -f1 -d">")"

	for i in ${IPV6_ONLY}
	do
		MIRROR_LIST="$(echo ${MIRROR_LIST} | sed "s|${i}||")"
	done
}

save_vhosts() {
	local i
	local X
	local COUNT
	local IPS

	for i in ${CHECKLIST}
	do	
		i="$(echo ${i} | cut -f3 -d/)"
		IPS="$(${CHECK_VHOSTS} ${i} | awk '/.*has address.*/ { print $4 }')"
		COUNT="$(echo ${IPS} | wc -w)"
		if [ ! "$((${COUNT}-1))" = 0 ] ; then
			for X in ${IPS}
			do
				HOST_IP="${HOST_IP} ${X}@${i}"
			done
		fi
	done
}

load_vhosts() {
	local X
	local CHOSEN_TMP
	local i

	for X in ${HOST_IP}
	do
		IP="$(echo ${X} | cut -f1 -d@)"
		HOST="$(echo ${X} | cut -f2 -d@)"
		SED="${SED} | sed 's|${IP}|${HOST}|g'"
	done
	
	CHOSEN=$(echo ${CHOSEN} | tr '\n' ' ')
	CHOSEN=$(eval echo ${CHOSEN} ${SED} | tr ' ' '\n' | uniq)
	#
	#	uniq doesnt actualy clear these out as much as hoped.
	#	so we need to finish the job off
	#
	
	CHOSEN_TMP="${CHOSEN}"
	CHOSEN=""
	
	for i in ${CHOSEN_TMP}
	do
		[ -z $(echo ${CHOSEN} | grep ${i}) ] && CHOSEN="${CHOSEN} ${i}"
	done
}

deeptest() {
	local YCOUNT
	local COUNT
	local TMPFILE
	local TMPFILEB
	local ETIME
	local STIME
	local PERCENT
		
	if [ -n "${RSYNC}" ] ; then
		echo "Currently RSYNC in Automatic Mode is not supported." >&2
		exit 1
	else
		CHECKLIST="$(echo $MIRROR_LIST | sed -e 's:http\::\n&:g' -e 's:ftp\::\n&:g' | cut -f1 -d">")"
		COUNT=$(echo ${CHECKLIST}|wc -w| tr -d ' ')
		
		echo "Downloading 100k segments from ${COUNT} server(s)" >&2
		TMPFILE="$(mktemp -p /tmp mirrorselect-deep-XXXXXX)"
		
		YCOUNT=0
		for i in ${CHECKLIST}
		do
			YCOUNT=$((${YCOUNT}+1))
			PERCENT=$(((${YCOUNT}*100)/${COUNT}))
			print_percent "${PERCENT}" "${i}"
			STIME=$(date +%s%N)
			$(wget -t 1 -C off -T 25 -O - ${i}/distfiles/mirrorselect-test 2>/dev/null)
			ETIME=$(date +%s%N)
			[ $? == 0 ] && $(echo "$((${ETIME}-${STIME})) ${i}" >> ${TMPFILE})
		done
		
		echo "Sorting by speed" >&2
		MIRRORS="$(sort -g < ${TMPFILE} | awk '{ print $2 }' | head -n ${SERVERS})"
		echo "Cleaning temporary files" >&2
		$(rm -Rf ${TMPFILE})
	fi
}

automatic() {
	local i
	local TMPFILE
	local CHECK_TMP

	if [ -n "${RSYNC}" ] ; then
		echo "Currently RSYNC in Automatic Mode is not supported." >&2
		exit 1
	else
		CHECKLIST="$(echo $MIRROR_LIST | sed -e 's:http\::\n&:g' -e 's:ftp\::\n&:g' | cut -f1 -d">")"
		[ -n "${CHECK_VHOSTS}" ] && save_vhosts
		
		echo "Running netselect to determine mirrors, this might take a moment..." >&2
		#
		#	thanks go to Alexey Spiridonov <[e-mail]>
		#	who pointed this out and provided the original patch
		#
		if [ -n "${BLOCK}" ] ; then
			echo "Splitting into blocks of ${BLOCK}." >&2
			echo "Please note that the smaller the block size the longer this takes..." >&2
			
			TMPFILE="$(mktemp -p /tmp mirrorselect-split-XXXXXX)"
			$(echo ${CHECKLIST} | tr ' ' '\n' | sort -u | split -l ${BLOCK} - ${TMPFILE}-)
			CHECKLIST=""
			
			for i in $(ls ${TMPFILE}-* | sed "s: :\n:g")
			do
				CHECK_TMP="$(cat ${i})"
				CHECK_TMP="$(netselect -s1 ${CHECK_TMP} | awk -F" " '{ print $2 }')"
				CHECKLIST="${CHECKLIST} ${CHECK_TMP}"
			done
			
			echo "Cleaning tmp files and preparing to select fastest servers..." >&2
			$(rm -Rf ${TMPFILE}*)
		fi

		if [ -z "${CHECKLIST}" ] ; then
			echo "Unable to get a list of mirrors even in block mode" >&2
			echo "You can try reducing the size of the blocks." >&2
			echo "If that fails please try -D." >&2
			echo "Quitting." >&2
			exit 1
		fi
		CHOSEN=$(netselect -s${SERVERS} ${CHECKLIST} | awk -F" " '{ print $2 }')
		[ -n "${CHECK_VHOSTS}" ] && load_vhosts
		
		for i in ${CHOSEN}
		do
			MIRRORS="${MIRRORS} ${i}"
		done
	fi
}

interactive() {
	local i
	local loc
	
	if [ -n "${RSYNC}" ] ; then
        	for i in ${RSYNC_LIST}
        	do
			loc="$(echo ${i} | sed 's:(: (:' | awk '{ print $2 }')"
			i="$(echo ${i} | sed 's:(: (:' | awk '{ print $1 }')"
			CHECKLIST="${CHECKLIST} rsync://rsync.${i}.gentoo.org/gentoo-portage ${loc} OFF"
		done

		MIRRORS=$(${DIALOG} --title " Gentoo RSYNC Mirrors " --radiolist "Please select your desired mirror:" 20 80 14	${CHECKLIST} 2>&1 | tr -d '"')

    	else
		CHECKLIST="$(echo $MIRROR_LIST | sed 's:http\::\n&:g' | \
		sed 's:ftp\::\n&:g' | tr '>' '(' | cut -f1,3 -d"(" | \
		tr -d ' ' | sed 's:(: (:g' | sed 's:):) OFF:g' | sed 's:/ftp):):g')"

		MIRRORS=$(${DIALOG} --title " Gentoo Download Mirrors " --checklist "Please select your desired mirror(s):" 20 80 14 ${CHECKLIST} 2>&1 | tr -d '"')
	fi
}

write_config() {
	local PATTERN

	MIRRORS="$(echo ${MIRRORS} | sed -e 's: \":\":' -e 's:\" :\":')"

	if [ -n "${RSYNC}" ] ; then
		PATTERN="SYNC"
	else   
		PATTERN="GENTOO_MIRRORS"
	fi
	
	if [ -z "${MIRRORS}" ] ; then
		echo "No mirrors have been selected. Exiting." >&2
		exit 1
	fi

        #
        # I don't know what this is, but it should stay in to prevent problems
        #
        if [ "$(echo ${MIRRORS} | grep "Can't make")" ] ; then
	        echo "Error, screen too small.  Please use at least 80x24 terminal" >&2
	        exit 1
        fi

	[ -n "${CHECK_VHOSTS}" ] && echo "VHOST entries have been removed. This list might be shorter than expected" >&2
        if [ -n "${SILENT}" ] ; then 
		echo "${PATTERN}=\"${MIRRORS}\""
		exit 0
	fi
	
	echo "Found $(echo $MIRRORS | wc -w | tr -d ' ') Mirrors: ${MIRRORS}" >&2
	echo "Updating /etc/make.conf" >&2
	#
	# Request/original patch by Bret Towe <[e-mail]>
	# fava: The patch unfortunatly killed any lines in make.conf that 
	#       cooresponded to commented out (via #) values.  The simplest 
	#       solution is to just remove the applicable (non commented) 
	#       lines and add the new ones to the end. There is no need for 
	#       any conditional statements as per the original patch
    	mv /etc/make.conf /etc/make.conf.old 
        sed "s!^[ \t]*${PATTERN}=.*!!" /etc/make.conf.old > /etc/make.conf
        echo "${PATTERN}=\"${MIRRORS}\"" >> /etc/make.conf

	#
	#	Its very unlikely that this check will ever fail anymore
	#	Although this is being kept in as a safety net.
	#
	if [ "$(grep ${PATTERN}= /etc/make.conf)" ] ; then 
		echo "Mirrors set successfully" >&2
		if [ -e /etc/make.conf.old ] ; then
			rm /etc/make.conf.old
		fi
	else
		echo "Failed updating /etc/make.conf"
		if [ -e /etc/make.conf.old ] ; then
			mv /etc/make.conf.old /etc/make.conf
			echo "Reverting to original" >&2
		fi
	fi    
}


while getopts "ais:rovb:Dc" opt
do
  case $opt in
    a     ) MODE="auto";;
    i     ) MODE="user";;
    s     ) SERVERS=$OPTARG;;
    r     ) RSYNC="1";;
    o     ) SILENT="1";;
    b     ) BLOCK=$OPTARG;;
    D     ) DEEP="1";;
    v     ) echo "Mirrorselect version: ${VERSION}"; exit 0;;
    c	  ) FORCECONSOLE="1";;
    *     ) print_usage; exit 1;;
  esac
done

if [ "$(whoami)" != "root" ] ; then 
	echo "Root access is required to run this program." >&2
	exit 1
fi

if [ -z "${MODE}" ] ; then print_usage ; exit 1 ; fi

if [ "${MODE}" = "auto" ] ; then
    grab_list
    sanity_check
    [ -z "$(portageq envvar USE | grep ' ipv6')" ] && strip_ipv6
    [ -n "${DEEP}" ] && deeptest || automatic
    write_config
else
    #
    # Original request/patch by Bryan Stine <[e-mail]>
    #
    XDIALOG="$(which Xdialog)"
    [ -n "${DISPLAY}" ] && [ -n "${XDIALOG}" ] && [ -z "${FORCECONSOLE}" ] && DIALOG="${XDIALOG}"
    grab_list
    sanity_check
    interactive
    write_config
fi

exit 0;