From d420815bc096e1eb86f9d2a90c303f0129865d6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Fri, 21 Feb 2014 00:01:06 +0100 Subject: [PATCH 1/3] Remove shallow clone support, always use full clones. As the community demanded, shallow clones are no longer supported. The eclass completely removes old ones, so that the repos are re-fetched from broken git servers with no extra action. Additionally, every branch and tag is fetched from the remote. --- eclass/git-r3.eclass | 195 +++++---------------------------------------------- 1 file changed, 18 insertions(+), 177 deletions(-) diff --git a/eclass/git-r3.eclass b/eclass/git-r3.eclass index 0341972..487b77b 100644 --- a/eclass/git-r3.eclass +++ b/eclass/git-r3.eclass @@ -8,8 +8,7 @@ # @BLURB: Eclass for fetching and unpacking git repositories. # @DESCRIPTION: # Third generation eclass for easing maitenance of live ebuilds using -# git as remote repository. The eclass supports lightweight (shallow) -# clones and bare clones of submodules. +# git as remote repository. case "${EAPI:-0}" in 0|1|2|3|4|5) @@ -83,18 +82,6 @@ fi # # EGIT_CHECKOUT_DIR=${WORKDIR}/${P} -# @ECLASS-VARIABLE: EGIT_NONSHALLOW -# @DEFAULT_UNSET -# @DESCRIPTION: -# Disable performing shallow fetches/clones. Shallow clones have -# a fair number of limitations. Therefore, if you'd like the eclass to -# perform complete clones instead, set this to a non-null value. -# -# This variable can be set in make.conf and ebuilds. The make.conf -# value specifies user-specific default, while ebuilds may use it -# to force deep clones when the server does not support shallow clones -# (e.g. Google Code). - # @FUNCTION: _git-r3_env_setup # @INTERNAL # @DESCRIPTION: @@ -217,14 +204,13 @@ _git-r3_set_gitdir() { fi addwrite "${EGIT3_STORE_DIR}" + if [[ -e ${GIT_DIR}/shallow ]]; then + einfo "${GIT_DIR} was a shallow clone, recreating..." + rm -r "${GIT_DIR}" || die + fi if [[ ! -d ${GIT_DIR} ]]; then mkdir "${GIT_DIR}" || die git init --bare || die - - if [[ ! ${EGIT_NONSHALLOW} ]]; then - # avoid auto-unshallow :) - touch "${GIT_DIR}"/shallow || die - fi fi } @@ -267,93 +253,6 @@ _git-r3_set_submodules() { done < <(echo "${data}" | git config -f /dev/fd/0 -l || die) } -# @FUNCTION: _git-r3_smart_fetch -# @USAGE: ... -# @DESCRIPTION: -# Try fetching without '--depth' and switch to '--depth 1' if that -# will involve less objects fetched. -_git-r3_smart_fetch() { - debug-print-function ${FUNCNAME} "$@" - - local sed_regexp='.*Counting objects: \([0-9]*\), done\..*' - - # start the main fetch - local cmd=( git fetch --progress "${@}" ) - echo "${cmd[@]}" >&2 - - # we copy the output to the 'sed' pipe for parsing. whenever sed finds - # the process count, it quits quickly to avoid delays in writing it. - # then, we start a dummy 'cat' to keep the pipe alive - - "${cmd[@]}" 2>&1 \ - | tee >( - sed -n -e "/${sed_regexp}/{s/${sed_regexp}/\1/p;q}" \ - > "${T}"/git-r3_main.count - exec cat >/dev/null - ) & - local main_pid=${!} - - # start the helper process - _git-r3_sub_fetch() { - # wait for main fetch to get object count; if the server doesn't - # output it, we won't even launch the parallel process - while [[ ! -s ${T}/git-r3_main.count ]]; do - sleep 0.25 - done - - # ok, let's see if parallel fetch gives us smaller count - # --dry-run will prevent it from writing to the local clone - # and sed should terminate git with SIGPIPE - local sub_count=$(git fetch --progress --dry-run --depth 1 "${@}" 2>&1 \ - | sed -n -e "/${sed_regexp}/{s/${sed_regexp}/\1/p;q}") - local main_count=$(<"${T}"/git-r3_main.count) - - # let's be real sure that '--depth 1' will be good for us. - # note that we have purely objects counts, and '--depth 1' - # may involve much bigger objects - if [[ ${main_count} && ${main_count} -ge $(( sub_count * 3/2 )) ]] - then - # signal that we want shallow fetch instead, - # and terminate the non-shallow fetch process - touch "${T}"/git-r3_want_shallow || die - kill ${main_pid} &>/dev/null - exit 0 - fi - - exit 1 - } - _git-r3_sub_fetch "${@}" & - local sub_pid=${!} - - # wait for main process to terminate, either of its own - # or by signal from subprocess - wait ${main_pid} - local main_ret=${?} - - # wait for subprocess to terminate, killing it if necessary. - # if main fetch finished before it, there's no point in keeping - # it alive. if main fetch was killed by it, it's done anyway - kill ${sub_pid} &>/dev/null - wait ${sub_pid} - - # now see if subprocess wanted to tell us something... - if [[ -f ${T}/git-r3_want_shallow ]]; then - rm "${T}"/git-r3_want_shallow || die - - # if fetch finished already (wasn't killed), ignore it - [[ ${main_ret} -eq 0 ]] && return 0 - - # otherwise, restart as shallow fetch - einfo "Restarting fetch using --depth 1 to save bandwidth ..." - local cmd=( git fetch --progress --depth 1 "${@}" ) - echo "${cmd[@]}" >&2 - "${cmd[@]}" - main_ret=${?} - fi - - return ${main_ret} -} - # @FUNCTION: _git-r3_is_local_repo # @USAGE: # @INTERNAL @@ -425,7 +324,7 @@ git-r3_fetch() { for r in "${repos[@]}"; do einfo "Fetching ${remote_ref} from ${r} ..." - local is_branch lookup_ref + local lookup_ref if [[ ${remote_ref} == refs/heads/* || ${remote_ref} == HEAD ]] then is_branch=1 @@ -436,80 +335,22 @@ git-r3_fetch() { lookup_ref=refs/tags/${remote_ref} fi - # first, try ls-remote to see if ${remote_ref} is a real ref - # and not a commit id. if it succeeds, we can pass ${remote_ref} - # to 'fetch'. otherwise, we will just fetch everything - - # split on whitespace - local ref=( - $(git ls-remote "${r}" "${lookup_ref}" || echo __FAIL__) + local fetch_command=( + git fetch --prune "${r}" + # mirror the remote branches as local branches + "refs/heads/*:refs/heads/*" + # pull tags explicitly in order to prune them properly + "refs/tags/*:refs/tags/*" ) - # normally, ref[0] is a hash, so we can do magic strings here - [[ ${ref[0]} == __FAIL__ ]] && continue - - local nonshallow=${EGIT_NONSHALLOW} - local ref_param=() - if [[ ! ${ref[0]} ]]; then - nonshallow=1 - fi - - # trying to do a shallow clone of a local repo makes git try to - # write to the repo. we don't want that to happen. - _git-r3_is_local_repo "${r}" && nonshallow=1 - - # 1. if we need a non-shallow clone and we have a shallow one, - # we need to unshallow it explicitly. - # 2. if we want a shallow clone, we just pass '--depth 1' - # to the first fetch in the repo. passing '--depth' - # to further requests usually results in more data being - # downloaded than without it. - # 3. if we update a shallow clone, we try without '--depth' - # first since that usually transfers less data. however, - # we use git-r3_smart_fetch that can switch into '--depth 1' - # if that looks beneficial. - - local fetch_command=( git fetch ) - if [[ ${nonshallow} ]]; then - if [[ -f ${GIT_DIR}/shallow ]]; then - ref_param+=( --unshallow ) - fi - # fetch all branches - ref_param+=( "refs/heads/*:refs/remotes/origin/*" ) - else - # 'git show-ref --heads' returns 1 when there are no branches - if ! git show-ref --heads -q; then - ref_param+=( --depth 1 ) - else - fetch_command=( _git-r3_smart_fetch ) - fi - fi - - # now, another important thing. we may only fetch a remote - # branch directly to a local branch. Otherwise, we need to fetch - # the commit and re-create the branch on top of it. - - if [[ ${ref[0]} ]]; then - if [[ ${is_branch} ]]; then - ref_param+=( -f "${remote_ref}:${local_id}/__main__" ) - else - ref_param+=( "refs/tags/${remote_ref}" ) - fi - fi - - # if ${remote_ref} is branch or tag, ${ref[@]} will contain - # the respective commit id. otherwise, it will be an empty - # array, so the following won't evaluate to a parameter. - set -- "${fetch_command[@]}" --no-tags "${r}" "${ref_param[@]}" + set -- "${fetch_command[@]}" echo "${@}" >&2 if "${@}"; then - if [[ ! ${is_branch} ]]; then - set -- git branch -f "${local_id}/__main__" \ - "${ref[0]:-${remote_ref}}" - echo "${@}" >&2 - if ! "${@}"; then - die "Creating branch for ${remote_ref} failed (wrong ref?)." - fi + set -- git branch -f "${local_id}/__main__" \ + "${ref[0]:-${remote_ref}}" + echo "${@}" >&2 + if ! "${@}"; then + die "Creating branch for ${remote_ref} failed (wrong ref?)." fi success=1 -- 1.9.0