ocfsv2 cluster management

o2cb (text/plain), 35.18 KB, created by Pierre Colot on 2008-11-29 22:15:50 UTC

#!/bin/bash
# init fragment for O2CB.
#
# chkconfig: 2345 24 20
# description: O2CB cluster services.
#

# Force LC_ALL=C
export LC_ALL=C

CLUSTERCONF=/etc/ocfs2/cluster.conf

OCFS2_SYS_DIR="/sys/fs/ocfs2"
LOADED_PLUGINS_FILE="${OCFS2_SYS_DIR}/loaded_cluster_plugins"
CLUSTER_STACK_FILE="${OCFS2_SYS_DIR}/cluster_stack"

CONFIGURATION=/etc/conf.d/ocfs2

# How long to wait for things to start
BRINGUP_TIMEOUT=5

# The default values should always be in sync with the kernel
DEF_HEARTBEAT_THRESHOLD=31
DEF_IDLE_TIMEOUT_MS=30000
DEF_KEEPALIVE_DELAY_MS=2000
DEF_RECONNECT_DELAY_MS=2000

# Minimum timeout values
MIN_HEARTBEAT_THRESHOLD=7
MIN_IDLE_TIMEOUT_MS=5000
MIN_KEEPALIVE_DELAY_MS=1000
MIN_RECONNECT_DELAY_MS=2000

VERSION=1.4.1

# Source configuration
[ -f "${CONFIGURATION}" ] && . "${CONFIGURATION}"
# Need this default
[ -z "$O2CB_STACK" ] && O2CB_STACK=o2cb

configfs_path()
{
    # Note that this is only valid *after* configfs is loaded
    if [ -d /sys/kernel/config ]
    then
        echo /sys/kernel/config
    fi
}


#
# if_fail()
#
# Evaluates return codes.  If 0, prints "OK", if 1, prints "Failed"
# and exits.  If 2, status is "already done" and nothing is printed.
# The rest of the functions in here all honor this convention.
#
if_fail()
{
    RC="$1"
    REASON="$2"
    if [ "$RC" = "0" ]
    then
        echo "OK"
        return
    elif [ "$RC" = "2" ]
    then
        return
    fi
    echo "Failed"
    if [ -n "${REASON}" ]
    then
        echo "${REASON}" >&2
    fi
    exit 1
}


#
# write_sysconfig()
#
# Writes the system configuration out
#
write_sysconfig()
{
    echo -n "Writing O2CB configuration: "
    cat >"${CONFIGURATION}" <<EOF
# Copyright 1999-2006 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
# $Header: /var/cvsroot/gentoo-x86/sys-fs/ocfs2-tools/files/ocfs2.conf,v 1.1 2006/07/20 05:13:14 dberkholz Exp $

## Generated file by o2cb


# Original Section related to O2CB
# ================================
#
# This is a configuration file for automatic startup of the O2CB
# driver.  It is generated by running /etc/init.d/o2cb configure.
#

# O2CB_ENABLED: 'true' means to load the driver on boot.
O2CB_ENABLED=${O2CB_ENABLED:-false}

# O2CB_STACK: The name of the cluster stack backing O2CB.
O2CB_STACK=${O2CB_STACK}

# O2CB_BOOTCLUSTER: If not empty, the name of a cluster to start.
O2CB_BOOTCLUSTER=${O2CB_BOOTCLUSTER}

# O2CB_HEARTBEAT_THRESHOLD: Iterations before a node is considered dead.
O2CB_HEARTBEAT_THRESHOLD=${O2CB_HEARTBEAT_THRESHOLD}

# O2CB_IDLE_TIMEOUT_MS: Time in ms before a network connection is considered dead.
O2CB_IDLE_TIMEOUT_MS=${O2CB_IDLE_TIMEOUT_MS}

# O2CB_KEEPALIVE_DELAY_MS: Max time in ms before a keepalive packet is sent
O2CB_KEEPALIVE_DELAY_MS=${O2CB_KEEPALIVE_DELAY_MS}

# O2CB_RECONNECT_DELAY_MS: Min time in ms between connection attempts
O2CB_RECONNECT_DELAY_MS=${O2CB_RECONNECT_DELAY_MS}

# Gentoo Section related to OCFS2
# ===============================

# Put your cluster names here, separated by space, ie.
# OCFS2_CLUSTER="cluster1 admincluster cluster2"
OCFS2_CLUSTER="${OCFS2_CLUSTER}"

# Some heartbeat tweaks to prevent self-fencing quite so much during heavy load.
# http://oss.oracle.com/projects/ocfs2/dist/documentation/ocfs2_faq.html

# How long to wait before a node is considered dead from lack of network activity.
OCFS2_IDLE_TIMEOUT_MS="${OCFS2_IDLE_TIMEOUT_MS}"

# How often we should attempt to send heartbeats.
OCFS2_KEEPALIVE_DELAY_MS="${OCFS2_KEEPALIVE_DELAY_MS}"
OCFS2_RECONNECT_DELAY_MS="${OCFS2_RECONNECT_DELAY_MS}"

# How often we should attempt to send heartbeats.
# How many interations before a node is considered dead from lack of IO activity.
# (dead_threshold - 1) * 2s
OCFS2_DEAD_THRESHOLD="${OCFS2_DEAD_THRESHOLD}"

EOF

    if [ $? != 0 ]
    then
        return 1
    fi
    return 0
}

#
# read_timeout()
#   Returns timeout value provided by user to caller in RET_VAL
#
read_timeout()
{
    if [ "$#" -lt "4" ]; then
        echo "read_timeout(): Requires more arguments" >&2
        exit 1
    fi

    ATTRIB_NAME=$1
    ATTRIB_VAL=$2
    MIN_ATTRIB_VAL=$3
    DEF_ATTRIB_VAL=$4
    RET_VAL=0

    while :
    do
        if [ -z $ATTRIB_VAL ]; then
            CUR=$DEF_ATTRIB_VAL
        else
            CUR=$ATTRIB_VAL
        fi
        echo -n "Specify ${ATTRIB_NAME} (>=$MIN_ATTRIB_VAL) [$CUR]: "
        read LINE
        case "$LINE" in
        "")
            RET_VAL="$ATTRIB_VAL"
            break
            ;;

        *[^0-9]*)
            echo "Invalid ${ATTRIB_NAME} value: $LINE" >&2
            ;;
        *)
            if [ $LINE -lt $MIN_ATTRIB_VAL ]; then
                echo "${ATTRIB_NAME} cannot be less than $MIN_ATTRIB_VAL" >&2
            else
                RET_VAL="$LINE"
                break
            fi
            ;;
        esac
    done
}

set_timeouts()
{
    O2CB_HEARTBEAT_THRESHOLD_FILE_OLD=/proc/fs/ocfs2_nodemanager/hb_dead_threshold
    O2CB_HEARTBEAT_THRESHOLD_FILE=$(configfs_path)/cluster/${CLUSTER}/heartbeat/dead_threshold
    if [ -n "$O2CB_HEARTBEAT_THRESHOLD" ]; then
        if [ -f "$O2CB_HEARTBEAT_THRESHOLD_FILE" ]; then
            echo "$O2CB_HEARTBEAT_THRESHOLD" > "$O2CB_HEARTBEAT_THRESHOLD_FILE"
        elif [ -f "$O2CB_HEARTBEAT_THRESHOLD_FILE_OLD" ]; then
            echo "$O2CB_HEARTBEAT_THRESHOLD" > "$O2CB_HEARTBEAT_THRESHOLD_FILE_OLD"
        fi
    fi

    O2CB_IDLE_TIMEOUT_MS_FILE=$(configfs_path)/cluster/${CLUSTER}/idle_timeout_ms
    if [ -n "$O2CB_IDLE_TIMEOUT_MS" ]; then
        if [ -f "$O2CB_IDLE_TIMEOUT_MS_FILE" ]; then
            echo "$O2CB_IDLE_TIMEOUT_MS" > "$O2CB_IDLE_TIMEOUT_MS_FILE"
        fi
    fi

    O2CB_KEEPALIVE_DELAY_MS_FILE=$(configfs_path)/cluster/${CLUSTER}/keepalive_delay_ms
    if [ -n "$O2CB_KEEPALIVE_DELAY_MS" ]; then
        if [ -f "$O2CB_KEEPALIVE_DELAY_MS_FILE" ]; then
            echo "$O2CB_KEEPALIVE_DELAY_MS" > "$O2CB_KEEPALIVE_DELAY_MS_FILE"
        fi
    fi

    O2CB_RECONNECT_DELAY_MS_FILE=$(configfs_path)/cluster/${CLUSTER}/reconnect_delay_ms
    if [ -n "$O2CB_RECONNECT_DELAY_MS" ]; then
        if [ -f "$O2CB_RECONNECT_DELAY_MS_FILE" ]; then
            echo "$O2CB_RECONNECT_DELAY_MS" > "$O2CB_RECONNECT_DELAY_MS_FILE"
        fi
    fi
}

show_timeouts()
{

    O2CB_HEARTBEAT_THRESHOLD_FILE_OLD=/proc/fs/ocfs2_nodemanager/hb_dead_threshold
    O2CB_HEARTBEAT_THRESHOLD_FILE=$(configfs_path)/cluster/${CLUSTER}/heartbeat/dead_threshold
    if [ -f "$O2CB_HEARTBEAT_THRESHOLD_FILE" ]; then
        VAL=`cat "$O2CB_HEARTBEAT_THRESHOLD_FILE"`
        echo "Heartbeat dead threshold = ${VAL}"
    elif [ -f "$O2CB_HEARTBEAT_THRESHOLD_FILE_OLD" ]; then
        VAL=`cat "$O2CB_HEARTBEAT_THRESHOLD_FILE_OLD"`
        echo "  Heartbeat dead threshold: ${VAL}"
    fi

    O2CB_IDLE_TIMEOUT_MS_FILE=$(configfs_path)/cluster/${CLUSTER}/idle_timeout_ms
    if [ -f "$O2CB_IDLE_TIMEOUT_MS_FILE" ]; then
        VAL=`cat "$O2CB_IDLE_TIMEOUT_MS_FILE"`
        echo "  Network idle timeout: ${VAL}"
    fi

    O2CB_KEEPALIVE_DELAY_MS_FILE=$(configfs_path)/cluster/${CLUSTER}/keepalive_delay_ms
    if [ -f "$O2CB_KEEPALIVE_DELAY_MS_FILE" ]; then
        VAL=`cat "$O2CB_KEEPALIVE_DELAY_MS_FILE"`
        echo "  Network keepalive delay: ${VAL}"
    fi

    O2CB_RECONNECT_DELAY_MS_FILE=$(configfs_path)/cluster/${CLUSTER}/reconnect_delay_ms
    if [ -f "$O2CB_RECONNECT_DELAY_MS_FILE" ]; then
        VAL=`cat "$O2CB_RECONNECT_DELAY_MS_FILE"`
        echo "  Network reconnect delay: ${VAL}"
    fi
}

#
# configure_ask()
#
# Ask configuration questions, setting the shell vars.
#
configure_ask()
{

    cat <<EOF
Configuring the O2CB driver.

This will configure the on-boot properties of the O2CB driver.
The following questions will determine whether the driver is loaded on
boot.  The current values will be shown in brackets ('[]').  Hitting
<ENTER> without typing an answer will keep that current value.  Ctrl-C
will abort.

EOF

    while :
    do
        if [ "$O2CB_ENABLED" = "true" ]
        then
            CUR=y
        else
            CUR=n
        fi
        echo -n "Load O2CB driver on boot (y/n) [$CUR]: "
        read LINE
        case "$LINE" in
        "")
            break
            ;;
        y|Y)
            O2CB_ENABLED=true
            break
            ;;
        n|N)
            O2CB_ENABLED=false
            break
            ;;
        *)
            echo "Invalid answer: $LINE" >&2
            ;;
        esac
    done

    while :
    do
        echo -n "Cluster stack backing O2CB [$O2CB_STACK]: "
        read LINE
        case "$LINE" in
        "")
            break
            ;;
        ????)
            O2CB_STACK="$LINE"
            break
            ;;
        *)
            echo "Invalid answer: $LINE" >&2
            ;;
        esac
    done

    while :
    do
        echo -n "Cluster to start on boot (Enter \"none\" to clear) [$O2CB_BOOTCLUSTER]: "
        read LINE
        case "$LINE" in
        "")
            break
            ;;
        none)
            O2CB_BOOTCLUSTER=
            break
            ;;

        *[^a-zA-Z0-9]*)
            echo "Invalid cluster name: $LINE" >&2
            ;;
        *)
            O2CB_BOOTCLUSTER="$LINE"
            break
            ;;
        esac
    done

    read_timeout "heartbeat dead threshold" "$O2CB_HEARTBEAT_THRESHOLD" "$MIN_HEARTBEAT_THRESHOLD" "$DEF_HEARTBEAT_THRESHOLD"
    O2CB_HEARTBEAT_THRESHOLD="$RET_VAL"

    read_timeout "network idle timeout in ms" "$O2CB_IDLE_TIMEOUT_MS" "$MIN_IDLE_TIMEOUT_MS" "$DEF_IDLE_TIMEOUT_MS"
    O2CB_IDLE_TIMEOUT_MS="$RET_VAL"

    read_timeout "network keepalive delay in ms" "$O2CB_KEEPALIVE_DELAY_MS" "$MIN_KEEPALIVE_DELAY_MS" "$DEF_KEEPALIVE_DELAY_MS"
    O2CB_KEEPALIVE_DELAY_MS="$RET_VAL"

    read_timeout "network reconnect delay in ms" "$O2CB_RECONNECT_DELAY_MS" "$MIN_RECONNECT_DELAY_MS" "$DEF_RECONNECT_DELAY_MS"
    O2CB_RECONNECT_DELAY_MS="$RET_VAL"

    # XXX ask about mount point base
}


#
# make_dir()
#
# Create $1
# Returns 0 on success, 1 on error, 2 if it already exists.
#
make_dir()
{
    if [ "$#" -lt "1" -o -z "$1" ]
    then
        echo "make_dir(): Requires an argument" >&2
        return 1
    fi
    DIR="$1"
    if [ -e "$DIR" ]
    then
        if [ -d "$DIR" ]
        then
            return 2
        fi
        echo "make_dir(): File $DIR is not a directory" >&2
        return 1
    fi

    echo -n "Creating directory '$DIR': "
    mkdir -p "$DIR" 2>/dev/null
    if [ $? != 0 ]
    then
        echo "Unable to create directory '$DIR'" >&2
        return 1
    fi
    return 0
}


#
# driver_filesystem()
# Check to see if a filesystem driver is loaded.
#
# 0 is loaded, 1 is not.
#
driver_filesystem()
{
    if [ -z "$1" ]
    then
        echo "driver_filesystem(): Missing an argument" >&2
        exit 1
    fi
    FSNAME="$1"
    PSEUDO="$2"

    FSOUT="$(awk '(NF == 1 && $1 ~ /^'$FSNAME'$/) || $2 ~ /^'$FSNAME'$/{
                      print $1;exit
                  }' /proc/filesystems 2>/dev/null)"

    test -n "$FSOUT"
    return $?
}


#
# load_filesystem()
# Load a filesystem driver.
#
# 0 is success, 1 is error, 2 is already loaded.
#
load_filesystem()
{
    if [ "$#" != "1" -o -z "$1" ]
    then
        echo "load_filesystem(): Missing an argument" >&2
        return 1
    fi
    FSNAME="$1"

    driver_filesystem "$FSNAME" && return 2

    echo -n "Loading filesystem \"$FSNAME\": "
    modprobe -s "$FSNAME"
    if [ "$?" != 0 ]
    then
        echo "Unable to load filesystem \"$FSNAME\"" >&2
        return 1
    fi

    return 0
}

#
# unload_filesystem()
# Unload a filesystem driver.  Be careful to notice if the driver is
# built-in and do nothing.
#
# 0 is success, 1 is error, 2 is already loaded.
#
unload_filesystem()
{
    if [ "$#" != "1" -o -z "$1" ]
    then
        echo "unload_filesystem(): Missing an argument" >&2
        return 1
    fi
    FSNAME="$1"

    driver_filesystem "$FSNAME" || return 2

    MODOUT="`awk '$1 ~ /^'$FSNAME'$/{print $1,$3;exit}' < /proc/modules 2>/dev/null`"
    if [ -z "$MODOUT" ]
    then
        # The driver is built in, we can't unload it.
        return 2
    fi

    case "$MODOUT" in
    $FSNAME\ 0)
        ;;
    $FSNAME\ *)
        # The driver is busy, leave it alone
        return 2
        ;;
    *)
        echo -n "Invalid module parsing! "
        return 1
        ;;
    esac

    echo -n "Unloading module \"$FSNAME\": "
    modprobe -rs "$FSNAME"
    if [ "$?" != 0 ]
    then
        echo "Unable to unload module \"$FSNAME\"" >&2
        return 1
    fi

    return 0
}

#
# check_filesystem()
# Check to see if a filesystem of type $1 is mounted at $2.
#
# 0 is mounted, 1 is not.
#
check_filesystem()
{
    if [ "$#" != "2" -o -z "$1" -o -z "$2" ]
    then
        echo "check_filesystem(): Missing arguments" >&2
        exit 1
    fi
    FSNAME="$1"
    MOUNTPOINT="$2"

    FULL_MOUNTSEARCH="`echo "$MOUNTPOINT" | sed -e 's/\//\\\\\//g'`"
    MOUNTOUT="`awk '$2 ~ /^'$FULL_MOUNTSEARCH'$/ && $3 ~ /^'$FSNAME'$/{print $2; exit}' < /proc/mounts 2>/dev/null`"
    test -n "$MOUNTOUT"
    return $?
}

#
# mount_filesystem()
# Mounts a pseudo-filesystem of type $1 on mountpoint $2.  It will
# load the drivers for $1 and create $2 if needed.
#
# 0 is success, 1 is error, 2 is already mounted.
#
mount_filesystem()
{
    if [ "$#" != "2" -o -z "$1" -o -z "$2" ]
    then
        echo "mount_filesystem(): Missing arguments" >&2
        return 1
    fi
    FSNAME="$1"
    MOUNTPOINT="$2"

    check_filesystem "$FSNAME" "$MOUNTPOINT" && return 2

    load_filesystem "$FSNAME"
    if_fail $?

    # XXX some policy?
    if [ ! -e "$MOUNTPOINT" ]; then
        make_dir $MOUNTPOINT
        if_fail "$?"
    fi

    echo -n "Mounting ${FSNAME} filesystem at ${MOUNTPOINT}: "
    mount -t ${FSNAME} ${FSNAME} ${MOUNTPOINT}
    if [ $? != 0 ]
    then
        echo "Unable to mount ${FSNAME} filesystem" >&2
        return 1
    fi

    return 0
}

#
# unmount_filesystem()
# Unmount a pseudo-filesystem of type $1 from mountpoint $2.  It will
# remove the driver for $1 if it can.
#
# 0 is success, 1 is error, 2 is not mounted
#
unmount_filesystem()
{
    if [ "$#" != "2" -o -z "$1" -o -z "$2" ]
    then
        echo "unmount_filesystem(): Missing arguments" >&2
        return 1
    fi
    FSNAME="$1"
    MOUNTPOINT="$2"

    if check_filesystem "$FSNAME" "$MOUNTPOINT"
    then
        echo -n "Unmounting ${FSNAME} filesystem: "
        umount $MOUNTPOINT
        RC=$?
        if [ $RC != 0 ]
        then
            echo "Unable to unmount ${FSNAME} filesystem" >&2
            return 1
        fi
        if_fail $RC  # For the success string
    fi

    unload_filesystem "$FSNAME"
    return $?
}

#
# status_filesystem()
# Report the status of a filesystem, whether it is mounted or not
#
# 0 is not mounted, 1 is error, 2 is already mounted
#
status_filesystem()
{
    if [ "$#" != "2" -o -z "$1" -o -z "$2" ]
    then
        echo "status_filesystem(): Missing arguments" >&2
        return 1
    fi
    FSNAME="$1"
    MOUNTPOINT="$2"

    echo -n "Driver for \"$FSNAME\": "
    if driver_filesystem "$FSNAME"
    then
        echo "Loaded"
    else
        echo "Not loaded"
        return 0
    fi

    echo -n "Filesystem \"$FSNAME\": "
    if check_filesystem "$FSNAME" "$MOUNTPOINT"
    then
        echo "Mounted"
        return 2
    fi
    echo "Not mounted"
    return 0
}

status_daemon()                                               
{                                                             
    DAEMON="/sbin/ocfs2_controld.${O2CB_STACK}"               
    echo -n "Checking for control daemon: "                   
    if [ -n "$(pidofproc "$DAEMON")" ]                        
    then                                                      
        echo "$DAEMON"                                        
        return 0                                              
    else                                                      
        echo "not running"                                    
        return 1                                              
    fi                                                        
}                                                             
                                                              
bringup_daemon()                                              
{                                                             
    DAEMON="/sbin/ocfs2_controld.${O2CB_STACK}"               
    echo -n "Starting $(basename "$DAEMON"): "                
    start_daemon "$DAEMON"                                    
    [ $? != 0 ] && return 1                                   
                                                              
    COUNT=0                                                   
    while [ -z "$(pidofproc "$DAEMON")" ]                     
    do                                                        
        COUNT="$(expr "$COUNT" + 1)"                          
        if [ "$COUNT" -gt "$BRINGUP_TIMEOUT" ]                
        then                                                  
            return 1                                          
        fi                                                    
        sleep 1                                               
    done                                                      
                                                              
    return 0                                                  
}                                                             
                                                              
kill_daemon()                                                 
{                                                             
    SIGNAL="$1"                                               
    DAEMON="/sbin/ocfs2_controld.${O2CB_STACK}"               
                                                              
    status_daemon >/dev/null 2>&1 || return 2                 
                                                              
    PID="$(pidofproc "$DAEMON")"                              
                                                              
    echo -n "Stopping $(basename "$DAEMON"): "                
    killproc "$DAEMON" $SIGNAL                                
                                                              
    TRIED=                                                    
    while :                                                   
    do                                                        
        NEWPID="$(pidofproc "$DAEMON")"                       
        if [ -z "$NEWPID" -o "$NEWPID" != "$PID" ]            
        then                                                  
            echo "Stopped"                                    
            return                                            
        fi                                                    
                                                              
        [ -n "$TRIED" ] && break                              
        TRIED=t                                               
                                                              
        # Give it one chance to exit                          
        sleep 1                                               
    done                                                      
                                                              
    echo "Failed"                                             
    return 1                                                  
}                                                             
                                                              
#
# check_heartbeat()
#
# 0 is hb not active, 1 is error, 2 is hb active
#
check_heartbeat()
{
    if [ "$#" -lt "1" -o -z "$1" ]
    then
        echo "check_heartbeat(): Requires an argument" >&2
        return 1
    fi
    CLUSTER="$1"

    RC=0
    if [ -d "$(configfs_path)/cluster/${CLUSTER}/heartbeat/" ]
    then
        ls -1 "$(configfs_path)/cluster/${CLUSTER}/heartbeat/" | while read HBUUID
        do
            if [ -d "$(configfs_path)/cluster/${CLUSTER}/heartbeat/${HBUUID}" ]
            then
                return 2;
            fi
        done
        if [ $? = 2 ]
        then
            RC=2
        fi
    fi

    return $RC
}

#
# clean_heartbeat()
# Removes the inactive heartbeat regions
#
clean_heartbeat()
{
    if [ "$#" -lt "1" -o -z "$1" ]
    then
        echo "clean_heartbeat(): Requires an argument" >&2
        return 1
    fi
    CLUSTER="$1"

    if [ ! -f "$(configfs_path)/cluster/${CLUSTER}/heartbeat/*" ]
    then
        return
    fi

    echo -n "Cleaning heartbeat on ${CLUSTER}: "

    ls -1 "$(configfs_path)/cluster/${CLUSTER}/heartbeat/" | while read HBUUID
    do
        if [ ! -d "$(configfs_path)/cluster/${CLUSTER}/heartbeat/${HBUUID}" ]
        then
            continue
        fi

        OUTPUT="`ocfs2_hb_ctl -I -u ${HBUUID} 2>&1`"
        if [ $? != 0 ]
        then
            echo "Failed"
            echo "${OUTPUT}" >&2
            exit 1
        fi

        REF="`echo ${OUTPUT} | awk '/refs/ {print $2; exit;}' 2>&1`"
        if [ $REF != 0 ]
        then
           echo "Failed"
           echo "At least one heartbeat region still active" >&2
           exit 1
        else
           OUTPUT="`ocfs2_hb_ctl -K -u ${HBUUID} 2>&1`"
        fi
    done
    if [ $? = 1 ]
    then
        exit 1
    fi
    echo "OK"
}

#
# clean_cluster()
# Force cleans configured cluster
#
# 0 is clean, 1 is error, 2 is not clean
#
clean_cluster()
{
    if [ "$#" -lt "1" -o -z "$1" ]
    then
        echo "clean_cluster(): Requires an argument" >&2
        return 1
    fi
    CLUSTER="$1"

    if [ -d "$(configfs_path)/cluster/${CLUSTER}/node/" ]
    then
        ls -1 "$(configfs_path)/cluster/${CLUSTER}/node/" | while read NODE
        do
            rmdir "$(configfs_path)/cluster/${CLUSTER}/node/${NODE}" >/dev/null 2>&1
        done
    fi

    if [ -d "$(configfs_path)/cluster/${CLUSTER}" ]
    then
        rmdir "$(configfs_path)/cluster/${CLUSTER}" >/dev/null 2>&1
    fi

    if [ -d "$(configfs_path)/cluster/${CLUSTER}" ]
    then
        return 2
    fi

    return 0
}

select_stack_plugin()
{
    case "$O2CB_STACK" in
    o2cb)
        echo o2cb
        ;;
    ????)
        echo user
        ;;
    *)
        echo -n "Checking if cluster stack \"$O2CB_STACK\" is supported: "
        if_fail 1 "Invalid cluster stack name \"$O2CB_STACK\""
        ;;
    esac
 }

#
# load_stack_o2cb()
# Load the o2cb stack.
#
# 0 is success, 1 is error, 2 is already loaded
#
load_stack_o2cb()
{
    PLUGIN="o2cb"
    PLUGIN_MODULE="ocfs2_stack_${PLUGIN}"

    #
    # Older drivers don't have stack plugins.  They only support the
    # classic stack.  It is loaded in ocfs2_nodemanager, which is pulled
    # in during the load of ocfs2_dlmfs.  Thus, if we are using the classic
    # stack and cannot load the stack plugin, we silently let the later
    # code try for older drivers.
    #
    # For a newer driver, if ocfs2_stackglue is loaded, we'll see
    # CLUSTER_STACK_FILE.  Thus, we can determine what mode we're in.
    #
    [ ! -e "$CLUSTER_STACK_FILE" ] && \
        modprobe -s ocfs2_stackglue

    # If we're a newer driver, CLUSTER_STACK_FILE will now appear
    if [ -e "$CLUSTER_STACK_FILE" ]
    then
        SP_OUT="$(awk '/^'$PLUGIN'$/{print; exit}' "$LOADED_PLUGINS_FILE" 2>/dev/null)"
        if [ -z "$SP_OUT" ]
        then
            echo -n "Loading stack plugin \"$PLUGIN\": "
            modprobe -s "$PLUGIN_MODULE"
            [ "$?" != 0 ] && if_fail 1 "Unable to load module \"$PLUGIN_MODULE\""
            if_fail 0
        fi
    fi

    mount_filesystem "ocfs2_dlmfs" "/dlm"
    if_fail $?

    return 0
}

#
# load_stack_user()
# Load the userspace stack plugin.
#
# 0 is success, 1 is error, 2 is already loaded
#
load_stack_user()
{
    PLUGIN="user"
    PLUGIN_MODULE="ocfs2_stack_${PLUGIN}"

    #
    # Older drivers don't have stack plugins.  They only support the
    # classic stack.  It is loaded in ocfs2_nodemanager, which is pulled
    # in during the load of ocfs2_dlmfs.  Thus, if we are using the classic
    # stack and cannot load the stack plugin, we silently let the later
    # code try for older drivers.
    #
    # For a newer driver, if ocfs2_stackglue is loaded, we'll see
    # CLUSTER_STACK_FILE.  Thus, we can determine what mode we're in.
    #
    [ ! -e "$CLUSTER_STACK_FILE" ] && \
        modprobe -s ocfs2_stackglue

    if [ ! -e "$CLUSTER_STACK_FILE" ]
    then
        echo -n "Checking if stack \"$O2CB_STACK\" is supported: "
        if_fail 1 "User stack plugin is not supported"
    fi

    SP_OUT="$(awk '/^'$PLUGIN'$/{print; exit}' "$LOADED_PLUGINS_FILE" 2>/dev/null)"
    if [ -z "$SP_OUT" ]
    then
        echo -n "Loading stack plugin \"$PLUGIN\": "
        modprobe -s "$PLUGIN_MODULE"
        [ "$?" != 0 ] && if_fail 1 "Unable to load module \"$PLUGIN_MODULE\""
        if_fail 0
    fi
}

#
# unload_stack_plugins()
# Since we can have both plugins unloaded, we just do the same work
# for all stack types
#
unload_stack_plugins()
{
    # Whether not loaded or old drivers, if we can't find the stack glue,
    # we have nothing to do.
    [ ! -e "$LOADED_PLUGINS_FILE" ] && return 2

    while read plugin
    do
        unload_module "ocfs2_stack_${plugin}"
        if_fail $? "Unable to unload ocfs2_stack_${plugin}"
    done <"$LOADED_PLUGINS_FILE"

    unload_module "ocfs2_stackglue"
    if_fail $? "Unable to unload ocfs2_stackglue"
}

#
#
# unload_stack_o2cb()
# Unload the o2cb stack plugin
#
# 0 is success, 1 is error, 2 is not loaded.
#
unload_stack_o2cb()
{
    PLUGIN="o2cb"
    PLUGIN_MODULE="ocfs2_stack_${PLUGIN}"

    if [ -d "$(configfs_path)/cluster/" ]
    then
        ls -1 $(configfs_path)/cluster/ | while read CLUSTER
        do
            if [ ! -L "$(configfs_path)/cluster/${CLUSTER}" -a \
                 -d "$(configfs_path)/cluster/${CLUSTER}" ]
            then
                echo "Unable to unload modules as O2CB cluster ${CLUSTER} is still online" >&2
                exit 1
            fi
        done
        if [ $? = 1 ]
        then
            exit 1
        fi
    fi

    unmount_filesystem "ocfs2_dlmfs" "/dlm"
    if_fail $?

    unload_stack_plugins
}

#
# unload_stack_user()
# Unload the user stack plugin
#
# 0 is success, 1 is error, 2 is not loaded.
#
unload_stack_user()
{
    PLUGIN="$(select_stack_plugin)"
    PLUGIN_MODULE="ocfs2_stack_${PLUGIN}"

    if status_daemon >/dev/null 2>&1
    then
        echo "Unable to unload modules as the cluster is still online" >&2
        exit 1
    fi

    unload_stack_plugins
}

status_stack_plugin()
{
    PLUGIN="$(select_stack_plugin)"

    if [ ! -e "$LOADED_PLUGINS_FILE" ]
    then
        # The old stack may be fine here.
        [ "$PLUGIN" = "o2cb" ] && return

        echo "Stack glue driver: Not loaded"
        return
    fi
    echo "Stack glue driver: Loaded"

    echo -n "Stack plugin \"$PLUGIN\": "
    if grep "$PLUGIN" "$LOADED_PLUGINS_FILE" >/dev/null 2>&1
    then
        echo "Loaded"
    else
        echo "Not loaded"
    fi
}

status_stack_o2cb()
{
    status_stack_plugin
    status_filesystem "ocfs2_dlmfs" "/dlm"
}

status_stack_user()
{
    status_stack_plugin

    echo -n "Driver for \"ocfs2\": "
    if driver_filesystem "ocfs2"
    then
        echo "Loaded"
    else
        echo "Not loaded"
    fi
 }

#
# unload_module()
# Unload a module
#
# 0 is success, 1 is error, 2 is not loaded
#
unload_module()
{
    if [ "$#" -lt "1" -o -z "$1" ]
    then
        echo "unload_module(): Requires an argument" >&2
        return 1
    fi
    MODNAME="$1"

    MODOUT="`awk '$1 ~ /^'$MODNAME'$/{print $1,$3;exit}' < /proc/modules 2>/dev/null`"
    if [ -z "$MODOUT" ]
    then
        return 2
    fi
    case "$MODOUT" in
    $MODNAME\ 0)
        ;;
    $MODNAME\ *)
        return 2
        ;;
    *)
        echo -n "Invalid module parsing! "
        return 1
        ;;
    esac

    echo -n "Unloading module \"$MODNAME\": "
    modprobe -rs "$MODNAME"
    if [ "$?" != 0 ]
    then
        echo "Unable to unload module \"$MODNAME\"" >&2
        return 1
    fi

    return 0
}

#
# check_load_module()
#
# 0 is not loaded, 1 is error, 2 is already loaded
#
check_load_module()
{
    if [ "$#" -lt "1" -o -z "$1" ]
    then
        echo "check_load_module(): Requires an argument" >&2
        return 1
    fi
    MODNAME="$1"

    echo -n "Module \"$MODNAME\": "
    MODOUT="`awk '$1 ~ /^'$MODNAME'$/{print $1,$3;exit}' < /proc/modules 2>/dev/null`"
    if [ -z "$MODOUT" ]
    then
        echo "Not loaded"
        return 0
    fi
    echo "Loaded"
    return 2
}

load()
{
    PLUGIN="$(select_stack_plugin)"

    # XXX: SPECIAL CASE!  We must load configfs for configfs_path() to work
    load_filesystem "configfs"
    if_fail $?

    mount_filesystem "configfs" "$(configfs_path)"
    if_fail $?

    load_stack_$PLUGIN

    return 0
}

load_status()
{
    PLUGIN="$(select_stack_plugin)"
    status_filesystem "configfs" "$(configfs_path)"
    status_stack_$PLUGIN

    return 0
}

online_o2cb()
{
    if [ "$#" -lt "1" -o -z "$1" ]
    then
        echo "online_o2cb(): Requires an argument" >&2
        return 1
    fi
    CLUSTER="$1"

    if ! [ -f ${CLUSTERCONF} ]
    then
        echo -n "Checking O2CB cluster configuration : "
        if_fail 1
    fi

    echo -n "Starting O2CB cluster ${CLUSTER}: "
    OUTPUT="`o2cb_ctl -H -n "${CLUSTER}" -t cluster -a online=yes 2>&1`"
    if [ $? = 0 ]
    then
        set_timeouts
        echo "OK"
        return
    else
        echo "Failed"
        echo "$OUTPUT"
    fi

    echo -n "Stopping O2CB cluster ${CLUSTER}: "
    OUTPUT="`o2cb_ctl -H -n "${CLUSTER}" -t cluster -a online=no 2>&1`"
    if_fail "$?" "$OUTPUT"
}

online_user()
{
    if [ "$#" -lt "1" -o -z "$1" ]
    then
        echo "online_user(): Requires an argument" >&2
        return 1
    fi
    CLUSTER="$1"

    load_filesystem "ocfs2"
    if_fail $? "Unable to load ocfs2 driver"

    bringup_daemon
    if_fail $? "Unable to start control daemon"
}

online()
{
    CLUSTER="${1:-${O2CB_BOOTCLUSTER}}"
    if [ -z "$CLUSTER" ]
    then
        echo "Cluster not known"
        return
    fi
    PLUGIN="$(select_stack_plugin)"

    check_online $CLUSTER
    if [ $? = 2 ]
    then
        echo "Cluster ${CLUSTER} already online"
        return
    fi

    if [ -f "$CLUSTER_STACK_FILE" ]
    then
        echo -n "Setting cluster stack \"$O2CB_STACK\": "
        echo "$O2CB_STACK" >"$CLUSTER_STACK_FILE" 2>/dev/null
        if_fail $? "Unable to store cluster stack \"$O2CB_STACK\""
    elif [ "$O2CB_STACK" != "o2cb" ]
    then
        echo -n "Setting cluster stack \"$O2CB_STACK\": "
        if_fail 1 "Stack \"$O2CB_STACK\" is not supported"
    fi

    online_$PLUGIN "$CLUSTER"
}

#
# check_online_o2cb()
#
# 0 is not online, 1 is error, 2 is online
#
check_online_o2cb()
{
    if [ "$#" -lt "1" -o -z "$1" ]
    then
        echo "check_online(): Requires an argument" >&2
        return 1
    fi
    CLUSTER="$1"

    RC=0
    if [ -d "$(configfs_path)/cluster/${CLUSTER}/node/" ]
    then
        ls -1 "$(configfs_path)/cluster/${CLUSTER}/node/" | while read NODE
        do
            LOCAL="`cat \"$(configfs_path)/cluster/${CLUSTER}/node/${NODE}/local\"`"
            if [ $LOCAL = 1 ]
            then
                return 2
            fi
        done
        if [ $? = 2 ]
        then
            RC=2
        fi
    fi
    return $RC
}

#
# check_online_user()
#
# 0 is not online, 1 is error, 2 is online
#
check_online_user()
{
    if status_daemon >/dev/null 2>&1
    then
        return 2
    else
        return 1
    fi
}

#
# check_online()
#
# 0 is not online, 1 is error, 2 is online
#
check_online()
{
    PLUGIN="$(select_stack_plugin)"

    check_online_$PLUGIN $1
}

offline_o2cb()
{
    if [ "$#" -lt "2" -o -z "$1" -o -z "$2" ]
    then
        echo "offline_o2cb(): Missing arguments" >&2
        return 1
    fi
    CLUSTER="$1"
    FORCE="$2"

    if [ ! -e "$(configfs_path)/cluster/${CLUSTER}" ]
    then
        return
    fi

    clean_heartbeat $CLUSTER

    echo -n "Stopping O2CB cluster ${CLUSTER}: "
    check_heartbeat $CLUSTER
    if [ $? != 0 ]
    then
        echo "Failed"
        echo "Unable to stop cluster as heartbeat region still active" >&2
        exit 1
    fi

    if [ "$FORCE" -eq 1 ]
    then
        clean_cluster $CLUSTER
        if_fail "$?" "Unable to force-offline cluster $CLUSTER" >&2
    else
        OUTPUT="`o2cb_ctl -H -n "${CLUSTER}" -t cluster -a online=no 2>&1`"
        if_fail "$?" "$OUTPUT - Try to force-offline the O2CB cluster"
    fi
}

offline_user()
{
    if [ "$#" -lt "2" -o -z "$1" -o -z "$2" ]
    then
        echo "offline_user(): Missing arguments" >&2
        return 1
    fi
    CLUSTER="$1"
    FORCE="$2"

    kill_daemon
    [ $? = 1 -a "$FORCE" -eq 1 ] && kill_daemon -9
}

offline()
{
    CLUSTER="${1:-${O2CB_BOOTCLUSTER}}"
    if [ -z "$CLUSTER" ]
    then
        return
    fi
    PLUGIN="$(select_stack_plugin)"

    if [ $# -gt 1 ]
    then
        FORCE=$2
    else
        FORCE=0
    fi

    offline_$PLUGIN "$CLUSTER" "$FORCE"

    unload_filesystem "ocfs2"
    if_fail "$?"
}

start()
{
    if [ "$O2CB_ENABLED" != "true" ]
    then
        exit 0
    fi

    load
    online "$1"
}

unload()
{
    PLUGIN="$(select_stack_plugin)"

    unload_stack_$PLUGIN

    # Only unmount configfs if there are no other users
    if [ -z "$(ls -1 "$(configfs_path)")" ]
    then
        unmount_filesystem "configfs" "/sys/kernel/config"
        if_fail $?
    fi
}

stop()
{
    offline "$1"
    unload
}

configure()
{
    configure_ask
    if [ -z "$DPKGRECONF" ]; then
	write_sysconfig
	if_fail "$?" "Unable to write the driver configuration"
    fi
}

online_status_o2cb()
{
    if [ "$#" -lt "1" -o -z "$1" ]
    then
        echo "online_status_o2cb(): Missing arguments" >&2
        return 1
    fi
    CLUSTER="$1"

    echo -n "Checking O2CB cluster $CLUSTER: "
    check_online_o2cb $CLUSTER
    if [ $? = 2 ]
    then
       echo "Online"
    else
       echo "Offline"
       return 0;
    fi

    show_timeouts

    echo -n "Checking O2CB heartbeat: "
    check_heartbeat $CLUSTER
    if [ $? = 2 ]
    then
        echo "Active"
    else
        echo "Not active"
        return 0;
    fi

}

online_status_user()
{
    status_daemon
}

online_status()
{
    if [ "$#" -lt "1" -o -z "$1" ]
    then
        echo "online_status(): Missing arguments" >&2
        return 1
    fi
    CLUSTER="$1"
    PLUGIN="$(select_stack_plugin)"

    online_status_$PLUGIN "$CLUSTER"
}

status()
{
    load_status

    CLUSTER="${1:-${O2CB_BOOTCLUSTER}}"
    if [ -z "$CLUSTER" ]
    then
        return 1;
    fi

    online_status "$CLUSTER"
}



case "$1" in
    start)
        start "$2"
        ;;

    status)
        status "$2"
        ;;

    stop)
        stop "$2"
        ;;

    restart)
        stop "$2"
        start "$2"
        ;;

    force-reload)
        stop "$2"
        start "$2"
        ;;

    load)
        load
        ;;

    online)
        load
        online "$2"
        ;;

    offline)
        offline "$2"
        ;;

    force-offline)
        offline "$2" 1
        ;;

    unload)
        offline "$2"
        unload
        ;;

    configure)
        configure
        if [ "$O2CB_ENABLED" = "true" ]
        then
            start
        else
            stop
        fi
        ;;

    enable)
        O2CB_ENABLED=true
        write_sysconfig
        if_fail "$?" "Unable to write the driver configuration"
        start
        ;;

    disable)
        O2CB_ENABLED=false
        write_sysconfig
        if_fail "$?" "Unable to write the driver configuration"
        stop
        ;;

    *)
        echo "Usage: $0 {start|stop|restart|force-reload|enable|disable|configure|load|unload|online|offline|force-offline|status}"
        exit 1
        ;;
esac

exit 0