Restoring a controller team
![[NOTE: ]](images/note.gif) | NOTE: Before running this script, re-install the controller. Otherwise an Error 404 condition results and the controller is not restored. See Restoring a controller from a backup . Because the scripts in this appendix cross page boundaries, be careful to avoid including the page number when copying a script. Copying a script one page at a time can prevent inclusion of page numbers. |
#!/bin/bash
#-------------------------------------------------------------------------------
# Copyright 2013 Hewlett Packard Co., All Rights Reserved.
#-------------------------------------------------------------------------------
#
# Restore a Team
#-------------------------------------------------------------------------------
export BACKUP_DIR="/opt/sdn/backup"
export BACKUP_TEAM_DIR="/opt/sdn/team_backup"
export RESTORE_TEAM_DIR="/opt/sdn/team_restore"
export TEAM_BACKUP_STATUS_FILE="$RESTORE_TEAM_DIR/teamRestore_status"
export TEAM_BACKUP_LOGFILE="$RESTORE_TEAM_DIR/teamRestore_log.log"
export RESTORE_BACKUP_FILESET="$RESTORE_TEAM_DIR/opt/sdn/team_backup"
export B_PID=$$
trap "exit 1" TERM
#==============================================================================
# F U N C T I O N S
#==============================================================================
#------------------------------------------------------------------------------
# Function extract_zip_and_ip ( )
# Extracts the team backup zip and the backed up IP addresses.
#------------------------------------------------------------------------------
function extract_zip_and_ip {
unzip -o "$RESTORE_TEAM_DIR/sdn_team_backup*" -d $RESTORE_TEAM_DIR
if [ $? -ne 0 ]; then
teamBackup_log "Failed to unzip the team backup file."
exitBackup 1
fi
teamBackup_log "Extracted the team backup file successfully."
rm -rf "$RESTORE_TEAM_DIR/sdn_team_backup*"
backupIp=($(ls $RESTORE_BACKUP_FILESET | grep "zip$" | sed "s/.zip//" | \
sed "s/.Leader//" | sed "s/sdn_controller_backup_//"))
numBackup=${#backupIp[@]}
teamBackup_log "Found $numBackup backup file sets in the team backup file."
}
#------------------------------------------------------------------------------
# Function create_restoreDir ( )
# Creates the team restore directory.
#------------------------------------------------------------------------------
function create_restoreDir {
rm -rf $RESTORE_TEAM_DIR
mkdir $RESTORE_TEAM_DIR
chmod 777 $RESTORE_TEAM_DIR
}
#------------------------------------------------------------------------------
# Function validate_my_Ip ( )
# Validates the configured node IP against the backed up IP addresses.
#------------------------------------------------------------------------------
function validate_my_Ip {
for (( v=0; v<numBackup; v++ )); do
myip='ifconfig|grep -o "${backupIp[$v]}"'
if [ "$myip" != "" ]; then
teamBackup_log "IP $myip is a valid member of the team."
return
fi
done
teamBackup_log "IP $myip is not a valid member of the team, exiting."
exitBackup 1
}
#------------------------------------------------------------------------------
# Function upload_backup_file ( <systemIp> <systemUUID> <authToken> <zipFile> )
# Uploads backup file to the specific nodes of the team.
#------------------------------------------------------------------------------
function upload_backup_file {
local sysIp=$1
local sysUUID=$2
local sysAuth=$3
local uploadUrl="https://$sysIp:8443/sdn/v2.0/restore/backup"
local zipFile=$4
if [ ! -f $zipFile ]; then
teamBackup_log "File $zipFile does not exist."
exitBackup 1
fi
curl --noproxy $sysIp -X POST --fail -ksSfL --url $uploadUrl \
-H "X-Auth-Token:$sysAuth"\
--data-binary @$zipFile
if [ $? -ne 0 ]; then
teamBackup_log "Failed to upload backup $zipFile to $sysIp."
exitBackup 1
fi
teamBackup_log "Backup $zipFile uploaded successfully to $sysIp."
}
#------------------------------------------------------------------------------
# Function restore_node ( <systemIp> <systemUUID> <authToken> )
# Restores a particular node.
#------------------------------------------------------------------------------
function restore_node {
local sysIp=$1
local sysUUID=$2
local sysAuth=$3
local restoreUrl="https://$sysIp:8443/sdn/v2.0/restore"
# Set the IP first. Ignore errors since this only works for standalone.
put $sysIp $sysAuth "https://$sysIp:8443/sdn/v2.0/systems/$sysUUID" \
"{\"system\":{\"ip\":\"$sysIp\"}}" > /dev/null 2>&1
restoreSession='post $sysIp $sysAuth $restoreUrl '
if [ $errorCode -ne 0 ]; then
teamBackup_log "Failed to start restore on node $sysIp."
exitBackup 1
fi
teamBackup_log "Started restore on node $sysIp."
}
#------------------------------------------------------------------------------
# Function validate_node_status ( )
# Validates node status after the restore.
#------------------------------------------------------------------------------
function validate_node_status {
local sysIp=$1
# Wait for the restore to complete.
local sysUrl="https://$sysIp:8443/sdn/v2.0/systems"
for (( k=0; k<100; k++ )); do
sleep 30
authToken='getAuthToken $sysIp'
[ "$authToken" == "" ] && continue
# Try to contact the system.
data='get $sysIp $authToken "$sysUr?ip=$sysIp"'
[ "$data" == "" ] && continue
teamBackup_log "Node:$sysIp came up successfully." && return
done
teamBackup_log "Node:$sysIP failed to come up."
exitBackup 1
}
#------------------------------------------------------------------------------
# Function restore_nodes ( <ipAddrArray> )
# Restores only the specified node(s).
#------------------------------------------------------------------------------
function restore_nodes {
local leaderindex=-1
local restoreIpArr=("$@")
local numNodes=${#restoreIpArr[@]}
for (( i=0; i<$numNodes; i++ )); do
# Get the auth token for a specific node.
restoreAuth[$i]='getAuthToken ${restoreIpArr[$i]}'
if [ "${restoreAuth[$i]}" == "" ]; then
teamBackup_log "Failed to get the auth Token for ${restoreIpArr[$i]}, can't start restore."
exitBackup 1
fi
uuidURL="https://${restoreIpArr[$i]}:8443/sdn/v2.0/systems"
restoreUUID[$i]='get ${restoreIpArr[$i]} ${restoreAuth[$i]} "$uuidURL"'
if [ "${restoreUUID[$i]}" == "" ]; then
teamBackup_log "Failed to get the UUID for ${restoreIpArr[$i]}, can't start restore."
exitBackup 1
fi
restoreUUID[$i]='extractJSONString "${restoreUUID[$i]}" "uid" | sed '/^$/d''
teamBackup_log "UUID for ${restoreIpArr[$i]} is ${restoreUUID[$i]}"
# Upload the backup files to a specific node.
local ipFileName="sdn_controller_backup_${restoreIpArr[$i]}*.zip"
local zipFile='ls $RESTORE_BACKUP_FILESET/$ipFileName'
upload_backup_file ${restoreIpArr[$i]} ${restoreUUID[$i]} \
${restoreAuth[$i]} $zipFile
# Check if this is the leader node from the backup set.
local leaderZip='echo $zipFile|grep "Leader"'
[ "$leaderZip" != "" ] && leaderIndex=$I
done
# Start restore in the leader node first before all the other nodes.
if [ $leaderIndex -ne -1 ]; then
restore_node ${restoreIpArr[$leaderIndex]} ${restoreUUID[$leaderIndex]} \
${restoreAuth[$leaderIndex]}
fi
# Verify the leader node is up after the restore.
validate_node_status ${restoreIpArr[$leaderIndex]}
# Continue restore on the remaining nodes.
for (( i=0; i<$numNodes; i++ )); do
# Skip the leader node; it's already done.
[ $i -eq $leaderIndex ] && continue
# Restore the specified node.
restore_node ${restoreIpArr[$i]} ${restoreUUID[$i]} ${restoreAuth[$i]}
done
sleep 200
# Validate that the restored nodes are up.
for (( n=0; n<$numNodes; n++ )); do
# Skip the leader node; it's already done.
[ $n -eq $leaderIndex ] && continue
validate_node_status ${restoreIpArr[$n]}
done
}
#------------------------------------------------------------------------------
# Function teamBackup_log ( <message> )
# Writes messages to the log for the team backup operation.
#------------------------------------------------------------------------------
function teamBackup_log {
msg="$1"
echo "$msg" |tee -a $TEAM_BACKUP_LOGFILE
}
#------------------------------------------------------------------------------
# Function exitBackup ( <exitStatus> )
# Exits the backup.
#------------------------------------------------------------------------------
function exitBackup {
[ $1 -ne 0 ] && teamBackup_log "Stopping backup/restore with errors."
rm -rf $TEAM_BACKUP_STATUS_FILE
kill -s TERM $B_PID
exit $1
}
#------------------------------------------------------------------------------
# Function get ( <ipAddr> <authToken> <url> )
# Performs a GET.
#------------------------------------------------------------------------------
function get {
local getIP=$1
local getToken=$2
local getUrl=$3
local attempts=0
while [ $attempts -lt 5 ]; do
curl --noproxy $getIP --header "X-Auth-Token:$getToken" \
--fail -ksS -L -f --request GET --url "$getUrl"
errorCode=$
?
let "attempts = $attempts + 1"
if [ 35 -eq $errorCode ]; then
teamBackup_log "SSL error on GET of $getUrl, retrying..."
continue;
fi
break;
done
}
#------------------------------------------------------------------------------
# Function post ( <ipAddr> <authToken> <url> <data>)
# Performs a POST of the specified data.
#------------------------------------------------------------------------------
function post {
local postIP=$1
local postToken=$2
local postUrl=$3
local postData=$4
local attempts=0
while [ $attempts -lt 5 ]; do
postRes='curl --noproxy $postIP --header "X-Auth-Token:$postToken" \
--fail -ksS --request POST --url "$postUrl" --data-binary "$postData"'
errorCode=$?
let "attempts = $attempts + 1"
if [ 35 -eq $errorCode ]; then
teamBackup_log "SSL error on POST to $postUrl, retrying..."
continue;
fi
break;
done
echo $postRes
}
#------------------------------------------------------------------------------
# Function put ( <ipAddr> <authToken> <url> <data> )
# Performs a PUT of the specified data.
#------------------------------------------------------------------------------
function put {
local putIP=$1
local putToken=$2
local putUrl=$3
local putData=$4
local attempts=0
while [ $attempts -lt 5 ]; do
putRes='curl --noproxy $putIP --header "X-Auth-Token:$putToken" \
--fail -ksS -L -f --request PUT "$putUrl" --data-binary "$putData"'
errorCode=$?
let "attempts = $attempts + 1"
if [ 35 -eq $errorCode ]; then
teamBackup_log "SSL error on POST to $putUrl, retrying"
continue;
fi
break;
done
echo $putRes
}
#------------------------------------------------------------------------------
# Function extractJSONString ( <json> <fieldName> )
# Extracts the Json value corresponding to the field name.
#------------------------------------------------------------------------------
function extractJSONString {
json=$1
field=$2
json='echo $json|tr -d '"'| sed -e 's/\,\|{/\n/g'|grep -w "$field"| \
cut -d ':' -f2-'
echo $json
}
#------------------------------------------------------------------------------
# Function getAuthToken ( <ipAddr> )
# Log-in and get the UID.
#------------------------------------------------------------------------------
function getAuthToken {
local nodeIP=$1
url="https://$nodeIP:8443/sdn/v2.0/auth"
login="{
\"login\": {
\"domain\": \"$domain\",
\"user\": \"$user\",
\"password\": \"$pass\"
}
}
}"
# Attempt to authenticate and extract token if successful.
auth=$(curl --noproxy $nodeIP -X POST --fail -ksSfL --url "$url" \
-H "Content-Type: application/json" --data-binary "$login" 2>&1)
if [ $? -ne 0 ]; then
teamBackup_log "Unable to authenticate as user $user in $domain domain."
exitBackup 1
fi
authToken='extractJSONString "$auth" "token" | sed '/^$/d''
if [ $restore_mode -ne 1 ] && [ "$authToken" == "" ]; then
teamBackup_log "Failed to get the authentication token."
exitBackup 1
fi
echo $authToken
}
#==============================================================================
# M A I N
#==============================================================================
restore_mode=1
selective_restore=0
# Check for unzip package.
command -v unzip &> /dev/null
if [ $? -ne 0 ]; then
echo "The unzip package must be installed to use this script."
exit 1
fi
# Check the user specified script parameters.
if [ $# -lt 3 ]; then
echo "Usage : restoreTeam <user> <domain> [<ip1> <ip2> ...] <user@IP:path>"
echo " <user> - user name to access the controller"
echo " <domain> - domain of the controller"
echo " [<ip1> <ip2> ...] - ip(s) of node(s) to be restored; if none are specified all nodes are restored"
echo " <user@IP:path> - remote location to retrieve backup file"
echo " user - the login name for the system"
echo " ip - the ip address of the system"
echo " path - where to copy the file from on the remote system"
exit 1
fi
create_restoreDir
user="$1"
echo -n "Enter Controller Password: "
read -s pass
echo
domain="$2"
file=""
if [ $# -eq 3 ]; then
teamBackup_log "Starting the team restore. This will restore all the nodes in a team."
file=$3
else
teamBackup_log "Starting selective restore on specified IPs. This restore will happen only on the specified nodes."
count=0
selective_restore=1
for ip in "$@"; do
restoreIp[$count]=$ip
let "count = $count + 1"
done
fileIndex=$(($# - 1))
file=${restoreIp[$fileIndex]} && unset restoreIp[$fileIndex]
fi
# Upload the team backup file from the user specified location.
scp $file $RESTORE_TEAM_DIR
if [ $? -ne 0 ]; then
teamBackup_log "Failed to upload team backup file to the node."
exitBackup 1
fi
# Unzip the team backup file.
extract_zip_and_ip
# Validate the IP address of the node.
validate_my_Ip
# Restore the node(s).
if [ $selective_restore -eq 1 ]; then
restore_nodes ${restoreIp[@]}
else
restore_nodes ${backupIp[@]}
fi
echo
teamBackup_log "The team was restored successfully."
exitBackup 0