bash Script Full Diff backup with retention

Details: Category: Backup Systems; Written by Mahdi Bahmani Ciahmard; Hits: 87

This Script make solution For Full Diff with Retention to Keep number of Backup Files

it-27:/home/mbahmani# /bin/sh /root/.scripts/Backup_run.sh /root/.scripts/Backup_info_etc

gbgcache:~/.scripts# ls
bin policy RubBK
gbgcache:~/.scripts# ls -cR *
RubBK

policy:
gbgcache-etc gbgcache-root gbgcache-var

policy/gbgcache-etc:
Backup_info_etc Backup_info_etc_exclude Backup_info_etc_exclude_dirs conv

policy/gbgcache-root:
Backup_info_root

policy/gbgcache-var:
Backup_info_var Backup_info_var_exclude

bin:
Backup_run.sh
gbgcache:~/.scripts#ls

How Run

gbgcache:~# sh .scripts/RubBK
Job Number is:26339
/root/.scripts/policy/gbgcache-var/Backup_info_var
/root/.scripts/policy/gbgcache-var/Backup_info_var_exclude

**********************************************
#########Initializing Variables#############

**********************************************

**********************************************
########Backup info file Items#############

**********************************************
/tmp/backuplog/26339/26339

**********************************************
#########Backup Exclude Files List##########

**********************************************
/tmp/backuplog/26339/3067

**********************************************
#########Backup Exclude Dirs List##########

**********************************************

**********************************************
##########verify_backup_dirs#############

**********************************************

**********************************************
########Retention Proccess Running#########

**********************************************

**********************************************
########Making Backup#################

**********************************************

gbgcache:~/.scripts#ls

gbgcache:~/.scripts# cat RubBK

bin/sh /root/.scripts/bin/Backup_run.sh /root/.scripts/policy/gbgcache-etc/Backup_info_etc /root/.scripts/policy/gbgcache-etc/Backup_info_etc_exclude /root/.scripts/policy/gbgcache-etc/Backup_info_etc_exclude_dirs

Backup_run.sh backupconfigfile backupexcludefile backupexcludedir

#########start#########

gbgcache:~# cat .scripts/bin/Backup_run.sh
#!/bin/bash
#This Script Created By Mehdi Bamani
#set -x
#Static Variables
INPUT_ARGS="$#"
#help
USAGE="./Backup_run.sh [Backup_info_path(FilePath)] [Backup_Exclude_file_path(FilePatch)] [Backup_Exclude_dirs_path(FilePatch)] -h [Print this help] " # Get todays day like Mon, Tue and so on NOW=$(date +"%a") # Get Current Date and Time for backup file name DateTime=$(date +-%d_%m_%Y-Time-%HH-%MM-%SS) # Tape devie name TAPE="/dev/st0"
# Exclude file
TAR_ARGS="-cpvvf"
# Path to binaries
TAR=/bin/tar
MT=/bin/mt
MKDIR=/bin/mkdir
RMTMPDIR="/bin/rm -rf "
ARGS=""
#Tempolary variables
# Import Backup_info_file from $ARG1 to $pwd/policy/$ARG1
RND1=$RANDOM
RND2=$RANDOM
RND3=$RANDOM
echo "Job Number is:$RND1"
echo $1
ARG1=$1

echo $2
ARG2=$2

echo $3
ARG3=$3

#Array to import backup info and set tmp Variables
declare -a linearray
#TMPF="$pwd/policy/tmp/$RND1"
TMPF="/tmp/backuplog/$RND1"
#Log
LOGBASE=""
# Backup Log file
LOGFIILE=""
#Full or Diff Backup Label Name
Backup_Label=""
# Backup root dirs; do not prefix /
Data_ROOT_DIR=""
#Data complete path
Data_Path=""
# Disk Device Path
Disk=""
#Retention for Full and Diff to keep files
RetentionFullDays=""
RetentionDiffDays=""
#Path of exclude files
EXCLUDE_CONF_files=""
#Path of exclude Directories
EXCLUDE_CONF_dirs=""

#### Main functions #####
# Make a full backup
full_backup(){
Full=$RND1-Full_$Backup_Label-$NOW$DateTime.tar
echo ""
echo "*********************************************"
echo "#######Making Full Backup####################"
echo "$TAR $TAR_ARGS $Disk$Full $EXCLUDE_CONF_dirs $EXCLUDE_CONF_files $Data_ROOT_DIR"
echo "*********************************************"
local old=$(pwd)
cd $Data_Path
$TAR $TAR_ARGS $Disk$Full $EXCLUDE_CONF_dirs $EXCLUDE_CONF_files $Data_ROOT_DIR
cd $old
exit 0
}

# Make a partial backup
partial_backup(){
Diff=$RND1-Diff_$Backup_Label-$NOW$DateTime.tar
echo ""
echo "**********************************************"
echo "#########Making Diff Backup###################"
echo "$TAR $TAR_ARGS $Disk$Diff $EXCLUDE_CONF_dirs $EXCLUDE_CONF_files -N '"$(date -d '1 day ago')"' $Data_ROOT_DIR"
echo "**********************************************"
local old=$(pwd)
cd $Data_Path
# $MT -f $TAPE rewind
# $MT -f $TAPE offline
$TAR $TAR_ARGS $Disk$Diff $EXCLUDE_CONF_dirs $EXCLUDE_CONF_files -N "$(date -d '1 day ago')" $Data_ROOT_DIR
cd $old
}

# Make sure all dirs exits
verify_backup_dirs(){
local s=0
for d in $Data_Path$Data_ROOT_DIR
do
if [ ! -d $d ];
then
echo "Error : /$d directory does not exits!"
s=1
fi
done
# if not; just die
[ $s -eq 1 ] && exit 1
}

# Function to keep Number of Backup Files
Retention_backup_files(){
cd $Disk
find . -name 'Full*.tar' -mtime $RetentionFullDays -print -exec rm -rvf {} \;
find . -name 'Diff*.tar' -mtime $RetentionDiffDays -print -exec rm -rvf {} \;

}

Clear_Variables(){
#Clear Variables

#Log
LOGBASE="" #Full or Diff Backup Label Name Backup_Label=""
# Backup root dirs; do not prefix /
Data_ROOT_DIR=""
#Data complete path
Data_Path=""
# Disk Device Path
Disk=""
#Retention for Full and Diff to keep files
RetentionFullDays=""
RetentionDiffDays=""
$RMTMPDIR $TMPF

}

Set_Variables(){
echo ""
echo "**********************************************"
echo "#########Initializing Variables###############"
echo ""
echo "**********************************************"
j=1
for line in `cat $TMPF/$RND1`;do
linearray[$j]=$line
let "j += 1"
done

#Log
LOGBASE=${linearray[1]}
LOGFIILE=$LOGBASE$RND1$DateTime.backup.log

#Full or Diff Backup Label Name
Backup_Label=${linearray[2]}

# Backup root dirs; do not prefix /
Data_ROOT_DIR=${linearray[3]}

#Data complete path
Data_Path=${linearray[4]}

# Disk Device Path
Disk=${linearray[5]}

#Retention for Full and Diff to keep files
RetentionFullDays=${linearray[6]}
RetentionDiffDays=${linearray[7]}

if [ -f "$ARG2" ];then

#Path of exclude file
EXCLUDE_CONF_files="-X $TMPF/$RND2" fi if [ -f "$ARG3" ];then
#Convert convert 1 column into 1 rows for exclude dirs
for j in `cat $TMPF/$RND3|grep -v "^$"|grep -v "^#"`
do
STR=${STR}" "${j}
done
#echo ${STR} | sed 's/^,//g'
#Path of exclude Directories
EXCLUDE_CONF_dirs=$STR
fi

}

#### Main logic ####

# Make sure log dir exits
[ ! -d $LOGBASE ] && $MKDIR -p $LOGBASE >> $LOGFIILE 2>&1

#check number of input ARGs it must be at least one
if [ "$#" == "0" ];then
echo ""
echo "**********************************************"
echo -n "$USAGE"
echo ""
echo "**********************************************"
#Clear_Variables
exit 1
fi

if [ $ARG1 = '-h' ]; then
echo ""
echo "**********************************************"
echo "$USAGE"
#lear_Variables
echo ""
echo "**********************************************"
exit 1
fi

[ ! -d $TMPF ] && $MKDIR -p $TMPF
if [ -f "$ARG1" ];then
sed '/#/ d' $ARG1 > $TMPF/$RND1
fi
if [ -f "$ARG2" ];then
sed '/#/ d' $ARG2 > $TMPF/$RND2
fi
if [ -f "$ARG3" ];then
sed '/#/ d' $ARG3 > $TMPF/$RND3
fi

# Test whether command-line argument is present (non-empty)
if [ -f $ARG1 ]; then
Set_Variables
echo ""
echo "**********************************************"
echo "########Backup info file Items################"
echo ""
echo "**********************************************"
echo "Job Number is:$RND1" >> $LOGFIILE 2>&1
echo "Backup Policy Name is:$Backup_Label" >> $LOGFIILE 2>&1
ls $TMPF/$RND1
echo "########Backup info file Items################" >> $LOGFIILE 2>&1
cat $TMPF/$RND1|grep -v ^# |grep -v ^$ >> $LOGFIILE 2>&1
if [ -f $ARG2 ]; then
echo ""
echo "**********************************************"
echo "#########Backup Exclude Files List############"
echo ""
echo "**********************************************"
if [ -f $TMPF/$RND2 ]; then
ls $TMPF/$RND2
echo "#########Backup Exclude Files List############" >> $LOGFIILE 2>&1
cat $TMPF/$RND2 >> $LOGFIILE 2>&1
fi
fi
if [ -f $ARG3 ]; then
echo ""
echo "**********************************************"
echo "#########Backup Exclude Dirs List#############"
echo ""
echo "**********************************************"
if [ -f $TMPF/$RND3 ]; then
ls $TMPF/$RND3
echo "#########Backup Exclude Dirs List#############" >> $LOGFIILE 2>&1
cat $TMPF/$RND3 >> $LOGFIILE 2>&1
fi
fi

fi

# Verify dirs
echo ""
echo "**********************************************"
echo "##########verify_backup_dirs##################"
echo ""
echo "**********************************************"
verify_backup_dirs

# start backup procedure
echo ""
echo "**********************************************"
echo "########Retention Proccess Running############"
echo ""
echo "**********************************************"
Retention_backup_files
echo ""
echo "**********************************************"
echo "########Making Backup#########################"
echo ""
echo "**********************************************"
case $NOW in
Fri)
echo ""
echo "**********************************************"
echo "#######Making Full Backup#####################"
echo ""
echo "**********************************************"
full_backup;;
Sun|Mon|Tue|Wed|Thu)
echo ""
echo "**********************************************"
echo "######Making Diff Backup######################"
echo ""
echo "**********************************************"
partial_backup;;
*);;

esac >> $LOGFIILE 2>&1

#set +x
gbgcache:~#ls

gbgcache:~/.scripts# cat policy/gbgcache-etc/Backup_info_etc
#!/bin/bash
#This Script Customized by Mehdi Bahmani
#Log #LOGBASE
/home/mbahmani/.log/
#Backup_Label Full or Diff Backup Label Name
gbgcache_etc
# Backup root dirs; do not prefix / Data_ROOT_DIR
etc
#Data complete path Data_Path
/
# Disk Device Path Disk
/mnt/bkibs/gbgcache/
#Retention for Full and Diff to keep files RetentionFullDays RetentionDiffDays
+14
+8

gbgcache:~/.scripts# cat policy/gbgcache-etc/Backup_info_etc_exclude
#EXCLUDE_CONF_files is the path of exclude file
# iso
# *.cpp~
#
#------------------------------------------------------------------------
#*.conf
*.conf

gbgcache:~/.scripts# cat policy/gbgcache-etc/Backup_info_etc_exclude_dirs
# Add files matching patterns such as follows (regex allowed):
--exclude exim4
--exclude apt
--exclude network
gbgcache:~/.scripts#

it-27:/home/mbahmani/.scripts#crontab -e

# m h dom mon dow command

20 8 * * * /bin/sh /root/.scripts/RubBK

gbgcache:~# cat /home/mbahmani/.log/26339-17_12_2009-Time-10H-38M-13S.backup.log
Job Number is:26339
Backup Policy Name is:gbgcache_var
########Backup info file Items################
/home/mbahmani/.log/
gbgcache_var
var
/
/mnt/bkibs/gbgcache/
+14
+8
#########Backup Exclude Files List############
daemon.log
kern.log

**********************************************
######Making Diff Backup######################

**********************************************

**********************************************
#########Making Diff Backup###################
/bin/tar -cpvvf /mnt/bkibs/gbgcache/26339-Diff_gbgcache_var-Thu-17_12_2009-Time-10H-38M-13S.tar -X /tmp/backuplog/26339/3067 -N 'Wed Dec 16 10:38:13 IRST 2009' var
**********************************************
/bin/tar: Option --after-date: Treating date `Wed Dec 16 10:38:13 IRST 2009' as 2009-12-16 10:38:13
drwxr-xr-x root/root 0 2008-11-29 08:28 var/
drwx------ root/root 0 2008-11-29 05:01 var/lost+found/
drwxr-xr-x root/root 0 2009-12-17 08:37 var/log/
/bin/tar: var/log/daemon.log.4.gz: file is unchanged; not dumped
/bin/tar: var/log/mail.warn: file is unchanged; not dumped

List the files:

gbgcache:~# tar -tvf /mnt/bkibs/gbgcache/Diff_gbgcache_etc-Tue-15_12_2009-Time-12H-55M-50S.tar |grep apt

tips:
Extract the entire archive into current directory:

mbahmani@it-27:~$ tar xvpf /mnt/work/bahmani/mbctux-site/backup/sitembctux/Full_Wed-27_05_2009-Time-13H-51M-01S.tar

Extract only certain files or dirs into current directory. For example.
mbahmani@it-27:~$ tar xvpf /mnt/work/bahmani/mbctux-site/backup/sitembctux/Full_Wed-27_05_2009-Time-13H-51M-01S.tar mbctux/language/en/admin.po

Nav view search

Navigation

Search

bash Script Full Diff backup with retention