Copy and Compress the Datafiles using multiple Processes.

Following are the scripts used for copying and compressing the Datafiles within the Same Server.

Scripts
*********
1. copy_process.sh ------> This File consists of all the functions which are used for copy and compress.

2. copy_file_process.sh ------> This File consists of commands used for copying , compressing , uncompressing . This file is being called by the copy_process.sh

3. worker_no ------> Define the No. of Workers (Process) for the whole process. This value can be dynamically changed by using the command , echo 4 > worker_no.


copy_process.sh
****************

#! /usr/bin/ksh
##################################################################################
# bkp_dir_path is the source (TO directory ) name
# src_path is the target (From directory ) name
# worker_pid is the worker pid file
# worker_no is the number of workers, can be adjusted while the script is running
# example, to set 3 workers, perform the following before running the script: echo 3 > worker_no
# file

function Gen_Env
{
bkp_dir_path="/tmp/to_data"
src_path="/tmp/from_data"
Log_Date=$(date +"%d%m%y")
worker_pid=/tmp/copy_process_
worker_no=/tmp/worker_no
worker_max_count=0
COPY_FILE_PROCESS=/tmp/copy_file_process.sh
File_List=$(cd $src_path; ls -l *.dbf |grep -v cntrl | awk '{print $9}')
}

function Copy_Phase
{
num_copy_workers=`cat $worker_no`
if [[ $worker_max_count -lt $num_copy_workers ]] then
worker_max_count=num_copy_workers
fi

worker=1
while [[ $worker -le $num_copy_workers ]]
do
worker_file=${worker_pid}${worker}.pid
if [[ ! -s $worker_file ]] then
echo "Busy" > $worker_file
$COPY_FILE_PROCESS $src_path $datafile $worker $worker_file $bkp_dir_path &
echo "Copy Assigned to Worker pid file: $worker"
Copy_Assigned=Yes
return
fi
let worker=$worker+1
done
sleep 5
}

function Check_Final_Copy
{
worker=1
while [[ $worker -le $worker_max_count ]]
do
worker_file=${worker_pid}${worker}.pid
if [[ -s $worker_file ]] then
echo "Background Copy is still Running... $(date)"
sleep 60
CheckFinal=No
return
else
rm -f $worker_file
fi
let worker=$worker+1
done
CheckFinal=Yes
}

function main
{
Gen_Env
echo "Total number of Datafiles in the Source Instance"
src_dbf_count=`ls $src_path|wc -l`
echo $src_dbf_count
date > $bkp_dir_path/time.log
for datafile in `echo $File_List`
do
echo "Copy $datafile : $(date)"
echo "Wait for the Next Worker ... $(date)"
Copy_Assigned=No
while [[ $Copy_Assigned == 'No' ]]
do
Copy_Phase
done
done

CheckFinal=No
while [[ $CheckFinal == 'No' ]]
do
Check_Final_Copy
done
date >> $bkp_dir_path/time.log
echo "**********************************************"
echo " Copy Process Completed Successfully"
echo "**********************************************"
echo "Number of Data files copied to the target "
target_dbf_count=`ls $bkp_dir_path|wc -l`
echo $target_dbf_count
}
main

################### End of Copy_process.sh ###########


copy_file_process.sh
**********************

#!/usr/bin/ksh
##############################################################
## Copy and compress script
## Phase - I = Copy the files from Source to Destination
## Phase - II = Compress the Destination Files.
## Phase - III = Uncompress the Destination File. (if required)
##
##
##############################################################
s_path=$1
filename=$2
worker_no=$3
statusfile=$4
d_path=$5
statuss=`cat $statusfile`
##############################################################
if [[ $statuss == 'Busy' ]] then
echo "$$" >$statusfile
## Phase I
echo "Copying $filename by worker $worker_no"
cp $s_path/$filename $d_path
sleep 5

# Phase II
echo "Zipping $filename by worker $worker_no"
/usr/bin/gzip $d_path/$filename
sleep 5

#Phase III
#echo "Unzipping $filename.gz by worker $worker_no"
#/usr/bin/gunzip $s_path/$filename.gz

>$statusfile
else
echo "Process is not Busy"
fi

echo '*********************'


################### END of copy_file_process.sh #############


Execution Syntax
*****************

1. Need to update the directory structure for the below variables within the script "copy_process.sh".

a) bkp_dir_path ---> Where to backup the datafiles.
b) src_path ---> Location of source data files.
c) COPY_FILE_PROCESS ---> Location of "copy_file_process.sh script.


Syntax
********

nohup ./Location of copy_process.sh &