#!/bin/bash

#======================================================
# Author: Patrick.Brockmann@cea.fr
#
#======================================================

#------------------------------------------------------
switchtotar=0
maxtries=3

#---------------------------------------------
while [ $# -ne 0 ]
do
        case $1 in
       --switchtotar)
		switchtotar=1
		shift 1 ;;
       --maxtries)
		maxtries=$2
		shift 2 ;;
       -h|--help|-help)
                echo "----------------------------------------------------------------------------"
		echo "Usage: enlarge_my_files.sh [--switchtotar] [--maxtries N] dirin dirout"
		echo
		echo "Description:"
                echo "   Pass a ncrcat command on files from prepared sets of files."
                echo "   2 shots will be done at each try".
                echo  
                echo "   A status file and a log file will be created in enlarge_my_files/with_ncrcat directory."
                echo "   Copy a set to enlarge_my_files/with_tar directory for later processing after N tries."
		echo
		echo "Arguments:"
                echo "   dirin is the input directory where files to concatenate will be read"
                echo "   dirout is the output directory where concatenated files will be written"
		echo
                echo "   The following directory must exist where the command enlarge_my_files is run:"
		echo "       enlarge_my_files"
		echo "                       |-- with_ncrcat"
		echo "                       |   |-- set_000001.txt"
		echo "                       |   |-- set_000002.txt"
		echo "                       |   |-- set_000003.txt"
		echo "                       |   |-- set_000004.txt"
		echo "                       |   \`-- set_000005.txt"
		echo "                       \`-- with_tar"
		echo
		echo "Options:"
                echo "   -h, --help, -help"
                echo "       Print this manual."
                echo "   --switchtotar"
                echo "       Abort ncrcat operation and switch to tar command"
                echo "   --maxtries N"
                echo "       Maximum number of tries before switching to tar command" 
                echo "       Default is 3"
		echo
		echo "----------------------------------------------------------------------------"
		exit ;;
        -*)
                `dirname $0`/enlarge_my_files.sh -h
                exit ;;
        *)
                break ;;
        esac
done

if [ $# -lt 2 ] ; then
        `dirname $0`/enlarge_my_files.sh -h
        exit
fi

if [[ ! -d $PATH_SIMU/with_ncrcat || ! -d $PATH_SIMU/with_tar ]] ; then
        `dirname $0`/enlarge_my_files.sh -h
        exit
fi

#------------------------------------------------------
dirin=$1
dirout=$2

#------------------------------------------------------
function find_varstoexclude {
nbfile=0
for file in `cat $1` ; do
    ncdump -h ${file} | gawk '{if (match($0, /(byte|char|short|int|float|double) (.*)\(/, arr)) print arr[2] }' >> tmp_$$.txt
    let nbfile=nbfile+1
done

varstoexclude=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if ($1 != nbfile) {print $2}}' | paste -s -d','`
varstoexcludefiltered=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if (($1 != nbfile) && !(match($2, /t_inst_(.*)/))) {print $2}}' | paste -s -d','`
nbvars=`echo $varstoexcludefiltered | gawk -F',' '{print NF}'`

rm -f tmp_$$.txt

return 0 
}

#------------------------------------------------------
for set in $PATH_SIMU/with_ncrcat/*list ; do

	#-----------------------------------
	echo "#-------------------"
	echo "Set: $set"
        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	statusfile=${set%%.list}.status
	logfile=${set%%.list}_${datestr}.log

	#-----------------------------------
	# Create status file if not exists
	if [ ! -e $statusfile ] ; then
		echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
	fi

	#-----------------------------------
	# Switch to tar command
	if [ $switchtotar -eq 1 ] ; then
		echo "====> DELEGATE to tar command with option --switchtotar"
		echo "DELEGATE to tar command with option --switchtotar" >> $statusfile
		mv $set $PATH_SIMU/with_tar
		continue
	fi

	#-----------------------------------
	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
	if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
		printf "====> "
		tail -1 $statusfile
		continue
	fi

	#-----------------------------------
	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
	nbtries=`wc -l $statusfile | cut -d' ' -f1`		
	echo "Nb of tries: $nbtries"

	#-----------------------------------
	# Set output file name from first and last files from the current set
	filefirst=`head -n 1 $set`
	fileoutpart1=`basename $filefirst | cut -d'_' -f-2`
	filelast=`tail -n 1 $set`
	fileoutpart2=`basename $filelast | cut -d'_' -f3-`

	diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"`
#	outputfile=$diroutputfile/ncrcat_${fileoutpart1}_${fileoutpart2}
	outputfile=$diroutputfile/${fileoutpart1}_${fileoutpart2}
	mkdir -p $diroutputfile
	echo "Output file to write: $outputfile"

	#-----------------------------------
	# Find variables to exclude to have homogeneous files (always exclude 't_inst_.*')
	extratar=0
	find_varstoexclude $set
	if [ $nbvars -eq 0 ] ; then
		varstoexclude="t_inst_.*"
	else
		# There are others variables than 't_inst_.*' so an extra tar file will be created
		varstoexclude="t_inst_.*,$varstoexcludefiltered"
		extratar=1
		# Copy to with_tar directory 
		cp $set $PATH_SIMU/with_tar
	fi

	#-----------------------------------
	# Try 2 times before fire "FAILED"
	ncrcatcmd=ncrcat
	#ncrcatoptions="--md5_digest"
	ncrcatoptions=""

	cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1
	if [ $? -eq 0 ] ; then
		datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
		echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
		echo "====> COMPLETED at try #$nbtries (at 1st shot on 2)"
		echo "COMPLETED at try #$nbtries (at 1st shot on 2)" >> $statusfile
	else
		# Add --md5_digest
		cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1
		if [ $? -eq 0 ] ; then
			datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
			echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
			echo "====> COMPLETED at try #$nbtries (at 2nd shot on 2)"
			echo "COMPLETED at try #$nbtries (at 2nd shot on 2)" >> $statusfile
		else
			datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
			echo "====> FAILED at try #$nbtries" 
			echo "FAILED ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
			if [ $nbtries -ge $maxtries ] ; then
				echo "====> DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)"
				echo "DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)" >> $statusfile
				cp $set $PATH_SIMU/with_tar
			fi
		fi
	fi
	
	#-----------------------------------
	
done	

#------------------------------------------------------

#------------------------------------------------------
for set in $PATH_SIMU/with_tar/*list ; do

	#-----------------------------------
	echo "#-------------------"
	echo "Set: $set"
        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	statusfile=${set%%.list}.status
	logfile=${set%%.list}_${datestr}.log

	#-----------------------------------
	# Create status file if not exists
	if [ ! -e $statusfile ] ; then
		echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
	fi

	#-----------------------------------
	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
	if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
		printf "====> "
		tail -1 $statusfile
		continue
	fi

	#-----------------------------------
	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
	# nbtries=`wc -l $statusfile | cut -d' ' -f1`		
	# echo "Nb of tries: $nbtries"

	#-----------------------------------
	# Set output file name from list name
	filefirst=`head -n 1 $set`
	dirin1=$( dirname $filefirst )
	fileout1=$( basename $set )
	fileout=${fileout1%.list}
	diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"`
#	outputfile=$diroutputfile/ncrcat_${fileoutpart1}_${fileoutpart2}
	outputfile=$diroutputfile/${fileout}.tar
	echo $outputfile
	mkdir -p $diroutputfile
	echo "Output file to write: $outputfile"

# 	#-----------------------------------
# 	# Find variables to exclude to have homogeneous files (always exclude 't_inst_.*')
# 	extratar=0
# 	find_varstoexclude $set
# 	if [ $nbvars -eq 0 ] ; then
# 		varstoexclude="t_inst_.*"
# 	else
# 		# There are others variables than 't_inst_.*' so an extra tar file will be created
# 		varstoexclude="t_inst_.*,$varstoexcludefiltered"
# 		extratar=1
# 		# Copy to with_tar directory 
# 		cp $set $PATH_SIMU/with_tar
# 	fi

	#-----------------------------------
	# Try 1 time before fire "FAILED"
	# Waiting for CCRT command
	tarcmd=echo
	taroptions="-cvf"
	
#	cd $dirin1
#	ls
	
	$tarcmd "$taroptions $outputfile $(cat $set)" > $logfile 2>&1
	if [ $? -eq 0 ] ; then
	    datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	    echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile ; $extratar" >> $statusfile
	    echo "====> COMPLETED at try #$nbtries (at 1st shot on 2)"
	    echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
	else
	    datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	    echo "====> FAILED at try #$nbtries" 
	    echo "FAILED ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile ; $extratar" >> $statusfile

	fi
	
	#-----------------------------------
	
done	



