Context Navigation

source: TOOLS/PACK_IPSL/enlarge_my_files.sh @ 1870

Last change on this file since 1870 was 1855, checked in by gpincka, 12 years ago
Traitement (code sequentiel) des simu de taille < 1 Go ou de type create_etat0
Property svn:executable set to ``*
File size: 25.1 KB

Line
1	#!/bin/bash
2	#------------------------------------------------------
3	switchtotar=0
4	maxtries=3
5	#---------------------------------------------
6	while [ $# -ne 0 ]
7	do
8	case $1 in
9	--switchtotar)
10	switchtotar=1
11	shift 1 ;;
12	--maxtries)
13	maxtries=$2
14	shift 2 ;;
15	-h\|--help\|-help)
16	echo "----------------------------------------------------------------------------"
17	echo "Usage: enlarge_my_files.sh [--switchtotar] [--maxtries N] dirin dirout"
18	echo
19	echo "Description:"
20	echo " Pass a ncrcat, tar or cp command on files from prepared sets of files."
21	echo " For ncrcat command 2 shots will be done at each try".
22	echo
23	echo " A status file and a log file will be created in output_ncrcat directory."
24	echo " Copy a set to output_tar directory for later processing after N tries."
25	echo
26	echo "Arguments:"
27	echo " dirin is the input directory where files to concatenate will be read"
28	echo " dirout is the output directory where concatenated files will be written"
29	echo
30	echo " The following directory must exist where the command enlarge_my_files is run:"
31	echo " enlarge_my_files"
32	echo " \|-- output_ncrcat"
33	echo " \| \|-- set_000001.list"
34	echo " \| \|-- set_000002.list"
35	echo " \| \|-- set_000003.list"
36	echo " \| \|-- set_000004.list"
37	echo " \| \`-- set_000005.list"
38	echo " \`-- output_tar"
39	echo
40	echo "Options:"
41	echo " -h, --help, -help"
42	echo " Print this manual."
43	echo " --switchtotar"
44	echo " Abort ncrcat operation and switch to tar command"
45	echo " --maxtries N"
46	echo " Maximum number of tries before switching to tar command"
47	echo " Default is 3"
48	echo
49	echo "----------------------------------------------------------------------------"
50	exit ;;
51	-*)
52	`dirname $0`/enlarge_my_files.sh -h
53	exit ;;
54	*)
55	break ;;
56	esac
57	done
58
59	if [ $# -lt 2 ] ; then
60	`dirname $0`/enlarge_my_files.sh -h
61	exit
62	fi
63
64	if ( \
65	[ ! -d $PATH_SIMU/output_ncrcat ] \|\| \
66	[ ! -d $PATH_SIMU/output_tar ] \|\| \
67	[ ! -d $PATH_SIMU/restart_tar ] \|\| \
68	[ ! -d $PATH_SIMU/store_cp ] \|\| \
69	[ ! -d $PATH_SIMU/work_cp ] \|\| \
70	[ ! -d $PATH_SIMU/debug_tar ] \|\| \
71	[ ! -d $PATH_SIMU/other_tar ] \
72	) \
73	&& \
74	[ ! -e $PATH_SIMU/tar_full_simul.list ]
75	then
76	`dirname $0`/enlarge_my_files.sh -h
77	exit
78	fi
79
80	#------------------------------------------------------
81	dirin=$1
82	dirout=$2
83	dirout_work=$3
84
85	#------------------------------------------------------
86	function find_varstoexclude {
87	nbfile=0
88	for file in `cat $1` ; do
89	# ncdump -h ${file} \| gawk '{if (match($0, /(byte\|char\|short\|int\|float\|double) (.*)\(/, arr)) print arr[2] }' >> tmp_$$.txt
90	ncdump -h ${file} \| grep -Ew "byte\|char\|short\|int\|float\|double" \| awk -F\( '{split($1,a," ") ; split($2,b,")") ; if (a[2] != b[1]) print a[2]}' >> tmp_$$.txt
91	let nbfile=nbfile+1
92	done
93
94	varstoexclude=`cat tmp_$$.txt \| sort \| uniq -c \| awk -v nbfile=$nbfile '{if ($1 != nbfile) {print $2}}' \| paste -s -d','`
95	varstoexcludefiltered=`cat tmp_$$.txt \| sort \| uniq -c \| awk -v nbfile=$nbfile '{if (($1 != nbfile) && !(match($2, /t_inst_(.*)/))) {print $2}}' \| paste -s -d','`
96	nbvars=`echo $varstoexcludefiltered \| gawk -F',' '{print NF}'`
97
98	rm -f tmp_$$.txt
99
100	return 0
101	}
102
103	# gpdebug : pour test showPackProgress.sh *************************
104	export RANDOM=$$
105
106	function gives_0_or_1
107	{
108	bit=-1
109	let "bit = RANDOM % 2"
110	echo $bit
111	}
112
113	resultCmd=
114	# gpdebug : **************************** fin ******************
115
116	source ${EXE_DIR}/DEM_utilities.sh # gpdebug : pour trouver les fonctions de mesure de temps
117
118	if ( \
119	[ -d $PATH_SIMU/output_ncrcat ] \|\| \
120	[ -d $PATH_SIMU/output_tar ] \|\| \
121	[ -d $PATH_SIMU/restart_tar ] \|\| \
122	[ -d $PATH_SIMU/store_cp ] \|\| \
123	[ -d $PATH_SIMU/work_cp ] \|\| \
124	[ -d $PATH_SIMU/debug_tar ] \|\| \
125	[ -d $PATH_SIMU/other_tar ] \
126	)
127	then
128
129	#------------------------------------------------------
130	for set in $( ls $PATH_SIMU/output_ncrcat/*list ) ; do
131
132	#-----------------------------------
133	echo "#-------------------"
134	echo "Set: $set"
135	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
136	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
137	statusfile=${set%%.list}.status
138	logfile=${set%%.list}_${datestr}.log
139
140	#-----------------------------------
141	# Create status file if not exists
142	if [ ! -e $statusfile ] ; then
143	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
144	fi
145
146	#-----------------------------------
147	# Switch to tar command
148	if [ $switchtotar -eq 1 ] ; then
149	echo "====> DELEGATE to tar command with option --switchtotar"
150	echo "DELEGATE to tar command with option --switchtotar" >> $statusfile
151	mv $set $PATH_SIMU/output_tar
152	continue
153	fi
154
155	#-----------------------------------
156	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
157	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
158	printf "====> "
159	tail -1 $statusfile
160	continue
161	fi
162
163	#-----------------------------------
164	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
165	nbtries=`wc -l $statusfile \| cut -d' ' -f1`
166	echo "Nb of tries: $nbtries"
167
168	#-----------------------------------
169	# Set output file name from list name
170
171	fileout1=$( basename $set )
172	fileout=${fileout1%.list}
173
174	filefirst=`head -n 1 $set`
175	diroutputfile=`dirname $filefirst \| sed -e "s%$dirin%$dirout%"`
176
177	outputfile=$diroutputfile/${fileout}
178	mkdir -p $diroutputfile
179	echo "Output file to write: $outputfile"
180
181	#-----------------------------------
182	# Find variables to exclude to have homogeneous files (always exclude 't_inst_.*')
183	extratar=0
184	find_varstoexclude $set
185	if [ $nbvars -eq 0 ] ; then
186	varstoexclude="t_inst_.*"
187	else
188	# There are others variables than 't_inst_.*' so an extra tar file will be created
189	varstoexclude="t_inst_.*,$varstoexcludefiltered"
190	extratar=1
191	# Copy to output_tar directory
192	echo "====> COPY to tar command because find other variables ($varstoexcludefiltered) than t_inst_*"
193	echo "COPY to tar command because find other variables ($varstoexcludefiltered) than t_inst_*" >> $statusfile
194	cp $set $PATH_SIMU/output_tar
195	fi
196
197	#-----------------------------------
198	# Try 2 times before fire "FAILED"
199	ncrcatcmd=ncrcat
200	ncrcatoptions="--md5_digest"
201	# ncrcatoptions=""
202
203	startTime=$( getDateMilliSeconds ) # gpdebug : time
204	resCmd=0
205	cat $set \| $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir
206	resCmd=$?
207	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
208
209	# resultCmd=$( gives_0_or_1 )
210	# resultCmd=1
211	# if [ $resultCmd -eq 0 ] ; then
212	if [ $resCmd -eq 0 ] ; then
213	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
214	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
215	echo "DONE ; $datestr1 ; $datestr2 ; cat $set \| $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
216	echo "====> COMPLETED at try #$nbtries (at 1st shot on 2)"
217	echo "COMPLETED at try #$nbtries (at 1st shot on 2)" >> $statusfile
218	else
219	# Add --md5_digest
220	rm -f $outputfile*ncrcat.tmp
221	startTime=$( getDateMilliSeconds ) # gpdebug : time
222	resCmd=0
223	cat $set \| $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir
224	resCmd=$?
225	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
226	# resultCmd=$( gives_0_or_1 )
227	# resultCmd=1
228	# if [ $resultCmd -eq 0 ] ; then
229	if [ $resCmd -eq 0 ] ; then
230	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
231	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
232	echo "DONE ; $datestr1 ; $datestr2 ; cat $set \| $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
233	echo "====> COMPLETED at try #$nbtries (at 2nd shot on 2)"
234	echo "COMPLETED at try #$nbtries (at 2nd shot on 2)" >> $statusfile
235	else
236	rm -f $outputfile*ncrcat.tmp
237	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
238	echo "====> FAILED at try #$nbtries"
239	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; cat $set \| $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
240	if [ $nbtries -ge $maxtries ] ; then
241	echo "====> DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)"
242	echo "DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)" >> $statusfile
243	cp $set $PATH_SIMU/output_tar # gpdebug : a retablir
244	fi
245	fi
246	fi
247
248	done
249
250	#------------------------------------------------------
251
252	for set in $( ls $PATH_SIMU/output_tar/*list ) ; do
253
254	#-----------------------------------
255	echo "#-------------------"
256	echo "Set: $set"
257	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
258	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
259	statusfile=${set%%.list}.status
260	logfile=${set%%.list}_${datestr}.log
261
262	#-----------------------------------
263	# Create status file if not exists
264	if [ ! -e $statusfile ] ; then
265	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
266	fi
267
268	#-----------------------------------
269	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
270	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
271	printf "====> "
272	tail -1 $statusfile
273	continue
274	fi
275
276	#-----------------------------------
277	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
278	# nbtries=`wc -l $statusfile \| cut -d' ' -f1`
279	# echo "Nb of tries: $nbtries"
280
281	#-----------------------------------
282	# Set output file name from list name
283
284	fileout1=$( basename $set )
285	fileout=${fileout1%.list}.tar
286
287	filefirst=`head -n 1 $set`
288	dirinputfile=`dirname $filefirst`
289	diroutputfile=`dirname $filefirst \| sed -e "s%$dirin%$dirout%"`
290
291	outputfile=$diroutputfile/${fileout}
292	mkdir -p $diroutputfile
293	echo "Output file to write: $outputfile"
294
295	#-----------------------------------
296	# Try 1 time before fire "FAILED"
297	# Waiting for CCRT command
298	tarcmd=tar
299	taroptions="--format=posix -W -cf"
300	#taroptions="--format=posix -cf"
301	cd $dirinputfile
302
303	set_local=$set".local"
304	for file in $( cat $set) ; do
305	basename $file >> $set".local"
306	done
307
308	# #-----------------------------------
309	startTime=$( getDateMilliSeconds ) # gpdebug : time
310	resCmd=0
311	$tarcmd $taroptions $outputfile --dereference --files-from $set.local > $logfile 2>&1 # gpdebug : a retablir
312	resCmd=$?
313	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
314
315	# resultCmd=$( gives_0_or_1 )
316	# if [ $resultCmd -eq 0 ] ; then
317	if [ $resCmd -eq 0 ] ; then
318	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
319	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
320	echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
321	echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
322	echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
323	else
324	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
325	echo "====> FAILED at try #$nbtries"
326	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
327
328	fi
329	rm -f $set".local"
330	#-----------------------------------
331
332	done
333
334
335	#------------------------------------------------------
336
337	for set in $( ls $PATH_SIMU/restart_tar/*list ) ; do
338	#-----------------------------------
339	echo "#-------------------"
340	echo "Set: $set"
341	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
342	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
343	statusfile=${set%%.list}.status
344	logfile=${set%%.list}_${datestr}.log
345
346	#-----------------------------------
347	# Create status file if not exists
348	if [ ! -e $statusfile ] ; then
349	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
350	fi
351
352	#-----------------------------------
353	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
354	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
355	printf "====> "
356	tail -1 $statusfile
357	continue
358	fi
359
360	#-----------------------------------
361	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
362	# nbtries=`wc -l $statusfile \| cut -d' ' -f1`
363	# echo "Nb of tries: $nbtries"
364
365	#-----------------------------------
366	# Set output file name from list name
367	dirin1=$PATH_SIMU/RESTART
368	dirin2=${dirin1##${IGCM_DEM}}
369	dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )}
370	diroutputfile=${OUTPUT_STORE}${dirin3}
371	fileout1=$( basename $set )
372	fileout=${fileout1%.list}.tar
373	outputfile=$diroutputfile/${fileout}
374	mkdir -p $diroutputfile
375	echo "Output file to write: $outputfile"
376
377	#-----------------------------------
378	# Try 1 time before fire "FAILED"
379	# Waiting for CCRT command
380	tarcmd=tar
381	taroptions="--format=posix -W -cf"
382	# taroptions="--format=posix -cf"
383	cd $dirin1
384	# ls
385
386	startTime=$( getDateMilliSeconds ) # gpdebug : time
387	resCmd=0
388	$tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir
389	resCmd=$?
390	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
391
392	# resultCmd=$( gives_0_or_1 )
393	# if [ $resultCmd -eq 0 ] ; then
394	if [ $resCmd -eq 0 ] ; then
395	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
396	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
397	echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
398	echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
399	echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
400	else
401	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
402	echo "====> FAILED at try #$nbtries"
403	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
404
405	fi
406
407	#-----------------------------------
408
409	done
410
411	#------------------------------------------------------
412	for set in $( ls $PATH_SIMU/debug_tar/*list ) ; do
413
414	#-----------------------------------
415	echo "#-------------------"
416	echo "Set: $set"
417	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
418	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
419	statusfile=${set%%.list}.status
420	logfile=${set%%.list}_${datestr}.log
421
422	#-----------------------------------
423	# Create status file if not exists
424	if [ ! -e $statusfile ] ; then
425	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
426	fi
427
428	#-----------------------------------
429	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
430	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
431	printf "====> "
432	tail -1 $statusfile
433	continue
434	fi
435
436	#-----------------------------------
437	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
438	# nbtries=`wc -l $statusfile \| cut -d' ' -f1`
439	# echo "Nb of tries: $nbtries"
440
441	#-----------------------------------
442	# Set output file name from list name
443	dirin1=$PATH_SIMU/DEBUG
444	dirin2=${dirin1##${IGCM_DEM}}
445	dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )}
446	diroutputfile=${OUTPUT_STORE}${dirin3}
447	fileout1=$( basename $set )
448	fileout=${fileout1%.list}.tar
449	outputfile=$diroutputfile/${fileout}
450	mkdir -p $diroutputfile
451	echo "Output file to write: $outputfile"
452
453	#-----------------------------------
454	# Try 1 time before fire "FAILED"
455	# Waiting for CCRT command
456	tarcmd=tar
457	taroptions="--format=posix -W -cf"
458	# taroptions="--format=posix -cf"
459	cd $dirin1
460	# ls
461
462	startTime=$( getDateMilliSeconds ) # gpdebug : time
463	resCmd=0
464	$tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir
465	resCmd=$?
466	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
467
468	# resultCmd=$( gives_0_or_1 )
469	# if [ $resultCmd -eq 0 ] ; then
470	if [ $resCmd -eq 0 ] ; then
471	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
472	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
473	echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
474	echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
475	echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
476	else
477	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
478	echo "====> FAILED at try #$nbtries"
479	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
480
481	fi
482
483	#-----------------------------------
484
485	done
486
487	#------------------------------------------------------
488	for set in $( ls $PATH_SIMU/store_cp/*list ) ; do
489
490	#-----------------------------------
491	echo "#-------------------"
492	echo "Set: $set"
493	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
494	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
495	statusfile=${set%%.list}.status
496	logfile=${set%%.list}_${datestr}.log
497
498	#-----------------------------------
499	# Create status file if not exists
500	if [ ! -e $statusfile ] ; then
501	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
502	fi
503
504	#-----------------------------------
505	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
506	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
507	printf "====> "
508	tail -1 $statusfile
509	continue
510	fi
511
512	#-----------------------------------
513	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
514	# nbtries=`wc -l $statusfile \| cut -d' ' -f1`
515	# echo "Nb of tries: $nbtries"
516
517	#-----------------------------------
518	# Set output file name from first and last files from the current set
519	for file in $( cat $set) ; do
520
521	diroutputfile=`dirname $file \| sed -e "s%$dirin%$dirout%"`
522	mkdir -p $diroutputfile
523	echo "Output file to write: $diroutputfile/$( basename $file )"
524
525	#-----------------------------------
526	# Try 1 time before fire "FAILED"
527	# Waiting for CCRT command
528	cpcmd="cp "
529	cpoptions="-rf"
530	startTime=$( getDateMilliSeconds ) # gpdebug : time
531	resCmd=0
532	$cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1 # gpdebug : a retablir
533	resCmd=$?
534	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
535	done
536
537	# resultCmd=$( gives_0_or_1 )
538	# if [ $resultCmd -eq 0 ] ; then
539	if [ $resCmd -eq 0 ] ; then
540	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
541	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
542	echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
543	echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
544	echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
545	else
546	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
547	echo "====> FAILED at try #$nbtries"
548	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
549
550	fi
551
552	#-----------------------------------
553
554	done
555
556	#------------------------------------------------------
557	for set in $( ls $PATH_SIMU/work_cp/*list ) ; do
558
559	#-----------------------------------
560	echo "#-------------------"
561	echo "Set: $set"
562	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
563	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
564	statusfile=${set%%.list}.status
565	logfile=${set%%.list}_${datestr}.log
566
567	#-----------------------------------
568	# Create status file if not exists
569	if [ ! -e $statusfile ] ; then
570	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
571	fi
572
573	#-----------------------------------
574	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
575	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
576	printf "====> "
577	tail -1 $statusfile
578	continue
579	fi
580
581	#-----------------------------------
582	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
583	# nbtries=`wc -l $statusfile \| cut -d' ' -f1`
584	# echo "Nb of tries: $nbtries"
585
586	#-----------------------------------
587	# Set output file name from first and last files from the current set
588	for file in $( cat $set) ; do
589
590	diroutputfile=`dirname $file \| sed -e "s%$dirin%$dirout_work%"`
591	mkdir -p $diroutputfile
592	echo "Output file to write: $diroutputfile/$( basename $file )"
593
594	#-----------------------------------
595	# Try 1 time before fire "FAILED"
596	# Waiting for CCRT command
597	cpcmd="cp "
598	cpoptions="-rf"
599	startTime=$( getDateMilliSeconds ) # gpdebug : time
600	resCmd=0
601	$cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1 # gpdebug : a retablir
602	resCmd=$?
603	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : timeS
604	done
605
606	# resultCmd=$( gives_0_or_1 )
607	# if [ $resultCmd -eq 0 ] ; then
608	if [ $resCmd -eq 0 ] ; then
609	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
610	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
611	echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
612	echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
613	echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
614	else
615	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
616	echo "====> FAILED at try #$nbtries"
617	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
618
619	fi
620
621	#-----------------------------------
622
623	done
624	fi
625	# if [ "$baseNameOfListFile" == "tar_full_simul.list" ]
626	for set in $( ls ${PATH_SIMU}/*.list \| grep "tar_full_simul" )
627	do
628	# set="${PATH_SIMU}/tar_full_simul.list" # gpdebug : foireux ?
629	#-----------------------------------
630	echo "#-------------------"
631	echo "Set: $set"
632	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
633	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
634	statusfile=${set%%.list}.status
635	logfile=${set%%.list}_${datestr}.log
636
637	#-----------------------------------
638	# Create status file if not exists
639	if [ ! -e $statusfile ] ; then
640	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
641	fi
642
643	#-----------------------------------
644	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
645	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
646	printf "====> "
647	tail -1 $statusfile
648	exit 0
649	fi
650
651	#-----------------------------------
652	# Set output file name from list name
653	# echo "PATH_SIMU=$PATH_SIMU"
654	dirOfOutputFile=`echo $PATH_SIMU \| sed "s;${IGCM_DEM};${OUTPUT_STORE};" `
655	fileout=${dirnameOfListFile}.tar
656	# echo "dirnameOfListFile=$dirnameOfListFile"
657	outputfile=$dirOfOutputFile/${fileout}
658	mkdir -p $dirOfOutputFile
659	echo "Output file to write: $outputfile tar_full_simul"
660
661	# exit 0 # a virer
662
663	#-----------------------------------
664	# Try 1 time before fire "FAILED"
665	# Waiting for CCRT command
666	tarcmd=tar
667	taroptions="--format=posix -W -cf"
668	# taroptions="--format=posix -cf"
669
670	filefirst=`head -n 1 $set`
671	dirinputfile=`dirname $filefirst`
672	cd $dirinputfile
673
674	set_local=${set%%.list}.local
675	> $set_local
676	for file in $( cat $set) ; do
677	basename $file >> $set_local
678	done
679	# ls
680
681	startTime=$( getDateMilliSeconds ) # gpdebug : time
682	resCmd=0
683	$tarcmd $taroptions $outputfile --dereference --files-from $set_local > $logfile 2>&1 # gpdebug : a retablir
684	resCmd=$?
685	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
686
687	# resultCmd=$( gives_0_or_1 )
688	# if [ $resultCmd -eq 0 ] ; then
689	if [ $resCmd -eq 0 ] ; then
690	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
691	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
692	echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
693	echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
694	echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
695	else
696	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
697	echo "====> FAILED at try #$nbtries"
698	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
699	exit 1
700	fi
701
702	#-----------------------------------
703	done

Note: See TracBrowser for help on using the repository browser.

Download in other formats: