Index: codes/icosagcm/trunk/arch/arch-SCOREP_IRENE.env
===================================================================
--- codes/icosagcm/trunk/arch/arch-SCOREP_IRENE.env	(revision 962)
+++ codes/icosagcm/trunk/arch/arch-SCOREP_IRENE.env	(revision 962)
@@ -0,0 +1,16 @@
+# module c++/gnu is needed to link with XIOS
+# module feature/openmpi/mpi_compiler/intel is made necessary by c++/gnu
+
+module purge
+module load feature/openmpi/mpi_compiler/intel
+module load c++/gnu
+module load mpi/openmpi
+module load flavor/hdf5/parallel
+module load netcdf-fortran
+module load mkl
+module load hdf5/1.8.20
+module swap intel/19.0.2.187
+module load scorep
+module load vampir
+export SCOREP_ENABLE_TRACING=true
+export SCOREP_ENABLE_PROFILING=false
Index: codes/icosagcm/trunk/arch/arch-SCOREP_IRENE.fcm
===================================================================
--- codes/icosagcm/trunk/arch/arch-SCOREP_IRENE.fcm	(revision 962)
+++ codes/icosagcm/trunk/arch/arch-SCOREP_IRENE.fcm	(revision 962)
@@ -0,0 +1,16 @@
+%COMPILER            scorep --nocompiler --nomemory --mpp=mpi --thread=omp --user mpif90 -DSCOREP
+%LINK                scorep --nocompiler --nomemory --mpp=mpi --thread=omp --user mpif90 -DSCOREP
+%FPP                 mpif90 `scorep-config --nocompiler --nomemory --mpp=mpi --thread=omp --user --fflags` -E
+%AR                  ar
+%MAKE                gmake
+%FPP_FLAGS           -P
+%FPP_DEF             KEY_NONE CPP_NETCDF4 CPP_SCOREP
+%BASE_FFLAGS         -i4 -r8 -auto -align all -I${MKLROOT}/include
+%PROD_FFLAGS         -g -traceback -O3 -xHost
+%DEV_FFLAGS          -g -O1 -traceback
+%DEBUG_FFLAGS        -g -traceback -check bounds -fp-model strict
+%MPI_FFLAGS
+%OMP_FFLAGS          -qopenmp
+%BASE_LD             -i4 -r8 -auto $MKL_LDFLAGS
+%MPI_LD
+%OMP_LD              -qopenmp
Index: codes/icosagcm/trunk/arch/arch-SCOREP_IRENE.path
===================================================================
--- codes/icosagcm/trunk/arch/arch-SCOREP_IRENE.path	(revision 962)
+++ codes/icosagcm/trunk/arch/arch-SCOREP_IRENE.path	(revision 962)
@@ -0,0 +1,32 @@
+NETCDF_INCDIR="$(nc-config --cflags) $(nc-config --fflags)"
+NETCDF_LIBDIR=$(nc-config --flibs)
+NETCDF_LIB=""
+
+HDF5_INCDIR=""
+HDF5_LIBDIR=""
+HDF5_LIB=""
+
+MPI_INCDIR=""
+MPI_LIBDIR=""
+MPI_LIB=""
+
+BOOST_INCDIR=""
+BOOST_LIBDIR=""
+BOOST_LIB=""
+
+BLITZ_INCDIR=""
+BLITZ_LIBDIR=""
+BLITZ_LIB=""
+
+OASIS_INCDIR=""
+OASIS_LIBDIR=""
+OASIS_LIB=""
+
+IOIPSL_INCDIR="-I$ROOT/../IOIPSL/inc"
+IOIPSL_LIBDIR="-L$ROOT/../IOIPSL/lib"
+IOIPSL_LIB="-lioipsl"
+
+XIOS_ROOTDIR="$PWD/../XIOS"
+XIOS_INCDIR="-I$XIOS_ROOTDIR/inc"
+XIOS_LIBDIR="-L$XIOS_ROOTDIR/lib"
+XIOS_LIB="-lxios -lstdc++"
Index: codes/icosagcm/trunk/src/base/profiling.F90
===================================================================
--- codes/icosagcm/trunk/src/base/profiling.F90	(revision 962)
+++ codes/icosagcm/trunk/src/base/profiling.F90	(revision 962)
@@ -0,0 +1,178 @@
+! Conditional compilation : use scorep user task api
+#ifdef CPP_SCOREP 
+
+#include "scorep/SCOREP_User.inc"
+#define SCOREP_DEFINE_HANDLE_ARRAY( handle ) SCOREP_USER_REGION_HANDLE, allocatable :: handle(:)
+#define SCOREP_ALLOCATE_HANDLE_ARRAY( handle, size ) allocate( scorep_handle(size) )
+
+#else
+
+#define SCOREP_DEFINE_HANDLE_ARRAY(handle)
+#define SCOREP_ALLOCATE_HANDLE_ARRAY(handle,size)
+#define SCOREP_USER_REGION_INIT(handle,name,region_type)
+#define SCOREP_USER_REGION_ENTER(handle)
+#define SCOREP_USER_REGION_END(handle)
+
+#endif
+
+MODULE profiling_mod
+  use omp_para, only : omp_size, omp_rank
+  IMPLICIT NONE
+  SAVE
+  PRIVATE
+  
+  ! Shared variables 
+  LOGICAL, PARAMETER :: check_accuracy = .true.
+  INTEGER, PARAMETER :: max_id=20, max_depth=10
+  INTEGER :: nb_id ! Number of timers
+  CHARACTER(15), DIMENSION(max_id) :: name ! Timer names
+  INTEGER :: id_profile
+  
+  ! Threadlocal variables
+  INTEGER, ALLOCATABLE :: depth(:)!(omp_size) ! Number of current nested timers
+  INTEGER, ALLOCATABLE :: current_id(:,:)!(omp_size, max_depth) ! Current nested timer ids
+  REAL, ALLOCATABLE :: chrono(:,:)!(omp_size, max_depth) ! Last timer start time
+  REAL, ALLOCATABLE :: elapsed(:,:)!(omp_size, max_id) ! Cumulative time for each timer
+  SCOREP_DEFINE_HANDLE_ARRAY(scorep_handle)
+  
+  PUBLIC :: init_profiling, reset_profiling, register_id, enter_profile, exit_profile, print_profile
+
+CONTAINS
+
+  SUBROUTINE init_profiling
+    use omp_para, only : get_omp_size
+    integer :: omp_size
+    ! omp_size from module omp_para cannot be used here since it is not initialized yet
+    ! TODO : conditional compilation to compile without openmp
+    omp_size = get_omp_size()
+    !$omp master
+    nb_id=0
+    allocate(depth(0:omp_size-1))
+    allocate(current_id(0:omp_size-1, max_depth))
+    allocate(chrono(0:omp_size-1, max_depth))
+    allocate(elapsed(0:omp_size-1, max_id))
+    depth(:)=0   
+    call reset_profiling()
+    !$omp end master
+    SCOREP_ALLOCATE_HANDLE_ARRAY(scorep_handle, max_id)
+    !$omp barrier
+    call register_id("print_profile", id_profile)
+  END SUBROUTINE init_profiling
+  
+  SUBROUTINE reset_profiling
+    use abort_mod
+    !$omp barrier
+    !$omp master
+    if( any( depth /= 0 ) ) call dynamico_abort("Impossible to reset profiling : a profiling region is still open")   
+    elapsed(:,:)=0   
+    !$omp end master
+    !$omp barrier 
+  END SUBROUTINE
+
+  SUBROUTINE register_id(thename, id)
+    use abort_mod
+    CHARACTER(*), INTENT(IN) :: thename
+    INTEGER, INTENT(OUT) :: id
+    !$OMP MASTER
+    nb_id = nb_id+1
+    if(nb_id > max_id) call dynamico_abort("Too many timers")
+    id = nb_id
+    name(id)=thename
+    !$OMP END MASTER
+    SCOREP_USER_REGION_INIT( scorep_handle(id), thename, SCOREP_USER_REGION_TYPE_COMMON )
+    !$omp barrier
+  END SUBROUTINE register_id
+
+  FUNCTION get_elapsed(start)
+    INTEGER(kind=8) :: count, count_rate
+    REAL :: start,get_elapsed
+    CALL SYSTEM_CLOCK(count,count_rate)
+    if(check_accuracy .and. count < 10) print *, "Warning : Profiling elapsed time too short to be accurately measured"
+    get_elapsed = (1.*count)/(1.*count_rate) - start
+    IF(get_elapsed<0.) get_elapsed=0.
+  END FUNCTION get_elapsed
+
+  SUBROUTINE enter_profile(id)
+    INTEGER, INTENT(IN) :: id
+    depth(omp_rank) = depth(omp_rank)+1
+    chrono(omp_rank, depth(omp_rank)) = get_elapsed(0.)
+    current_id(omp_rank,depth(omp_rank)) = id
+    SCOREP_USER_REGION_ENTER(scorep_handle(id))
+  END SUBROUTINE enter_profile
+   
+  SUBROUTINE exit_profile(id)
+    use abort_mod
+    INTEGER, INTENT(IN) :: id
+    INTEGER :: parent_id
+    REAL :: my_elapsed
+    
+    IF(depth(omp_rank)<=0) call dynamico_abort("exit_profile called without a matching enter_profile (depth=0)")
+    IF(id /= current_id(omp_rank, depth(omp_rank))) call dynamico_abort("wrong timer id : exit_profile id doesn't match enter_profile")
+    
+    SCOREP_USER_REGION_END(scorep_handle(id))
+    
+    my_elapsed = get_elapsed(chrono(omp_rank, depth(omp_rank)))
+    ! add elapsed to current profile
+    elapsed(omp_rank,id) = elapsed(omp_rank,id) + my_elapsed
+    depth(omp_rank) = depth(omp_rank)-1
+    ! and substract from parent profile
+    IF(depth(omp_rank)>0) THEN
+       parent_id = current_id(omp_rank,depth(omp_rank))
+       elapsed(omp_rank,parent_id) = elapsed(omp_rank,parent_id) - my_elapsed
+    END IF
+  END SUBROUTINE exit_profile
+
+  SUBROUTINE print_profile
+    use mpi_mod
+    use mpipara
+    use omp_para
+    INTEGER :: i
+    REAL :: mean_total, min_total, max_total, mean_total_local, min_total_local, max_total_local 
+    REAL :: omp_mean_time_local, omp_min_time_local, omp_max_time_local
+    REAL :: mean_time, max_process, min_process, percent
+
+    call enter_profile(id_profile)
+
+    !$OMP MASTER
+    ! mean_total : mean of sum of all timers on all threads and all procs
+    mean_total_local = SUM(elapsed(:,1:nb_id))/omp_size/mpi_size
+    call MPI_Reduce(mean_total_local, mean_total, 1, MPI_DOUBLE, MPI_SUM, 0, comm_icosa, ierr)
+    ! min_total : min on all procs of sum of all timers on all threads
+    min_total_local = MINVAL( SUM(elapsed(:,1:nb_id), 2) )
+    call MPI_Reduce(min_total_local, min_total, 1, MPI_DOUBLE, MPI_MIN, 0, comm_icosa, ierr)
+    ! max_total : max on all procs of sum of all timers on all threads
+    max_total_local = MAXVAL( SUM(elapsed(:,1:nb_id), 2) )
+    call MPI_Reduce(max_total_local, max_total, 1, MPI_DOUBLE, MPI_MAX, 0, comm_icosa, ierr)
+
+    if(is_master) PRINT *, '---------------------- Profiling -----------------------'
+    if(is_master) PRINT ('(A15, F12.3, A, F12.3, A, F12.3, A)'), 'Total (s) : ', mean_total, "  [", min_total, ",", max_total, "]"
+    if(is_master) PRINT ('(A15, A7, A47, A47)'), " ","  %  ","  Process mean(s)[ min, max, diff/mean(%) ]  ", "Process 1 threads(s)[ min, max, diff/mean(%) ]"
+    DO i=1,nb_id
+       omp_mean_time_local = SUM(elapsed(:,i))/omp_size ! Mean time of timer on local threads 
+       omp_min_time_local = MINVAL(elapsed(:,i)) ! Max time of timer on local threads
+       omp_max_time_local = MAXVAL(elapsed(:,i)) ! Min time of timer on local threads
+       !omp_inbalance_local = 100*(max_time-min_time)/min_time
+       
+       ! mean_time : mean of timer on all threads and all procs
+       call MPI_Reduce(omp_mean_time_local, mean_time, 1, MPI_DOUBLE, MPI_SUM, 0, comm_icosa, ierr); mean_time = mean_time/mpi_size
+       ! max_process_time : max on procs of mean omp time 
+       call MPI_Reduce(omp_mean_time_local, max_process, 1, MPI_DOUBLE, MPI_MAX, 0, comm_icosa, ierr)
+       ! min_process_time : min on procs of mean omp time 
+       call MPI_Reduce(omp_mean_time_local, min_process, 1, MPI_DOUBLE, MPI_MIN, 0, comm_icosa, ierr)
+       
+       percent = 100*mean_time/mean_total
+
+       if(is_master) PRINT ('(A15, F5.1, F12.3, A, F12.3, A, F12.3, A, F6.1, A, F12.3, A, F12.3, A, F12.3, A, F6.1, A)'), name(i), percent, mean_time, " [", min_process, ",", max_process, "," , inbalance(mean_time, min_process, max_process),  "]", omp_mean_time_local, " [", omp_min_time_local, ",", omp_max_time_local, "," , inbalance(omp_mean_time_local, omp_min_time_local, omp_max_time_local) , "]"
+    END DO
+    if(is_master) PRINT *, '---------------------- Profiling -----------------------'
+    !$OMP END MASTER
+    call exit_profile(id_profile)
+  contains
+    function inbalance( mean, min, max )
+      real :: mean, min, max
+      real :: inbalance
+      inbalance = 100*(max-min)/(mean+1E-10)
+    end function
+  END SUBROUTINE print_profile
+    
+END MODULE profiling_mod
Index: codes/icosagcm/trunk/src/base/profiling.f90
===================================================================
--- codes/icosagcm/trunk/src/base/profiling.f90	(revision 961)
+++ 	(revision )
@@ -1,89 +1,0 @@
-MODULE profiling_mod
-  IMPLICIT NONE
-  SAVE
-  PRIVATE
-
-  INTEGER, PARAMETER :: max_id=20, max_depth=10
-  INTEGER :: nb_id=-10, depth=-10, current_id(max_depth)
-  CHARACTER(10), DIMENSION(max_id) :: name
-  REAL :: chrono(max_depth), elapsed(max_id)
-
-  PUBLIC :: init_profiling, register_id, enter_profile, exit_profile, print_profile
-
-CONTAINS
-
-  SUBROUTINE init_profiling
-    !$OMP MASTER
-    nb_id=0
-    depth=0
-    elapsed(:)=0
-    !$OMP END MASTER
-  END SUBROUTINE init_profiling
-
-  SUBROUTINE register_id(thename, id)
-    CHARACTER(*), INTENT(IN) :: thename
-    INTEGER, INTENT(OUT) :: id
-    !$OMP MASTER
-    nb_id = nb_id+1
-    id = nb_id
-    name(id)=thename
-    !$OMP END MASTER
-  END SUBROUTINE register_id
-
-  FUNCTION get_elapsed(start)
-    INTEGER(kind=8) :: count, count_rate
-    REAL :: start,get_elapsed
-    CALL SYSTEM_CLOCK(count,count_rate)
-    get_elapsed = (1.*count)/(1.*count_rate) - start
-    IF(get_elapsed<0.) get_elapsed=0.
-  END FUNCTION get_elapsed
-
-  SUBROUTINE enter_profile(id)
-    INTEGER, INTENT(IN) :: id
-    !$OMP MASTER
-    depth = depth+1
-    chrono(depth) = get_elapsed(0.)
-    current_id(depth) = id
-    !$OMP END MASTER
-  END SUBROUTINE enter_profile
-   
-  SUBROUTINE exit_profile(id)
-    INTEGER, INTENT(IN) :: id
-    INTEGER :: parent_id
-    REAL :: my_elapsed
-    !$OMP MASTER
-    IF(depth<=0) THEN
-       PRINT *, 'exit_profile called at depth=0 !!'
-       STOP
-    END IF
-    IF(id /= current_id(depth)) THEN
-       PRINT *,' exit_profile : at depth ', depth, ' exiting ', TRIM(name(id)), ' after entering ', TRIM(name(current_id(depth)))
-       STOP
-    END IF
-    my_elapsed = get_elapsed(chrono(depth))
-    ! add elapsed to current profile
-    elapsed(id) = elapsed(id) + my_elapsed
-    depth = depth-1
-    ! and substract from parent profile
-    IF(depth>0) THEN
-       parent_id = current_id(depth)
-       elapsed(parent_id) = elapsed(parent_id) - my_elapsed
-    END IF
-    !$OMP END MASTER
-  END SUBROUTINE exit_profile
-
-  SUBROUTINE print_profile
-    INTEGER :: i
-    REAL :: total
-    !$OMP MASTER
-    PRINT *, '---------------------- Profiling -----------------------'
-    total = SUM(elapsed(1:nb_id))
-    PRINT *, 'Total (s) : ', total
-    DO i=1,nb_id
-       PRINT *, name(i), INT(elapsed(i)), INT(1000.*elapsed(i)/total)
-    END DO
-    PRINT *, '---------------------- Profiling -----------------------'
-    !$OMP END MASTER
-  END SUBROUTINE print_profile
-    
-END MODULE profiling_mod
Index: codes/icosagcm/trunk/src/parallel/omp_para.F90
===================================================================
--- codes/icosagcm/trunk/src/parallel/omp_para.F90	(revision 961)
+++ codes/icosagcm/trunk/src/parallel/omp_para.F90	(revision 962)
@@ -59,4 +59,15 @@
 
 CONTAINS
+
+  FUNCTION get_omp_size() result(omp_size)
+#ifdef CPP_USING_OMP
+    use omp_lib, only : omp_get_max_threads
+    integer :: omp_size
+    omp_size = omp_get_max_threads()
+#else
+    integer :: omp_size
+    omp_size = 1
+#endif
+  END FUNCTION
 
   SUBROUTINE init_omp_para(is_mpi_master)
Index: codes/icosagcm/trunk/src/parallel/transfert_mpi.f90
===================================================================
--- codes/icosagcm/trunk/src/parallel/transfert_mpi.f90	(revision 961)
+++ codes/icosagcm/trunk/src/parallel/transfert_mpi.f90	(revision 962)
@@ -88,4 +88,6 @@
   END INTERFACE
 
+  integer :: profile_mpi_copies, profile_mpi_waitall, profile_mpi_omp_barrier
+
 CONTAINS
        
@@ -103,4 +105,7 @@
 
     CALL register_id('MPI', id_mpi)
+    CALL register_id('MPI_copies', profile_mpi_copies)
+    CALL register_id('MPI_waitall', profile_mpi_waitall)
+    CALL register_id('MPI_omp_barrier', profile_mpi_omp_barrier)
 
     CALL create_request(field_t,req_i1)
@@ -1068,25 +1073,8 @@
 
 
-    IF (field(1)%data_type==type_real) THEN
-
-      IF (field(1)%ndim==2) THEN
-     
-        DO ireq=1,message%nreq
-          CALL free_mpi_buffer(message%buffers(ireq)%r)
-        ENDDO
-      
-      ELSE  IF (field(1)%ndim==3) THEN
-
-        DO ireq=1,message%nreq
-          CALL free_mpi_buffer(message%buffers(ireq)%r)
-        ENDDO
-      
-      ELSE  IF (field(1)%ndim==4) THEN
-
-        DO ireq=1,message%nreq
-          CALL free_mpi_buffer(message%buffers(ireq)%r)
-        ENDDO
-
-      ENDIF
+    IF (message%field(1)%data_type==type_real) THEN
+      DO ireq=1,message%nreq
+        CALL free_mpi_buffer(message%buffers(ireq)%r)
+      ENDDO    
     ENDIF
 
@@ -1220,5 +1208,7 @@
     if( field(1)%ondevice .AND. .NOT. message%ondevice ) call update_device_message_mpi(message)
 
+CALL enter_profile(profile_mpi_omp_barrier)
 !$OMP BARRIER
+CALL exit_profile(profile_mpi_omp_barrier)
 
 
@@ -1241,5 +1231,7 @@
     ENDIF
 !$OMP END MASTER
+CALL enter_profile(profile_mpi_omp_barrier)
 !$OMP BARRIER
+CALL exit_profile(profile_mpi_omp_barrier)
      
     IF (field(1)%data_type==type_real) THEN
@@ -1263,8 +1255,10 @@
               offset=send%offset
               msize=send%size
+              call enter_profile(profile_mpi_copies)
               !$acc parallel loop default(present) async if (field(ind)%ondevice)
               DO n=1,msize
                 buffer_r(offset+n)=rval2d(value(n))
               ENDDO
+              call exit_profile(profile_mpi_copies)
 
               IF (mpi_threading_mode==MPI_THREAD_SERIALIZED) THEN
@@ -1299,9 +1293,10 @@
               sgn=>recv%sign
               msize=recv%size
+              call enter_profile(profile_mpi_copies)
               !$acc parallel loop default(present) async if (field(ind)%ondevice)
               DO n=1,msize
                 rval2d(value(n))=src_rval2d(src_value(n))*sgn(n)
               ENDDO
-                
+              call exit_profile(profile_mpi_copies)
                     
             ELSE
@@ -1353,5 +1348,5 @@
               msize=send%size
               moffset=send%offset
-              CALL trace_start("copy_to_buffer")
+              call enter_profile(profile_mpi_copies)
 
               !$acc parallel loop default(present) async if (field(ind)%ondevice)
@@ -1363,8 +1358,11 @@
                 ENDDO
               ENDDO
-              CALL trace_end("copy_to_buffer")
+              call exit_profile(profile_mpi_copies)
 
               IF (mpi_threading_mode==MPI_THREAD_SERIALIZED.OR.mpi_threading_mode==MPI_THREAD_MULTIPLE .AND. omp_level_size>1) THEN
-                !$OMP BARRIER
+CALL enter_profile(profile_mpi_omp_barrier)
+!$OMP BARRIER
+CALL exit_profile(profile_mpi_omp_barrier)
+
               ENDIF
               
@@ -1384,5 +1382,8 @@
             ELSE
               IF (mpi_threading_mode==MPI_THREAD_SERIALIZED.OR.mpi_threading_mode==MPI_THREAD_MULTIPLE .AND. omp_level_size>1) THEN
-                !$OMP BARRIER
+CALL enter_profile(profile_mpi_omp_barrier)
+!$OMP BARRIER
+CALL exit_profile(profile_mpi_omp_barrier)
+
               ENDIF
             ENDIF
@@ -1391,5 +1392,8 @@
           IF (mpi_threading_mode==MPI_THREAD_SERIALIZED.OR.mpi_threading_mode==MPI_THREAD_MULTIPLE .AND. omp_level_size>1) THEN
             DO isend=req%nsend+1,max_req
-              !$OMP BARRIER
+CALL enter_profile(profile_mpi_omp_barrier)
+!$OMP BARRIER
+CALL exit_profile(profile_mpi_omp_barrier)
+
             ENDDO
           ENDIF
@@ -1414,4 +1418,5 @@
               msize=recv%size
 
+              call enter_profile(profile_mpi_copies)
               CALL trace_start("copy_data")
               !$acc parallel loop collapse(2) default(present) async if (field(ind)%ondevice)
@@ -1421,4 +1426,5 @@
                 ENDDO
               ENDDO
+              call exit_profile(profile_mpi_copies)
               CALL trace_end("copy_data")
 
@@ -1473,4 +1479,5 @@
               moffset=send%offset
 
+              call enter_profile(profile_mpi_copies)
               CALL trace_start("copy_to_buffer")
               !$acc parallel loop default(present) collapse(2) async if (field(ind)%ondevice)
@@ -1486,7 +1493,11 @@
               ENDDO
               CALL trace_end("copy_to_buffer")
+              call exit_profile(profile_mpi_copies)
 
               IF (mpi_threading_mode==MPI_THREAD_SERIALIZED.OR.mpi_threading_mode==MPI_THREAD_MULTIPLE .AND. omp_level_size>1) THEN
-                !$OMP BARRIER
+CALL enter_profile(profile_mpi_omp_barrier)
+!$OMP BARRIER
+CALL exit_profile(profile_mpi_omp_barrier)
+
               ENDIF
 
@@ -1506,5 +1517,8 @@
             ELSE
               IF (mpi_threading_mode==MPI_THREAD_SERIALIZED.OR.mpi_threading_mode==MPI_THREAD_MULTIPLE .AND. omp_level_size>1) THEN
-                !$OMP BARRIER
+CALL enter_profile(profile_mpi_omp_barrier)
+!$OMP BARRIER
+CALL exit_profile(profile_mpi_omp_barrier)
+
               ENDIF
             ENDIF
@@ -1513,5 +1527,8 @@
           IF (mpi_threading_mode==MPI_THREAD_SERIALIZED.OR.mpi_threading_mode==MPI_THREAD_MULTIPLE .AND. omp_level_size>1) THEN
             DO isend=req%nsend+1,max_req
-              !$OMP BARRIER
+CALL enter_profile(profile_mpi_omp_barrier)
+!$OMP BARRIER
+CALL exit_profile(profile_mpi_omp_barrier)
+
             ENDDO
           ENDIF
@@ -1527,5 +1544,5 @@
           rval4d=>field(ind)%rval4d
           req=>message%request(ind)
-
+          
           DO irecv=1,req%nrecv
             recv=>req%recv(irecv)
@@ -1536,6 +1553,6 @@
               sgn=>recv%sign
               msize=recv%size
+              call enter_profile(profile_mpi_copies)
               CALL trace_start("copy_data")
-
               !$acc parallel loop collapse(3) default(present) async if (field(ind)%ondevice)
               DO d4=1,dim4
@@ -1546,5 +1563,5 @@
                 ENDDO
               ENDDO
-                
+              call exit_profile(profile_mpi_copies)
               CALL trace_end("copy_data")
                    
@@ -1573,6 +1590,8 @@
 
       IF (mpi_threading_mode==MPI_THREAD_FUNNELED .OR. mpi_threading_mode==MPI_THREAD_SINGLE) THEN
+CALL enter_profile(profile_mpi_omp_barrier)
 !$acc wait
 !$OMP BARRIER
+CALL exit_profile(profile_mpi_omp_barrier)
 !$OMP MASTER        
 
@@ -1612,6 +1631,8 @@
       ENDIF              
     ENDIF
-    
+CALL enter_profile(profile_mpi_omp_barrier)
 !$OMP BARRIER
+CALL exit_profile(profile_mpi_omp_barrier)
+ 
 !    CALL trace_end("send_message_mpi")
 
@@ -1672,10 +1693,12 @@
       IF (field(1)%ndim==2) THEN
 
-!$OMP MASTER
-        IF (.NOT. message%completed) CALL MPI_WAITALL(nreq,message%mpi_req,          &
+call enter_profile(profile_mpi_waitall)
+!$OMP MASTER         
+         IF (.NOT. message%completed) CALL MPI_WAITALL(nreq,message%mpi_req,          &
           message%status,ierr)
 !$OMP END MASTER
 !$OMP BARRIER
-        
+call exit_profile(profile_mpi_waitall)
+        call enter_profile(profile_mpi_copies)
         DO ind=1,ndomain
           IF (.NOT. assigned_domain(ind) .OR. .NOT. is_omp_level_master) CYCLE
@@ -1701,13 +1724,15 @@
         
         ENDDO
+        call exit_profile(profile_mpi_copies)
       
       
       ELSE  IF (field(1)%ndim==3) THEN
-
+         call enter_profile(profile_mpi_waitall)
 !$OMP MASTER
-        IF (.NOT. message%completed) CALL MPI_WAITALL(nreq,message%mpi_req,          &
-          message%status,ierr)
+         IF (.NOT. message%completed) CALL MPI_WAITALL(nreq,message%mpi_req,          &
+              message%status,ierr)
 !$OMP END MASTER
 !$OMP BARRIER
+        call exit_profile(profile_mpi_waitall)
 
         
@@ -1730,4 +1755,5 @@
               msize=recv%size
               moffset=recv%offset
+              call enter_profile(profile_mpi_copies)
               CALL trace_start("copy_from_buffer")
               
@@ -1753,4 +1779,5 @@
                 
               CALL trace_end("copy_from_buffer")
+              call exit_profile(profile_mpi_copies)
             ENDIF
           ENDDO
@@ -1759,4 +1786,5 @@
 
       ELSE  IF (field(1)%ndim==4) THEN
+call enter_profile(profile_mpi_waitall)
 !$OMP MASTER
         IF (.NOT. message%completed) CALL MPI_WAITALL(nreq,message%mpi_req,          &
@@ -1764,4 +1792,5 @@
 !$OMP END MASTER
 !$OMP BARRIER
+        call exit_profile(profile_mpi_waitall)
 
                 
@@ -1784,4 +1813,5 @@
               msize=recv%size
               moffset=recv%offset
+              call enter_profile(profile_mpi_copies)
               CALL trace_start("copy_from_buffer")
               !$acc parallel loop default(present) collapse(2) async if (field(ind)%ondevice)
@@ -1797,4 +1827,5 @@
               ENDDO
               CALL trace_end("copy_from_buffer")
+              call exit_profile(profile_mpi_copies)
             ENDIF
           ENDDO
Index: codes/icosagcm/trunk/src/time/timeloop_gcm.F90
===================================================================
--- codes/icosagcm/trunk/src/time/timeloop_gcm.F90	(revision 961)
+++ codes/icosagcm/trunk/src/time/timeloop_gcm.F90	(revision 962)
@@ -45,5 +45,5 @@
     IF (.NOT. enable_io) itau_out=HUGE(itau_out)
 
-    itau_prof=100
+    itau_prof=1000
     CALL getin('itau_profiling',itau_prof)
 
@@ -257,8 +257,9 @@
     CALL update_device_field(f_wflux)
     CALL update_device_field(f_rhodz)
+    call reset_profiling()  
 
 
     DO it=itau0+1,itau0+itaumax
-       IF (is_master) CALL print_iteration(it, itau0, itaumax, start_clock, rate_clock)
+       CALL print_iteration(it, itau0, itaumax, start_clock, rate_clock)
 
        CALL enter_profile(id_timeloop)
@@ -432,16 +433,19 @@
     INTEGER(kind=8) :: start_clock, stop_clock, rate_clock
     REAL :: per_step,total, elapsed
-    WRITE(*,'(A,I7,A,F14.1)') "It No :",it,"   t :",dt*it
-    IF(MOD(it,10)==0) THEN
-       CALL SYSTEM_CLOCK(stop_clock)
-       elapsed = (stop_clock-start_clock)*1./rate_clock
-       per_step = elapsed/(it-itau0)
-       throughput = INT(dt/per_step)
-       total = per_step*itaumax
-       WRITE(*,'(A,I5,A,F6.2,A,I6)') 'Time spent (s):',INT(elapsed), &
-            '  -- ms/step : ', 1000*per_step, &
-            '  -- Throughput :', throughput
-       WRITE(*,'(A,I5,A,I5)') 'Whole job (min) :', INT(total/60.), &
-            '  -- Completion in (min) : ', INT((total-elapsed)/60.)
+    
+    IF(is_master) THEN
+       WRITE(*,'(A,I7,A,F14.1)') "It No :",it,"   t :",dt*it
+       IF(MOD(it,10)==0) THEN
+          CALL SYSTEM_CLOCK(stop_clock)
+          elapsed = (stop_clock-start_clock)*1./rate_clock
+          per_step = elapsed/(it-itau0)
+          throughput = INT(dt/per_step)
+          total = per_step*itaumax
+          WRITE(*,'(A,I5,A,F6.2,A,I6)') 'Time spent (s):',INT(elapsed), &
+               '  -- ms/step : ', 1000*per_step, &
+               '  -- Throughput :', throughput
+          WRITE(*,'(A,I5,A,I5)') 'Whole job (min) :', INT(total/60.), &
+               '  -- Completion in (min) : ', INT((total-elapsed)/60.)
+       END IF
     END IF
     IF(MOD(it,itau_prof)==0) CALL print_profile
