Index: RELEASE
===================================================================
--- RELEASE	(.../tags/FLASH4.4_release)	(revision 25574)
+++ RELEASE	(.../branches/flash4_lukat)	(revision 25574)
@@ -1 +1 @@
-4.4 ${SVNBRIEF}
+4.4.flash4_lukat (r25574)
Index: source/physics/Gravity/GravityMain/Poisson/GPU_BH_Tree/Config
===================================================================
--- source/physics/Gravity/GravityMain/Poisson/GPU_BH_Tree/Config	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/physics/Gravity/GravityMain/Poisson/GPU_BH_Tree/Config	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,16 @@
+REQUIRES Grid/GridSolvers/BHTree/GPU
+
+# for compatibility with other Poisson solvers
+# available options are: isolated, periodic and mixed
+PARAMETER   grav_boundary_type      STRING      "mixed" ["isolated","periodic","mixed"]
+
+# if grav_boundary_type == "mixed", boundaries can be set
+# independently for each direction; available options are:
+# isolated and periodic
+D   grav_boundary_type_x Gravity boundary type for the X direction, used if grav_boundary_type == "mixed"
+D   grav_boundary_type_y Gravity boundary type for the Y direction, used if grav_boundary_type == "mixed"
+D   grav_boundary_type_z Gravity boundary type for the Z direction, used if grav_boundary_type == "mixed"
+PARAMETER   grav_boundary_type_x    STRING      "isolated" ["isolated","periodic"]
+PARAMETER   grav_boundary_type_y    STRING      "isolated" ["isolated","periodic"]
+PARAMETER   grav_boundary_type_z    STRING      "isolated" ["isolated","periodic"]
+
Index: source/physics/Gravity/GravityMain/Poisson/GPU_BH_Tree/Gravity_init.F90
===================================================================
--- source/physics/Gravity/GravityMain/Poisson/GPU_BH_Tree/Gravity_init.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/physics/Gravity/GravityMain/Poisson/GPU_BH_Tree/Gravity_init.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,120 @@
+!!****if* source/physics/Gravity/GravityMain/Poisson/GPU_BH_Tree/Gravity_init
+!!
+!! NAME
+!!
+!!  Gravity_init
+!!
+!! 
+!! SYNOPSIS
+!!
+!!  call Gravity_init()
+!!
+!!
+!! DESCRIPTION
+!!
+!!  Initialize the Barnes-Hut tree Poisson solver.  Read in any of the
+!!  runtime parameters for this solver.  All solver common data
+!!  is stored in the tree_common module
+!!
+!!***
+
+subroutine Gravity_init()
+
+  use Gravity_data, ONLY : grv_bhNewton, grav_boundary, grav_geometry,&
+       grav_poisfact, grv_meshMe, grv_meshNumProcs, grv_meshComm,&
+       updateGravity, useGravity, grav_unjunkPden
+  use Driver_interface, ONLY : Driver_abortFlash, Driver_getMype,&
+       Driver_getComm, Driver_getNumProcs
+  use RuntimeParameters_interface, ONLY : RuntimeParameters_get, &
+    RuntimeParameters_mapStrToInt, RuntimeParameters_set
+  use PhysicalConstants_interface, ONLY : PhysicalConstants_get
+
+  implicit none
+#include "constants.h"
+
+  character(len=MAX_STRING_LENGTH) :: strGeometry
+  character(len=MAX_STRING_LENGTH) :: grav_boundary_type, grav_boundary_type_x,&
+     grav_boundary_type_y, grav_boundary_type_z
+
+  call Driver_getMype(MESH_COMM,grv_meshMe)
+  call Driver_getComm(MESH_COMM,grv_meshComm)
+  call Driver_getNumProcs(MESH_COMM,grv_meshNumProcs)
+
+  call RuntimeParameters_get("geometry", strGeometry)
+  call RuntimeParameters_mapStrToInt(strGeometry, grav_geometry)
+
+  ! needed by Gravity_potentialListOfBlocks (CF 2013)
+  call RuntimeParameters_get("grav_unjunkPden", grav_unjunkPden)  
+
+  call RuntimeParameters_get("useGravity", useGravity)
+  call RuntimeParameters_get("updateGravity", updateGravity)
+
+  call RuntimeParameters_get("grav_boundary_type", grav_boundary_type)
+  call RuntimeParameters_get("grav_boundary_type_x", grav_boundary_type_x)
+  call RuntimeParameters_get("grav_boundary_type_y", grav_boundary_type_y)
+  call RuntimeParameters_get("grav_boundary_type_z", grav_boundary_type_z)
+  
+  call PhysicalConstants_get("Newton", grv_bhNewton)
+
+  ! print *, 'Gravity_init entering.'
+
+  !! FUTURE : need to add scale factor for cosmology
+  !grav_scaleFactor = 1
+  grav_poisfact = 4. * PI * grv_BhNewton
+
+  select case (grav_boundary_type)
+    case("isolated")
+      grav_boundary = ISOLATED
+      grav_boundary_type_x = "isolated"
+      grav_boundary_type_y = "isolated"
+      grav_boundary_type_z = "isolated"
+      call RuntimeParameters_set("grav_boundary_type_x", grav_boundary_type_x)
+      call RuntimeParameters_set("grav_boundary_type_y", grav_boundary_type_y)
+      call RuntimeParameters_set("grav_boundary_type_z", grav_boundary_type_z)
+    case("periodic")
+      grav_boundary = PERIODIC
+      grav_boundary_type_x = "periodic"
+      grav_boundary_type_y = "periodic"
+      grav_boundary_type_z = "periodic"
+      call RuntimeParameters_set("grav_boundary_type_x", grav_boundary_type_x)
+      call RuntimeParameters_set("grav_boundary_type_y", grav_boundary_type_y)
+      call RuntimeParameters_set("grav_boundary_type_z", grav_boundary_type_z)
+    case("mixed")
+      select case (grav_boundary_type_x)
+        case("isolated")
+          grav_boundary(1) = ISOLATED
+          grav_boundary(2) = ISOLATED
+        case("periodic")
+          grav_boundary(1) = PERIODIC
+          grav_boundary(2) = PERIODIC
+        case default
+          call Driver_abortFlash("Gravity_init: unrecognized or unsupported gravity boundary type in x direction")
+      end select
+      select case (grav_boundary_type_y)
+        case("isolated")
+          grav_boundary(3) = ISOLATED
+          grav_boundary(4) = ISOLATED
+        case("periodic")
+          grav_boundary(3) = PERIODIC
+          grav_boundary(4) = PERIODIC
+        case default
+          call Driver_abortFlash("Gravity_init: unrecognized or unsupported gravity boundary type in y direction")
+      end select
+      select case (grav_boundary_type_z)
+        case("isolated")
+          grav_boundary(5) = ISOLATED
+          grav_boundary(6) = ISOLATED
+        case("periodic")
+          grav_boundary(5) = PERIODIC
+          grav_boundary(6) = PERIODIC
+        case default
+          call Driver_abortFlash("Gravity_init: unrecognized or unsupported gravity boundary type in z direction")
+      end select
+    case default
+      call Driver_abortFlash("Gravity_init: unrecognized or unsupported gravity boundary type")
+  end select
+
+  ! print *, 'Gravity_init exting...', grav_boundary_type, grav_boundary_type_x, grav_boundary_type_y, grav_boundary_type_z
+
+  return
+end subroutine Gravity_init
Index: source/physics/Gravity/GravityMain/Poisson/GPU_BH_Tree/Gravity_data.F90
===================================================================
--- source/physics/Gravity/GravityMain/Poisson/GPU_BH_Tree/Gravity_data.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/physics/Gravity/GravityMain/Poisson/GPU_BH_Tree/Gravity_data.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,38 @@
+!!****if* source/physics/Gravity/GravityMain/Poisson/GPU_BH_Tree/Gravity_data
+!!
+!! NAME
+!!
+!!  Gravity_data
+!!  
+!! SYNOPSIS
+!!
+!!  use Gravity_data
+!!
+!! DESCRIPTION
+!!
+!!  This modules stores the data for the Gravity unit
+!!
+!!***
+
+module Gravity_data
+
+#include "constants.h"
+#include "Flash.h"
+
+  ! Common Gravity module variables
+  character(len=MAX_STRING_LENGTH), save :: grav_boundary_type !string boundary condition
+  integer, save :: grav_boundary(6)  !integer boundary condition
+
+  integer, save :: grav_geometry  !mesh geometry
+  integer, save :: grv_meshMe, grv_meshNumProcs, grv_meshComm
+  integer, save :: grv_commSize=1
+
+  logical, save :: useGravity, updateGravity
+  logical, save :: grav_temporal_extrp !extrapolate or otherwise rescale
+  logical, save :: grav_unjunkPden
+
+  real,    save :: grav_poisfact
+  real,    save :: grv_bhNewton
+  integer, save :: grv_defaultGpotVar = GPOT_VAR
+
+end module Gravity_data
Index: source/Simulation/Flash_mpi.h
===================================================================
--- source/Simulation/Flash_mpi.h	(.../tags/FLASH4.4_release)	(revision 25574)
+++ source/Simulation/Flash_mpi.h	(.../branches/flash4_lukat)	(revision 25574)
@@ -6,3 +6,4 @@
    integer, parameter :: FLASH_SUM = MPI_SUM
    integer, parameter :: FLASH_LOR = MPI_LOR
    integer, parameter :: FLASH_COMM = MPI_COMM_WORLD
+   integer, parameter :: FLASH_CHAR = MPI_CHAR
Index: source/Simulation/SimulationMain/MacLaurin_GPU/Simulation_initBlock.F90
===================================================================
--- source/Simulation/SimulationMain/MacLaurin_GPU/Simulation_initBlock.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Simulation/SimulationMain/MacLaurin_GPU/Simulation_initBlock.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,409 @@
+!!****if* source/Simulation/SimulationMain/unitTest/Multipole/Simulation_initBlock
+!!
+!! NAME
+!!
+!!  Simulation_initBlock
+!!
+!! SYNOPSIS
+!!
+!!  Simulation_initBlock (integer (in) :: blockID)
+!!
+!! DESCRIPTION
+!!
+!!  Initializes the Maclaurin spheroid fluid data (density only) for a specified block
+!!  and also sets the analytical gravitational solution by calling a separate routine.
+!!
+!!  References:  Maclaurin, C. 1742, 
+!!               Chandrasekhar, S. 1987, Ellipsoidal Figures of Equilibrium
+!!
+!! ARGUMENTS
+!!
+!!  blockID : the identification number of the block to be initialize
+!!
+!!***
+
+subroutine Simulation_initBlock (blockID)
+
+  use Simulation_data, ONLY : sim_xctr,               &
+                              sim_yctr,               &
+                              sim_zctr,               &
+                              sim_nsubinv,            &
+                              sim_nsubzones,          &
+                              sim_initGeometry,       &
+                              sim_a3inv,              &
+                              sim_a1inv,              &
+                              sim_density,            &
+                              sim_smallRho,           &
+                              GRID_3DCARTESIAN,       &
+                              GRID_3DCYLINDRICAL,     &
+                              GRID_2DCYLINDRICAL,     &
+                              GRID_1DSPHERICAL,       &
+                              GRID_2DSPHERICAL
+
+  use Grid_interface,  ONLY : Grid_getBlkIndexLimits, &
+                              Grid_getCellCoords,     &
+                              Grid_putRowData,        &
+                              Grid_getDeltas
+
+  implicit none
+
+#include "constants.h"
+#include "Flash.h"
+
+  integer, intent(in)  :: blockID
+
+  logical  :: gcell=.true.
+
+  integer  :: sizeX, sizeY, sizeZ
+  integer  :: i, j, k, ii, jj, kk
+  integer  :: ibeg,iend,jbeg,jend,kbeg,kend
+
+  real     :: dx, dy, dz
+  real     :: xdist, ydist, zdist, Rdist, rFunc, r2, z2
+  real     :: xx, yy, zz, dxx, dyy, dzz
+  real     :: theta
+  real     :: sum_rho
+
+  integer, dimension(3)      :: startingPos
+  integer, dimension(2,MDIM) :: blkLimits, blkLimitsGC
+  real,    dimension(MDIM)   :: deltas
+
+  real, dimension(:), allocatable :: xLeft, yLeft, zLeft
+  real, dimension(:), allocatable :: rho
+!
+!
+!    ...Loop over all cells in current block and initialize the cells with
+!       the needed density values. Work on a row-by-row basis. Break each cell
+!       into nsubzones^ndim sub-zones and average the results to get the values
+!       for the cell. This prevents a blocky spheroid shell.
+!
+!
+  call Grid_getBlkIndexLimits (blockID,    &
+                               blkLimits,  &
+                               blkLimitsGC )
+
+  select case (sim_initGeometry)
+!
+!
+!   ...The 3D cartesian case.
+!
+!         Full consideration of subzones in all 3 dimensions necessary,
+!         as square of radius (r2) and square of z coordinate (z2) for
+!         evaluating the MacLaurin density vary with all 3 dimensions.
+!
+!
+  case (GRID_3DCARTESIAN)
+
+    sizeX = blkLimitsGC (HIGH,IAXIS) - blkLimitsGC (LOW,IAXIS) + 1
+    sizeY = blkLimitsGC (HIGH,JAXIS) - blkLimitsGC (LOW,JAXIS) + 1
+    sizeZ = blkLimitsGC (HIGH,KAXIS) - blkLimitsGC (LOW,KAXIS) + 1
+
+    allocate (xLeft (sizeX))
+    allocate (yLeft (sizeY))
+    allocate (zLeft (sizeZ))
+
+    call Grid_getCellCoords (KAXIS, blockID, LEFT_EDGE, gcell, zLeft, sizeZ)
+    call Grid_getCellCoords (JAXIS, blockID, LEFT_EDGE, gcell, yLeft, sizeY)
+    call Grid_getCellCoords (IAXIS, blockID, LEFT_EDGE, gcell, xLeft, sizeX)
+
+    call Grid_getDeltas     (blockID, deltas)
+
+    dx = deltas(IAXIS)
+    dy = deltas(JAXIS)
+    dz = deltas(KAXIS)
+
+    kbeg = blkLimitsGC (LOW, KAXIS)
+    kend = blkLimitsGC (HIGH,KAXIS)
+    jbeg = blkLimitsGC (LOW, JAXIS)
+    jend = blkLimitsGC (HIGH,JAXIS)
+    ibeg = blkLimitsGC (LOW, IAXIS)
+    iend = blkLimitsGC (HIGH,IAXIS)
+
+    allocate (rho (sizeX))
+
+    do k = kbeg,kend
+       dzz = dz * sim_nsubinv
+       do j = jbeg,jend
+          dyy = dy * sim_nsubinv
+          do i = ibeg,iend
+             dxx = dx * sim_nsubinv
+
+             sum_rho = 0.0
+
+             do kk = 0, sim_nsubzones-1
+                zz    = zLeft (k) + (kk + 0.5) * dzz 
+                zdist = zz - sim_zctr
+                do jj = 0, sim_nsubzones-1
+                   yy    = yLeft (j) + (jj + 0.5) * dyy
+                   ydist = yy - sim_yctr
+                   do ii = 0, sim_nsubzones-1
+                      xx    = xLeft (i) + (ii + 0.5) * dxx
+                      xdist = xx - sim_xctr
+
+                      r2    = xdist**2 + ydist**2
+                      z2    = zdist**2
+
+                      rFunc = r2 * sim_a1inv * sim_a1inv  +  z2 * sim_a3inv * sim_a3inv
+
+                      if (rFunc <= 1.) then                        ! inside the spheroid
+                          sum_rho = sum_rho + sim_density
+                      end if
+
+                   enddo
+                enddo
+             enddo
+
+             rho (i) = max (sim_smallRho + sim_smallRho , sum_rho * sim_nsubinv**3)
+
+          enddo
+
+          startingPos (1) = 1
+          startingPos (2) = j
+          startingPos (3) = k
+
+          call Grid_putRowData (blockID, CENTER, DENS_VAR, EXTERIOR, IAXIS, startingPos, rho, sizeX)
+ 
+       enddo
+    enddo
+
+    deallocate (rho)
+    deallocate (xLeft)
+    deallocate (yLeft)
+    deallocate (zLeft)
+!
+!
+!   ...The 3D cylindrical case.
+!
+!         Subzones need to be considered only for the radial and z dimension.
+!         The angular dimension does not need to be split into subzones (square
+!         of radius r2 and square of z coordinate (z2) for evaluating the MacLaurin
+!         potential vary only with radial and z dimension component of 3D cylindrical
+!         coordinate system.
+!
+!
+  case (GRID_3DCYLINDRICAL)
+
+    sizeX = blkLimitsGC (HIGH,IAXIS) - blkLimitsGC (LOW,IAXIS) + 1
+    sizeY = blkLimitsGC (HIGH,JAXIS) - blkLimitsGC (LOW,JAXIS) + 1
+
+    allocate (xLeft (sizeX))
+    allocate (yLeft (sizeY))
+
+    call Grid_getCellCoords (JAXIS, blockID, LEFT_EDGE, gcell, yLeft, sizeY)
+    call Grid_getCellCoords (IAXIS, blockID, LEFT_EDGE, gcell, xLeft, sizeX)
+    call Grid_getDeltas     (blockID, deltas)
+
+    dx = deltas(IAXIS)
+    dy = deltas(JAXIS)
+
+    kbeg = blkLimitsGC (LOW, KAXIS)
+    kend = blkLimitsGC (HIGH,KAXIS)
+    jbeg = blkLimitsGC (LOW, JAXIS)
+    jend = blkLimitsGC (HIGH,JAXIS)
+    ibeg = blkLimitsGC (LOW, IAXIS)
+    iend = blkLimitsGC (HIGH,IAXIS)
+
+    allocate (rho (sizeX))
+
+    do j = jbeg,jend
+       dyy = dy * sim_nsubinv           ! z component
+       do i = ibeg,iend
+          dxx = dx * sim_nsubinv        ! radial component
+
+          sum_rho = 0.0
+
+          do jj = 0, sim_nsubzones-1
+             yy    = yLeft (j) + (jj + 0.5) * dyy
+             zdist = yy - sim_yctr
+             do ii = 0, sim_nsubzones-1
+                xx    = xLeft (i) + (ii + 0.5) * dxx
+                Rdist = xx - sim_xctr
+
+                r2 = Rdist ** 2
+                z2 = zdist ** 2
+
+                rFunc = r2 * sim_a1inv * sim_a1inv  +  z2 * sim_a3inv * sim_a3inv
+
+                if (rFunc <= 1.) then                        ! inside the spheroid
+                    sum_rho = sum_rho + sim_density
+                end if
+
+             enddo
+          enddo
+
+          rho (i) = max (sim_smallRho + sim_smallRho , sum_rho * sim_nsubinv**2)
+
+       enddo
+
+       do k = kbeg,kend           ! angular component -> no subzones -> identical potentials
+
+          startingPos (1) = 1
+          startingPos (2) = j
+          startingPos (3) = k
+
+          call Grid_putRowData (blockID, CENTER, DENS_VAR, EXTERIOR, IAXIS, startingPos, rho, sizeX)
+ 
+       enddo
+
+    enddo
+
+    deallocate (rho)
+    deallocate (xLeft)
+    deallocate (yLeft)
+!
+!
+!   ...The 2D cylindrical case.
+!
+!  
+  case (GRID_2DCYLINDRICAL)
+
+    sizeX = blkLimitsGC (HIGH,IAXIS) - blkLimitsGC (LOW,IAXIS) + 1
+    sizeY = blkLimitsGC (HIGH,JAXIS) - blkLimitsGC (LOW,JAXIS) + 1
+
+    allocate (xLeft (sizeX))
+    allocate (yLeft (sizeY))
+
+    call Grid_getCellCoords (JAXIS, blockID, LEFT_EDGE, gcell, yLeft, sizeY)
+    call Grid_getCellCoords (IAXIS, blockID, LEFT_EDGE, gcell, xLeft, sizeX)
+
+    call Grid_getDeltas     (blockID, deltas)
+
+    dx = deltas(IAXIS)
+    dy = deltas(JAXIS)
+
+    jbeg = blkLimitsGC (LOW, JAXIS)
+    jend = blkLimitsGC (HIGH,JAXIS)
+    ibeg = blkLimitsGC (LOW, IAXIS)
+    iend = blkLimitsGC (HIGH,IAXIS)
+
+    allocate (rho (sizeX))
+
+    do j = jbeg,jend
+       dyy = dy * sim_nsubinv
+       do i = ibeg,iend
+          dxx = dx * sim_nsubinv
+
+          sum_rho = 0.0
+
+          do jj = 0, sim_nsubzones-1
+             yy    = yLeft (j) + (jj + 0.5) * dyy
+             zdist = yy - sim_yctr
+             do ii = 0, sim_nsubzones-1
+                xx    = xLeft (i) + (ii + 0.5) * dxx
+                Rdist = xx - sim_xctr
+
+                r2    = Rdist**2
+                z2    = zdist**2
+
+                rFunc = r2 * sim_a1inv * sim_a1inv  +  z2 * sim_a3inv * sim_a3inv
+
+                if (rFunc <= 1.) then                        ! inside the spheroid
+                    sum_rho = sum_rho + sim_density
+                end if
+
+             enddo
+          enddo
+
+          rho (i) = max (sim_smallRho + sim_smallRho , sum_rho * sim_nsubinv**2)
+
+       enddo
+
+       startingPos(1) = 1
+       startingPos(2) = j
+
+       call Grid_putRowData (blockID, CENTER, DENS_VAR, EXTERIOR, IAXIS, startingPos, rho, sizeX)
+ 
+    enddo
+
+    deallocate (rho)
+    deallocate (xLeft)
+    deallocate (yLeft)
+!
+!
+!   ...The 2D spherical case.
+!
+!  
+  case (GRID_2DSPHERICAL)
+
+    sizeX = blkLimitsGC (HIGH,IAXIS) - blkLimitsGC (LOW,IAXIS) + 1
+    sizeY = blkLimitsGC (HIGH,JAXIS) - blkLimitsGC (LOW,JAXIS) + 1
+
+    allocate (xLeft (sizeX))
+    allocate (yLeft (sizeY))
+
+    call Grid_getCellCoords (JAXIS, blockID, LEFT_EDGE, gcell, yLeft, sizeY)
+    call Grid_getCellCoords (IAXIS, blockID, LEFT_EDGE, gcell, xLeft, sizeX)
+
+    call Grid_getDeltas     (blockID, deltas)
+
+    dx = deltas(IAXIS)
+    dy = deltas(JAXIS)
+
+    jbeg = blkLimitsGC (LOW, JAXIS)
+    jend = blkLimitsGC (HIGH,JAXIS)
+    ibeg = blkLimitsGC (LOW, IAXIS)
+    iend = blkLimitsGC (HIGH,IAXIS)
+
+    allocate (rho (sizeX))
+
+    do j = jbeg,jend
+       dyy = dy * sim_nsubinv
+       do i = ibeg,iend
+          dxx = dx * sim_nsubinv
+
+          sum_rho = 0.0
+
+          do jj = 0, sim_nsubzones-1
+             yy    = yLeft (j) + (jj + 0.5) * dyy
+             theta = yy - sim_yctr
+             do ii = 0, sim_nsubzones-1
+                xx    = xLeft (i) + (ii + 0.5) * dxx
+                Rdist = xx - sim_xctr                     ! Rdist is x^2 + y^2 + z^2 here!
+
+                zdist = Rdist * cos (theta)
+
+                z2    = zdist**2                          ! this is z^2
+                r2    = Rdist**2 - z2                     ! this is x^2 + y^2
+
+                rFunc = r2 * sim_a1inv * sim_a1inv  +  z2 * sim_a3inv * sim_a3inv
+
+                if (rFunc <= 1.) then                        ! inside the spheroid
+                    sum_rho = sum_rho + sim_density
+                end if
+
+             enddo
+          enddo
+
+          rho (i) = max (sim_smallRho + sim_smallRho , sum_rho * sim_nsubinv**2)
+
+       enddo
+
+       startingPos(1) = 1
+       startingPos(2) = j
+
+       call Grid_putRowData (blockID, CENTER, DENS_VAR, EXTERIOR, IAXIS, startingPos, rho, sizeX)
+ 
+    enddo
+
+    deallocate (rho)
+    deallocate (xLeft)
+    deallocate (yLeft)
+
+  case default
+
+    call Driver_abortFlash ('Multipole unit test: unsupported geometry (Simulation_initBlock)!')
+
+  end select
+!
+!
+!    ...Now calculate the analytical gravitational solution on this block.
+!
+!
+  call sim_initBlockAnalytical (blockID)
+!
+!
+!   ...Ready!
+!
+!
+  return
+end subroutine Simulation_initBlock
Index: source/Simulation/SimulationMain/MacLaurin_GPU/Gravity_potentialListOfBlocks.F90
===================================================================
--- source/Simulation/SimulationMain/MacLaurin_GPU/Gravity_potentialListOfBlocks.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Simulation/SimulationMain/MacLaurin_GPU/Gravity_potentialListOfBlocks.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,270 @@
+!!****if* source/physics/Gravity/GravityMain/Poisson/Gravity_potentialListOfBlocks
+!!
+!! NAME 
+!!
+!!     Gravity_potentialListOfBlocks
+!!
+!! SYNOPSIS
+!!
+!!  call Gravity_potentialListOfBlocks(integer(IN) :: blockCount,
+!!                                     integer(IN) :: blockList(blockCount),
+!!                            optional,integer(IN) :: potentialIndex)
+!!
+!! DESCRIPTION
+!!
+!!      This routine computes the gravitational potential on all
+!!      blocks specified in the list, for the gravity implementations
+!!      (i.e., various Poisson implementations), which make use of it
+!!      in computing the gravitational acceleration.
+!!
+!!      Supported boundary conditions are isolated (0) and
+!!      periodic (1).  The same boundary conditions are applied
+!!      in all directions.  For some implementation of Gravity,
+!!      in particular with Barnes-Hut tee solver, additional combinations
+!!      of boundary conditions may be supported.
+!!
+!! ARGUMENTS
+!!
+!!   blockCount   : The number of blocks in the list
+!!   blockList(:) : The list of blocks on which to calculate potential
+!!   potentialIndex : If present, determines which variable in UNK to use
+!!                    for storing the updated potential.  If not present,
+!!                    GPOT_VAR is assumed.
+!!                    Presence or absense of this optional dummy argument
+!!                    also determines whether some side effects are enabled
+!!                    or disabled; see discussion of two modes under NOTES
+!!                    below.
+!!
+!! NOTES
+!!
+!!  Gravity_potentialListOfBlocks can operate in one of two modes:
+!!  * automatic mode  - when called without the optional potentialIndex.
+!!    Such a call will usually be made once per time step, usually
+!!    from the main time advancement loop in Driver_evolveFlash.
+!!    Various side effects are enabled in this mode, see SIDE EFFECT below.
+!!
+!!  * explicit mode  - when called with the optional potentialIndex.
+!!    The potential is stored in the variable explicitly given, and
+!!    side effects like saving the previous potential in GPOL_VAR
+!!    and updating some sink particle state and properties are
+!!    suppressed.
+!!
+!! SIDE EFFECTS
+!!
+!!  Updates certain variables in permanent UNK storage to contain the
+!!  gravitational potential.  Invokes a solver (of the Poisson equation)
+!!  if necessary. On return, if potentialIndex is not present,
+!!     GPOT_VAR:  contains potential for the current simulation time.
+!!     GPOL_VAR (if defined): contains potential at the previous simulation time.
+!!  On return, if potentialIndex is present, the UNK variable given by
+!!  potentialIndex contains the newly computed potential.
+!!
+!!  May affect other variables related to particle properties if particles
+!!  are included in the simulation.  In particular,
+!!     PDEN_VAR (if defined): may get updated to the current density from
+!!                particles if particles have mass.
+!!
+!!  There are additional side effects if sink particles are used.
+!!  These effects happen by the call to Particles_sinkAccelGasOnSinksAndSinksOnGas,
+!!  which may update sink particle properties and additional UNK variables that store
+!!  accelerations. The calls are only made in automatic mode.
+!!
+!!  May modify certain variables used for intermediate results by the solvers
+!!  invoked. The list of variables depends on the Gravity implementation.
+!!  The following information is subject to change without notice.
+!!  For the Multigrid implementation:
+!!     ISLS_VAR (residual)
+!!     ICOR_VAR (correction)
+!!     IMGM_VAR (image mass)
+!!     IMGP_VAR (image potential)
+!!  For the Multipole implementation:
+!!     (none)
+!!
+!!***
+
+!!REORDER(4): solnVec
+
+subroutine Gravity_potentialListOfBlocks(blockCount,blockList, potentialIndex)
+
+
+  use Gravity_data, ONLY : grav_poisfact, grav_temporal_extrp, grav_boundary, &
+       grav_unjunkPden, &
+       useGravity, updateGravity, grv_meshComm,grv_meshMe
+  use Cosmology_interface, ONLY : Cosmology_getRedshift, &
+       Cosmology_getOldRedshift
+  use Driver_interface, ONLY : Driver_abortFlash
+  use Timers_interface, ONLY : Timers_start, Timers_stop
+  use Particles_interface, ONLY: Particles_updateGridVar, &
+       Particles_sinkAccelGasOnSinksAndSinksOnGas
+  use Grid_interface, ONLY : GRID_PDE_BND_PERIODIC, GRID_PDE_BND_NEUMANN, &
+       GRID_PDE_BND_ISOLATED, GRID_PDE_BND_DIRICHLET, &
+       Grid_getBlkPtr, Grid_releaseBlkPtr, &
+       Grid_notifySolnDataUpdate, &
+       Grid_solvePoisson
+  implicit none
+
+#include "Flash.h"
+#include "constants.h"
+#include "Flash_mpi.h"
+
+  integer,intent(IN) :: blockCount
+  integer,dimension(blockCount),intent(IN) :: blockList
+  integer, intent(IN), optional :: potentialIndex
+
+  logical ::perfect
+
+  real, POINTER, DIMENSION(:,:,:,:) :: solnVec
+
+  integer       :: ierr
+
+  real          :: redshift, oldRedshift
+  real          :: scaleFactor, oldScaleFactor
+  real          :: invscale, rescale
+  integer       :: lb
+  integer       :: bcTypes(6)
+  real          :: bcValues(2,6) = 0.
+  integer       :: density
+  integer       :: newPotVar
+  logical       :: saveLastPot
+
+  saveLastPot = (.NOT. present(potentialIndex))
+  if (present(potentialIndex)) then
+     newPotVar = potentialIndex
+  else
+     newPotVar = GPOT_VAR
+  end if
+
+  call Cosmology_getRedshift(redshift)
+  call Cosmology_getOldRedshift(oldRedshift)
+  
+  scaleFactor = 1./(1.+redshift)
+  oldScaleFactor = 1./(1.+oldRedshift)
+  
+  invscale = 1./scaleFactor**3
+
+! Rescaling factor to try and keep initial guess at potential close to
+! final solution (in cosmological simulations).  Source term in Poisson
+! equation has 1/a(t)^3 in it; and in linear theory (Omega=1, matter dom.)
+! the comoving density increases as a(t), so comoving peculiar potential
+! (which is what we are calculating here) should vary as 1/a(t)^2.  For
+! noncosmological simulations this has no effect, since oldscale = scale = 1.
+
+  rescale = (oldScaleFactor/scaleFactor)**2
+
+!=========================================================================
+
+  if(.not.useGravity) return
+  
+  if(.not.updateGravity) return
+
+  call Timers_start("gravity Barrier")
+  call MPI_Barrier (grv_meshComm, ierr)
+  call Timers_stop("gravity Barrier")
+
+  call Timers_start("gravity")
+
+  bcTypes = grav_boundary
+  where (bcTypes == PERIODIC)
+     bcTypes = GRID_PDE_BND_PERIODIC
+  elsewhere (bcTypes == ISOLATED)
+     bcTypes = GRID_PDE_BND_ISOLATED
+  elsewhere (bcTypes == DIRICHLET)
+     bcTypes = GRID_PDE_BND_DIRICHLET
+  elsewhere (bcTypes == OUTFLOW)
+     bcTypes = GRID_PDE_BND_NEUMANN
+  end where
+  bcValues = 0.
+     
+  if (grav_temporal_extrp) then
+     
+     call Driver_abortFlash("shouldn't be here right now")
+     !call extrp_initial_guess( igpot, igpol, igpot )
+     
+  else 
+     
+     do lb = 1, blockCount
+        call Grid_getBlkPtr(blocklist(lb), solnVec)
+#ifdef GPOL_VAR
+        if (saveLastPot) solnVec(GPOL_VAR,:,:,:) = solnVec(GPOT_VAR,:,:,:)
+#endif
+        solnVec(newPotVar,:,:,:) = solnVec(newPotVar,:,:,:) * rescale
+
+        ! CTSS - We should also be storing the old sink particle accelerations:
+#if defined(SGXO_VAR) && defined(SGYO_VAR) && defined(SGZO_VAR)
+        if (saveLastPot) then   !... but only if we are saving the old potential - kW
+           solnVec(SGXO_VAR,:,:,:) = solnVec(SGAX_VAR,:,:,:)
+           solnVec(SGYO_VAR,:,:,:) = solnVec(SGAY_VAR,:,:,:)
+           solnVec(SGZO_VAR,:,:,:) = solnVec(SGAZ_VAR,:,:,:)
+        end if
+#endif
+
+        ! for direct acceleration calculation by tree solver, added by R. Wunsch
+#if defined(GAOX_VAR) && defined(GAOY_VAR) && defined(GAOZ_VAR)
+        if (saveLastPot) then 
+           solnVec(GAOX_VAR,:,:,:) = solnVec(GACX_VAR,:,:,:)
+           solnVec(GAOY_VAR,:,:,:) = solnVec(GACY_VAR,:,:,:)
+           solnVec(GAOZ_VAR,:,:,:) = solnVec(GACZ_VAR,:,:,:)
+        end if
+#endif
+        call Grid_releaseBlkPtr(blocklist(lb), solnVec)
+     enddo
+     
+#ifdef GPOL_VAR
+     if (saveLastPot) call Grid_notifySolnDataUpdate( (/GPOL_VAR/) )
+#endif
+
+  endif
+
+! Poisson is solved with the total density of PDEN_VAR + DENS_VAR 
+  density=DENS_VAR
+! This only gets called if there are active particles.
+#ifdef PDEN_VAR
+#ifdef MASS_PART_PROP
+  call Particles_updateGridVar(MASS_PART_PROP, PDEN_VAR)
+  if (.NOT. grav_unjunkPden) call Grid_notifySolnDataUpdate( (/PDEN_VAR/) )
+  density = PDEN_VAR
+#ifdef DENS_VAR           
+  do lb = 1, blockCount
+     call Grid_getBlkPtr(blocklist(lb), solnVec)
+     solnVec(density,:,:,:) = solnVec(density,:,:,:) + &
+          solnVec(DENS_VAR,:,:,:)
+     call Grid_releaseBlkPtr(blocklist(lb), solnVec)
+  enddo
+#endif
+#endif
+#endif
+
+  invscale=grav_poisfact*invscale
+  call Grid_solvePoisson (newPotVar, density, bcTypes, bcValues, &
+       invscale)
+  call Grid_notifySolnDataUpdate( (/newPotVar/) )
+
+! Un-junk PDEN if it exists and if requested.
+
+#ifdef PDEN_VAR
+  if (grav_unjunkPden) then
+     density = PDEN_VAR
+#ifdef DENS_VAR           
+     do lb = 1, blockCount
+        call Grid_getBlkPtr(blocklist(lb), solnVec)
+        solnVec(density,:,:,:) = solnVec(density,:,:,:) - solnVec(DENS_VAR,:,:,:)
+        call Grid_releaseBlkPtr(blocklist(lb), solnVec)
+     enddo
+     if (density .NE. PDEN_VAR) call Grid_notifySolnDataUpdate( (/density/) )
+#endif
+  end if
+#endif
+
+  if (.NOT. present(potentialIndex)) then
+    ! Compute acceleration of the sink particles caused by gas and vice versa
+    call Particles_sinkAccelGasOnSinksAndSinksOnGas()
+  end if
+
+
+#ifdef USEBARS
+  call MPI_Barrier (grv_meshComm, ierr)
+#endif  
+  call Timers_stop ("gravity")
+   call sim_analyzePotential (perfect)
+  return
+end subroutine Gravity_potentialListOfBlocks
Index: source/Simulation/SimulationMain/MacLaurin_GPU/Simulation_init.F90
===================================================================
--- source/Simulation/SimulationMain/MacLaurin_GPU/Simulation_init.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Simulation/SimulationMain/MacLaurin_GPU/Simulation_init.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,108 @@
+!!****if* source/Simulation/SimulationMain/unitTest/Multipole/Simulation_init
+!!
+!! NAME
+!!
+!!  Simulation_init
+!!
+!! SYNOPSIS
+!!
+!!  Simulation_init ()
+!!
+!! DESCRIPTION
+!!
+!!  Prepares initialization for the multipole solver unit test, based on the MacLaurin
+!!  spheroid problem.
+!!
+!!  References:  Maclaurin, C. 1742,
+!!               Chandrasekhar, S. 1987, Ellipsoidal Figures of Equilibrium
+!!
+!! ARGUMENTS
+!!
+!!***
+
+subroutine Simulation_init ()
+
+  use Simulation_data
+  use Driver_data,                 ONLY : dr_globalMe
+  use Grid_interface,              ONLY : Grid_getGeometry
+  use Driver_interface,            ONLY : Driver_abortFlash
+  use RuntimeParameters_interface, ONLY : RuntimeParameters_get, &
+                                          RuntimeParameters_set
+  use PhysicalConstants_interface, ONLY : PhysicalConstants_get
+  use Logfile_interface,           ONLY : Logfile_stamp
+
+  implicit none
+
+#include "constants.h"
+#include "Flash.h"
+
+  integer :: geometry
+
+  call PhysicalConstants_get ("pi",                        sim_pi)
+  call PhysicalConstants_get ("Newton",                    sim_Newton)
+
+  call RuntimeParameters_get ("eccentricity",              sim_e)
+  call RuntimeParameters_get ("equatorialSemimajorAxis",   sim_a1)
+  call RuntimeParameters_get ("density",                   sim_density)
+  call RuntimeParameters_get ("smlrho",                    sim_smallRho)
+  call RuntimeParameters_get ("xctr",                      sim_xctr)
+  call RuntimeParameters_get ("yctr",                      sim_yctr)
+  call RuntimeParameters_get ("zctr",                      sim_zctr)
+  call RuntimeParameters_get ("nsubzones",                 sim_nsubzones)
+  call RuntimeParameters_get ("passTolerance",             sim_passTolerance)
+! 
+!
+!   ...Compute derived quantities for the MacLaurin spheroid.
+!
+!
+  sim_nsubinv = 1./sim_nsubzones
+  sim_a3      = sim_a1 * sqrt(1.0-sim_e**2)
+  sim_a1inv   = 1.0/sim_a1
+  sim_a3inv   = 1.0/sim_a3
+! 
+!
+!   ...Set the geometry.
+!
+!
+  call Grid_getGeometry (geometry)
+
+  if ((NDIM == 3) .and. (geometry == CARTESIAN)) then
+      sim_initGeometry = GRID_3DCARTESIAN
+  else if ((NDIM == 3) .and. (geometry == CYLINDRICAL)) then
+      sim_initGeometry = GRID_3DCYLINDRICAL
+  else if ((NDIM == 2) .and. (geometry == CYLINDRICAL)) then
+      sim_initGeometry = GRID_2DCYLINDRICAL
+  else if ((NDIM == 2) .and. (geometry == SPHERICAL)) then
+      sim_initGeometry = GRID_2DSPHERICAL
+  else if ((NDIM == 1) .and. (geometry == SPHERICAL)) then
+      sim_initGeometry = GRID_1DSPHERICAL
+  else
+      call Driver_abortFlash ('MacLaurin unit test: unsupported geometry!')
+  endif
+! 
+!
+!   ...Write a message to stdout describing the MacLaurin spheroid.
+!
+!
+  if (dr_globalMe .eq. MASTER_PE) then
+
+     call Logfile_stamp ("Initializing for multipole unit test (MacLaurin problem)","[Simulation_init]")
+
+     write (*,*)
+     write (*,*) "MacLaurin spheroid data."
+     write (*,*) "------------------------"
+     write (*,*)
+     write (*,*) "Density      = ", sim_density
+     write (*,*) "eccentricity = ", sim_e
+     write (*,*) "a1           = ", sim_a1
+     write (*,*) "a3           = ", sim_a3
+     write (*,*)
+
+  endif
+!
+!
+!   ...Ready!
+!
+!
+  return
+end subroutine Simulation_init
Index: source/Simulation/SimulationMain/MacLaurin_GPU/Config
===================================================================
--- source/Simulation/SimulationMain/MacLaurin_GPU/Config	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Simulation/SimulationMain/MacLaurin_GPU/Config	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,42 @@
+# Configuration file for the Maclaurin spheroid problem
+# [source/Simulation/SimulationMain/MacLaurin_GPU]
+# recommended MAXBLOCKS 1000 or greater
+
+REQUIRES physics/Hydro
+REQUIRES physics/Gravity/GravityMain/Poisson
+REQUIRES physics/Eos/EosMain/Gamma
+REQUIRES physics/Gravity/GravityMain/Poisson/GPU_BH_Tree
+
+USESETUPVARS nDim
+IF nDim <> 3
+  SETUPERROR MacLaurin must run in 3d.  Use ./setup MacLaurin_GPU -auto -3d blah blah
+ENDIF
+
+
+# Runtime parameters
+VARIABLE dens TYPE: PER_VOLUME   # mass density
+VARIABLE gpot                    # calculated flash potential
+VARIABLE apot                    # analytic potential
+VARIABLE errm		         # analytical minus calculated potential 
+VARIABLE errd	                 # analytical divided by calculated potential
+VARIABLE errn	                 # analytical minus calculated potential, divided by analytical potential
+
+D  passTolerance               Allowed error for testing.  0.015 = 1.5 percent error
+D  eccentricity                Eccentricity of the ellipsoid (e)
+D  equatorialSemimajorAxis     Equatorial semimajor axis (a1)
+D  density                     Spheroid density (rho): set to -1 to generate spheroid mass of 1.0
+D  xctr                        X-coordinate of center of spheroid
+D  yctr                        Y-coordinate of center of spheroid
+D  zctr                        Z-coordinate of center of spheroid
+D  nsubzones                   Number of sub-zones per dimension
+
+PARAMETER passTolerance               REAL     0.015  [0.00000000000001 to 1.0]
+PARAMETER eccentricity                REAL     0.   [0.0 to 1.0]
+PARAMETER equatorialSemimajorAxis     REAL     0.25    [0.1 to ]
+PARAMETER density                     REAL     1.   [-1.0 to ]
+PARAMETER xctr                        REAL     0.5
+PARAMETER yctr                        REAL     0.5
+PARAMETER zctr                        REAL     0.5
+PARAMETER nsubzones                   INTEGER  2    [1 to ]
+
+
Index: source/Simulation/SimulationMain/MacLaurin_GPU/sim_initBlockAnalytical.F90
===================================================================
--- source/Simulation/SimulationMain/MacLaurin_GPU/sim_initBlockAnalytical.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Simulation/SimulationMain/MacLaurin_GPU/sim_initBlockAnalytical.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,509 @@
+!!****if* source/Simulation/SimulationMain/unitTest/Multipole/sim_initBlockAnalytical
+!!
+!! NAME
+!!
+!!  sim_initBlockAnalytical
+!!
+!! SYNOPSIS
+!!
+!!  sim_initBlockAnalytical (integer (in) :: blockID)
+!!
+!! DESCRIPTION
+!!
+!!  Provides analytical gravity potential solution per block for the Maclaurin spheroid
+!!  problem. Analytical solution given in: Ricker, P. 2006 "A Direct Multigrid Poisson
+!!  Solver for Oct-tree adaptive meshes".
+!!
+!!  References:  Maclaurin, C. 1742, 
+!!               Chandrasekhar, S. 1987, Ellipsoidal Figures of Equilibrium
+!!
+!! ARGUMENTS
+!!
+!!  blockID : current grid block identification number
+!!
+!! NOTES
+!!
+!!  The following geometries can be handled:
+!!
+!!                     3D cartesian
+!!                     3D cylindrical
+!!                     2D cylindrical
+!!                     2D spherical
+!!
+!!***
+
+subroutine sim_initBlockAnalytical (blockID)
+
+  use Simulation_data, ONLY:  sim_Newton,             &
+                              sim_pi,                 &
+                              sim_e,                  &
+                              sim_a1,                 &
+                              sim_a3,                 &
+                              sim_a1inv,              &
+                              sim_a3inv,              &
+                              sim_xctr,               &
+                              sim_yctr,               &
+                              sim_zctr,               &
+                              sim_density,            &
+                              sim_nsubinv,            &
+                              sim_nsubzones,          &
+                              sim_initGeometry,       &
+                              GRID_3DCARTESIAN,       &
+                              GRID_3DCYLINDRICAL,     &
+                              GRID_2DCYLINDRICAL,     &
+                              GRID_1DSPHERICAL,       &
+                              GRID_2DSPHERICAL
+
+  use Grid_interface,  ONLY : Grid_getBlkIndexLimits, &
+                              Grid_getCellCoords,     &
+                              Grid_putPointData,      &
+                              Grid_getDeltas
+
+  implicit none
+
+#include "constants.h"
+#include "Flash.h"
+
+  integer, intent (in) :: blockID
+
+  logical  :: gcell = .true.
+
+  integer  :: sizeX, sizeY, sizeZ
+  integer  :: k, kk, i, ii, j, jj
+  integer  :: ibeg,iend,jbeg,jend,kbeg,kend
+
+  real     :: a,b,c
+  real     :: a1squared, a3squared, a1squaredInv, a3squaredInv
+  real     :: AA1ch, AA3ch, Ich, distTerm
+  real     :: DD, Mass, phiSum, potentialAnalytical
+  real     :: dx, dy, dz, xx, yy, zz, dzz, dyy, dxx
+  real     :: xdist, ydist, zdist, rInv, rFunc, r2, z2, Rdist
+  real     :: lambda, h, hinv, H0, atan_h, H1, H3
+  real     :: theta
+
+  integer, dimension (MDIM)   :: Position
+  integer, dimension (2,MDIM) :: blkLimits, blkLimitsGC
+  real,    dimension (MDIM)   :: deltas
+
+  real, dimension(:), allocatable :: xLeft, yLeft, zLeft
+!
+!
+!    ...Set some convenience variables.
+!
+!
+  a1squared    = sim_a1 * sim_a1
+  a3squared    = sim_a3 * sim_a3
+  a1squaredInv = 1.0 / a1squared
+  a3squaredInv = 1.0 / a3squared
+!
+!
+!    ...The coefficients: A1                       -> AA1ch
+!                         A3                       -> AA3ch
+!                         {x,y,z} independent term -> Ich
+!                         potential prefactor      -> DD
+!                         total mass spheroid      -> Mass
+!
+!
+  if (abs(sim_e) < tiny(0.0)) then                ! sim_e=0, a sphere
+     AA1ch = 2.0 / 3.0
+     AA3ch = AA1ch
+  else if (abs(sim_e - 1.0) < tiny(0.0) ) then    ! sim_e = 1, a disk?
+     AA1ch = 0.0
+     AA3ch = 2
+  else                                            ! general case
+     AA1ch = sqrt(1.0-sim_e**2)*asin(sim_e)/sim_e**3 - (1.0-sim_e**2)/sim_e**2
+     AA3ch = 2.0/sim_e**2 - 2.0*sqrt(1.0-sim_e**2)*asin(sim_e)/sim_e**3
+  end if
+
+  Ich  = 2.0*AA1ch*a1squared + AA3ch*a3squared
+  DD   = sim_pi*sim_Newton*sim_density
+  Mass = 4.0/3.0*sim_pi*a1squared*sim_a3*sim_density
+!
+!
+!    ...Loop over all cells in current block, get the cell's {x,y,z} and
+!       calculate the analytical potential. Break each cell into nsubzones^ndim
+!       sub-zones and average the results to get the values for the cell.
+!       This prevents a blocky spheroid shell.
+!
+!
+  call Grid_getBlkIndexLimits (blockID,    &
+                               blkLimits,  &
+                               blkLimitsGC )
+
+  select case (sim_initGeometry)
+!
+!
+!   ...The 3D cartesian case.
+!
+!         Full consideration of subzones in all 3 dimensions necessary,
+!         as square of radius (r2) and square of z coordinate (z2) for
+!         evaluating the MacLaurin potential vary with all 3 dimensions.
+!
+!
+  case (GRID_3DCARTESIAN)
+
+    sizeX = blkLimitsGC (HIGH,IAXIS) - blkLimitsGC (LOW,IAXIS) + 1
+    sizeY = blkLimitsGC (HIGH,JAXIS) - blkLimitsGC (LOW,JAXIS) + 1
+    sizeZ = blkLimitsGC (HIGH,KAXIS) - blkLimitsGC (LOW,KAXIS) + 1
+
+    allocate (xLeft (sizeX))
+    allocate (yLeft (sizeY))
+    allocate (zLeft (sizeZ))
+
+    call Grid_getCellCoords (KAXIS, blockID, LEFT_EDGE, gcell, zLeft, sizeZ)
+    call Grid_getCellCoords (JAXIS, blockID, LEFT_EDGE, gcell, yLeft, sizeY)
+    call Grid_getCellCoords (IAXIS, blockID, LEFT_EDGE, gcell, xLeft, sizeX)
+    call Grid_getDeltas     (blockID, deltas)
+
+    dx = deltas(IAXIS)
+    dy = deltas(JAXIS)
+    dz = deltas(KAXIS)
+
+    kbeg = blkLimitsGC (LOW, KAXIS)
+    kend = blkLimitsGC (HIGH,KAXIS)
+    jbeg = blkLimitsGC (LOW, JAXIS)
+    jend = blkLimitsGC (HIGH,JAXIS)
+    ibeg = blkLimitsGC (LOW, IAXIS)
+    iend = blkLimitsGC (HIGH,IAXIS)
+
+    do k = kbeg,kend
+       dzz = dz * sim_nsubinv
+       do j = jbeg,jend
+          dyy = dy * sim_nsubinv
+          do i = ibeg,iend
+             dxx = dx * sim_nsubinv
+ 
+             phiSum = 0.0
+
+             do kk = 0,sim_nsubzones-1
+                zz    = zLeft (k) + (kk + 0.5) * dzz 
+                zdist = zz - sim_zctr
+                do jj = 0,sim_nsubzones-1
+                   yy    = yLeft (j) + (jj + 0.5) * dyy
+                   ydist = yy - sim_yctr
+                   do ii = 0,sim_nsubzones-1
+                      xx    = xLeft (i) + (ii + 0.5) * dxx
+                      xdist = xx - sim_xctr
+
+                      r2    = xdist**2 + ydist**2
+                      z2    = zdist**2
+
+                      rFunc = r2 * sim_a1inv * sim_a1inv  +  z2 * sim_a3inv * sim_a3inv
+
+                      if (rFunc <= 1.) then                                   ! inside the spheroid
+                          phiSum = phiSum + DD * (Ich - AA1ch*r2 - AA3ch*z2) 
+                      else                                                    ! outside the spheroid
+                          a      = 1.0
+                          b      = a1squared + a3squared - r2 - z2
+                          c      = a1squared * a3squared - r2 * a3squared - z2 * a1squared
+                          lambda = (-b + sqrt (b*b - 4.0*a*c)) / (2.0*a)
+                          Rinv   = 1.0 / sqrt (a3squared + lambda)
+
+                          if (abs(sim_e) < tiny(0.0)) then                    ! case for e=0  outside the sphere
+                              distTerm = 1.0 - (1.0/3.0)*Rinv**2*(r2 + z2)
+                          else                                                ! general case outside the spheroid
+                              h        = sim_a1 * sim_e * Rinv
+                              hinv     = 1.0/h                 
+                              atan_h   = atan(h)
+                              H0       = atan_h * hinv
+                              H1       = (atan_h - h/(1.0+h**2))*hinv**3
+                              H3       = (h - atan_h)*2.0*hinv**3
+                              distTerm = H0 - 0.5d0*Rinv**2*(r2*H1 + z2*H3)
+                          endif
+                          phiSum = phiSum + 1.50*sim_Newton*Mass*Rinv*distTerm
+                      endif
+
+                   enddo
+                enddo
+             enddo
+
+             potentialAnalytical = -phiSum/(sim_nsubzones**3)   
+
+             Position (KAXIS) = k
+             Position (JAXIS) = j
+             Position (IAXIS) = i
+
+             call Grid_putPointData (blockID, CENTER, APOT_VAR, EXTERIOR, Position, potentialAnalytical)
+
+          end do
+       enddo
+    enddo
+
+    deallocate(xLeft)
+    deallocate(yLeft)
+    deallocate(zLeft)
+!
+!
+!   ...The 3D cylindrical case.
+!
+!         Subzones need to be considered only for the radial and z dimension.
+!         The angular dimension does not need to be split into subzones (square
+!         of radius r2 and square of z coordinate (z2) for evaluating the MacLaurin
+!         potential vary only with radial and z dimension component of 3D cylindrical
+!         coordinate system.
+!
+!
+  case (GRID_3DCYLINDRICAL)
+
+    sizeX = blkLimitsGC (HIGH,IAXIS) - blkLimitsGC (LOW,IAXIS) + 1
+    sizeY = blkLimitsGC (HIGH,JAXIS) - blkLimitsGC (LOW,JAXIS) + 1
+
+    allocate (xLeft (sizeX))
+    allocate (yLeft (sizeY))
+
+    call Grid_getCellCoords (JAXIS, blockID, LEFT_EDGE, gcell, yLeft, sizeY)
+    call Grid_getCellCoords (IAXIS, blockID, LEFT_EDGE, gcell, xLeft, sizeX)
+    call Grid_getDeltas     (blockID, deltas)
+
+    dx = deltas(IAXIS)
+    dy = deltas(JAXIS)
+
+    kbeg = blkLimitsGC (LOW, KAXIS)
+    kend = blkLimitsGC (HIGH,KAXIS)
+    jbeg = blkLimitsGC (LOW, JAXIS)
+    jend = blkLimitsGC (HIGH,JAXIS)
+    ibeg = blkLimitsGC (LOW, IAXIS)
+    iend = blkLimitsGC (HIGH,IAXIS)
+
+    do j = jbeg,jend
+       dyy = dy * sim_nsubinv           ! z component
+       do i = ibeg,iend
+          dxx = dx * sim_nsubinv        ! radial component
+ 
+          phiSum = 0.0
+
+          do jj = 0,sim_nsubzones-1
+             yy    = yLeft (j) + (jj + 0.5) * dyy
+             zdist = yy - sim_yctr
+             do ii = 0,sim_nsubzones-1
+                xx    = xLeft (i) + (ii + 0.5) * dxx
+                Rdist = xx - sim_xctr
+
+                r2 = Rdist ** 2
+                z2 = zdist ** 2
+
+                rFunc = r2 * sim_a1inv * sim_a1inv  +  z2 * sim_a3inv * sim_a3inv
+
+                if (rFunc <= 1.) then                                   ! inside the spheroid
+                    phiSum = phiSum + DD * (Ich - AA1ch*r2 - AA3ch*z2) 
+                else                                                    ! outside the spheroid
+                    a      = 1.0
+                    b      = a1squared + a3squared - r2 - z2
+                    c      = a1squared * a3squared - r2 * a3squared - z2 * a1squared
+                    lambda = (-b + sqrt (b*b - 4.0*a*c)) / (2.0*a)
+                    Rinv   = 1.0 / sqrt (a3squared + lambda)
+
+                    if (abs(sim_e) < tiny(0.0)) then                    ! case for e=0  outside the sphere
+                        distTerm = 1.0 - (1.0/3.0)*Rinv**2*(r2 + z2)
+                    else                                                ! general case outside the spheroid
+                        h        = sim_a1 * sim_e * Rinv
+                        hinv     = 1.0/h                 
+                        atan_h   = atan(h)
+                        H0       = atan_h * hinv
+                        H1       = (atan_h - h/(1.0+h**2))*hinv**3
+                        H3       = (h - atan_h)*2.0*hinv**3
+                        distTerm = H0 - 0.5d0*Rinv**2*(r2*H1 + z2*H3)
+                    endif
+                    phiSum = phiSum + 1.50*sim_Newton*Mass*Rinv*distTerm
+                endif
+
+             enddo
+          enddo
+
+          potentialAnalytical = - phiSum / (sim_nsubzones**2)   
+
+          do k = kbeg,kend           ! angular component -> no subzones -> identical potentials
+
+             Position (KAXIS) = k
+             Position (JAXIS) = j
+             Position (IAXIS) = i
+
+             call Grid_putPointData (blockID, CENTER, APOT_VAR, EXTERIOR, Position, potentialAnalytical)
+
+          end do
+
+       enddo
+    enddo
+
+    deallocate(xLeft)
+    deallocate(yLeft)
+!
+!
+!   ...The 2D cylindrical case.
+!
+!
+  case (GRID_2DCYLINDRICAL)
+
+    sizeX = blkLimitsGC (HIGH,IAXIS) - blkLimitsGC (LOW,IAXIS) + 1
+    sizeY = blkLimitsGC (HIGH,JAXIS) - blkLimitsGC (LOW,JAXIS) + 1
+
+    allocate (xLeft (sizeX))
+    allocate (yLeft (sizeY))
+
+    call Grid_getCellCoords (JAXIS, blockID, LEFT_EDGE, gcell, yLeft, sizeY)
+    call Grid_getCellCoords (IAXIS, blockID, LEFT_EDGE, gcell, xLeft, sizeX)
+    call Grid_getDeltas     (blockID, deltas)
+
+    dx = deltas(IAXIS)
+    dy = deltas(JAXIS)
+
+    jbeg = blkLimitsGC (LOW, JAXIS)
+    jend = blkLimitsGC (HIGH,JAXIS)
+    ibeg = blkLimitsGC (LOW, IAXIS)
+    iend = blkLimitsGC (HIGH,IAXIS)
+
+    do j = jbeg,jend
+       dyy = dy * sim_nsubinv
+       do i = ibeg,iend
+          dxx = dx * sim_nsubinv
+
+          phiSum = 0.0
+
+          do jj = 0,sim_nsubzones-1
+             yy    = yLeft (j) + (jj + 0.5) * dyy
+             zdist = yy - sim_yctr
+             do ii = 0,sim_nsubzones-1
+                xx    = xLeft (i) + (ii + 0.5) * dxx
+                Rdist = xx - sim_xctr
+
+                r2    = Rdist**2
+                z2    = zdist**2
+
+                rFunc = r2 * sim_a1inv * sim_a1inv  +  z2 * sim_a3inv * sim_a3inv
+
+                if (rFunc <= 1.) then                                     ! inside the spheroid
+                    phiSum = phiSum + DD * (Ich - AA1ch*r2 - AA3ch*z2) 
+                else                                                      ! outside the spheroid
+                    a      = 1.0
+                    b      = a1squared + a3squared - r2 - z2
+                    c      = a1squared * a3squared - r2 * a3squared - z2 * a1squared
+                    lambda = (-b + sqrt (b*b - 4.0*a*c)) / (2.0*a)
+                    Rinv   = 1.0 / sqrt (a3squared + lambda)
+
+                    if (abs(sim_e) < tiny(0.0)) then                      ! case for e=0  outside the sphere
+                        distTerm = 1.0 - (1.0/3.0)*Rinv**2*(r2 + z2)
+                    else                                                  ! general case outside the spheroid
+                        h        = sim_a1 * sim_e * Rinv
+                        hinv     = 1.0/h                 
+                        atan_h   = atan(h)
+                        H0       = atan_h * hinv
+                        H1       = (atan_h - h/(1.0+h**2))*hinv**3
+                        H3       = (h - atan_h)*2.0*hinv**3
+                        distTerm = H0 - 0.5d0*Rinv**2*(r2*H1 + z2*H3)
+                    endif
+                    phiSum = phiSum + 1.50*sim_Newton*Mass*Rinv*distTerm
+                endif
+
+             enddo
+          enddo
+
+          potentialAnalytical = -phiSum/(sim_nsubzones**2)   
+
+          Position (JAXIS) = j
+          Position (IAXIS) = i
+
+          call Grid_putPointData(blockID, CENTER, APOT_VAR, EXTERIOR, Position, potentialAnalytical)
+
+       enddo
+    enddo
+
+    deallocate(xLeft)
+    deallocate(yLeft)
+!
+!
+!   ...The 2D spherical case.
+!
+!
+  case (GRID_2DSPHERICAL)
+
+    sizeX = blkLimitsGC (HIGH,IAXIS) - blkLimitsGC (LOW,IAXIS) + 1
+    sizeY = blkLimitsGC (HIGH,JAXIS) - blkLimitsGC (LOW,JAXIS) + 1
+
+    allocate (xLeft (sizeX))
+    allocate (yLeft (sizeY))
+
+    call Grid_getCellCoords (JAXIS, blockID, LEFT_EDGE, gcell, yLeft, sizeY)
+    call Grid_getCellCoords (IAXIS, blockID, LEFT_EDGE, gcell, xLeft, sizeX)
+    call Grid_getDeltas     (blockID, deltas)
+
+    dx = deltas(IAXIS)
+    dy = deltas(JAXIS)
+
+    jbeg = blkLimitsGC (LOW, JAXIS)
+    jend = blkLimitsGC (HIGH,JAXIS)
+    ibeg = blkLimitsGC (LOW, IAXIS)
+    iend = blkLimitsGC (HIGH,IAXIS)
+
+    do j = jbeg,jend
+
+       dyy = dy * sim_nsubinv
+       do i = ibeg,iend
+          dxx = dx * sim_nsubinv
+
+          phiSum = 0.0
+
+          do jj = 0,sim_nsubzones-1
+             yy    = yLeft (j) + (jj + 0.5) * dyy
+             theta = yy - sim_yctr
+             do ii = 0,sim_nsubzones-1
+                xx    = xLeft (i) + (ii + 0.5) * dxx
+                Rdist = xx - sim_xctr                                      ! Rdist is x^2 + y^2 + z^2 here!
+
+                zdist = Rdist * cos (theta)
+
+                z2    = zdist**2                                           ! this is z^2
+                r2    = Rdist**2  - z2                                     ! this is x^2 + y^2
+
+                rFunc = r2 * sim_a1inv * sim_a1inv  +  z2 * sim_a3inv * sim_a3inv
+
+                if (rFunc <= 1.) then                                      ! inside the spheroid
+                    phiSum = phiSum + DD * (Ich - AA1ch*r2 - AA3ch*z2) 
+                else                                                       ! outside the spheroid
+                    a      = 1.0
+                    b      = a1squared + a3squared - r2 - z2
+                    c      = a1squared * a3squared - r2 * a3squared - z2 * a1squared
+                    lambda = (-b + sqrt (b*b - 4.0*a*c)) / (2.0*a)
+                    Rinv   = 1.0 / sqrt (a3squared + lambda)
+
+                    if (abs(sim_e) < tiny(0.0)) then                       ! case for e=0  outside the sphere
+                        distTerm = 1.0 - (1.0/3.0)*Rinv**2*(r2 + z2)
+                    else                                                   ! general case outside the spheroid
+                        h        = sim_a1 * sim_e * Rinv
+                        hinv     = 1.0/h                 
+                        atan_h   = atan(h)
+                        H0       = atan_h * hinv
+                        H1       = (atan_h - h/(1.0+h**2))*hinv**3
+                        H3       = (h - atan_h)*2.0*hinv**3
+                        distTerm = H0 - 0.5d0*Rinv**2*(r2*H1 + z2*H3)
+                    endif
+                    phiSum = phiSum + 1.50*sim_Newton*Mass*Rinv*distTerm
+                endif
+
+             enddo
+          enddo
+
+          potentialAnalytical = -phiSum/(sim_nsubzones**2)   
+
+          Position (JAXIS) = j
+          Position (IAXIS) = i
+
+          call Grid_putPointData(blockID, CENTER, APOT_VAR, EXTERIOR, Position, potentialAnalytical)
+
+       enddo
+    enddo
+
+    deallocate(xLeft)
+    deallocate(yLeft)
+
+  case (GRID_1DSPHERICAL)
+
+    write (*,*) ' Analytical MacLaurin for 1D spherical not implemented! '
+
+  end select
+!
+!
+!   ...Ready!
+!
+!
+  return
+end subroutine sim_initBlockAnalytical
Index: source/Simulation/SimulationMain/MacLaurin_GPU/sim_analyzePotential.F90
===================================================================
--- source/Simulation/SimulationMain/MacLaurin_GPU/sim_analyzePotential.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Simulation/SimulationMain/MacLaurin_GPU/sim_analyzePotential.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,199 @@
+!!****if* source/Simulation/SimulationMain/unitTest/Multipole/sim_analyzePotential
+!!
+!! NAME
+!!
+!!  sim_analyzePotential
+!! 
+!! SYNOPSIS
+!!
+!!  call sim_analyzePotential (logical (out) :: perfect)
+!!
+!! DESCRIPTION
+!!
+!!  This function analyzes the gravitational MacLaurin potential obtained using
+!!  the grid multipole solver. The analytical and numerical gravitational potentials
+!!  have been calculated and several additional data is determined and stored away
+!!  for eventual visualization.
+!!
+!!  The following data is generated for each cell:
+!!
+!!    1) The difference     :    analytical - numerical
+!!    2) The ratio          :    analytical / numerical
+!!    3) The absolute error : | (analytical - numerical) / analytical |
+!!
+!!  The maximum of the absolute error is determined on the current processor and
+!!  used as a criterion for success or failure. The current error tolerance is
+!!  based on errors obtained from previous runs for the current refinement level.
+!!  This can be extended to include several refinement levels, each of which would
+!!  then have a specific error tolerance which must be determined previously.
+!!
+!! ARGUMENTS 
+!! 
+!!  perfect : test status indicator (if true, test ran without errors).
+!!
+!!***
+
+!!REORDER(4): solnData
+
+subroutine sim_analyzePotential (perfect)
+
+  use Driver_data,       ONLY : dr_globalMe
+  use Grid_data, ONLY : gr_meshMe,gr_meshNumProcs, gr_meshComm
+
+  use Grid_interface,    ONLY : Grid_getBlkIndexLimits, &
+                                Grid_getBlkPtr,         &
+                                Grid_getListOfBlocks,   &
+                                Grid_releaseBlkPtr
+
+  use Simulation_data,   ONLY:  sim_passTolerance
+
+  implicit none
+
+# include "constants.h"
+# include "Flash.h"
+# include "Flash_mpi.h"
+
+  logical, intent(out) :: perfect
+
+  integer :: ib,ie,jb,je,kb,ke,i
+  integer :: blkCount
+  integer :: blockID
+  real    :: potError, potErrorMax,potErrorMin ,all_potError, all_potErrorMax,all_potErrorMin
+  real    :: factorMin, factorMax,all_factorMin, all_factorMax
+  real    :: apotAbsMax, gpotAbsMax, all_apotAbsMax, all_gpotAbsMax
+
+  integer :: blkLimits   (1:2,1:MDIM)
+  integer :: blkLimitsGC (1:2,1:MDIM)
+  integer :: blkList     (1:MAXBLOCKS)
+
+  real, pointer :: solnData (:,:,:,:)
+
+
+  integer :: ii,jj,kk,ierr
+
+
+!
+!
+!   ...Get all leaf blocks and test the obtained potential.
+!
+!  
+  call Grid_getListOfBlocks (LEAF, blkList, blkCount)
+
+  potErrorMax =  tiny (0.0)
+  potErrorMin =  10e99
+  apotAbsMax  =  tiny (0.0)
+  gpotAbsMax  =  tiny (0.0)
+  factorMin   =  1.0E10
+  factorMax   = -1.0E10
+
+  do i = 1,blkCount
+
+     blockID = blkList (i)
+
+     call Grid_getBlkPtr         (blockID, solnData)
+     call Grid_getBlkIndexLimits (blockID, blkLimits, blkLimitsGC)
+
+     ib = blkLimits (LOW, IAXIS)
+     ie = blkLimits (HIGH,IAXIS)
+     jb = blkLimits (LOW, JAXIS)
+     je = blkLimits (HIGH,JAXIS)
+     kb = blkLimits (LOW, KAXIS)
+     ke = blkLimits (HIGH,KAXIS)
+!
+!
+!   ...Compute error variables: ERRM = difference, ERRD = division, ERRN = normalized
+!
+!  
+     solnData (ERRM_VAR,ib:ie,jb:je,kb:ke) =   solnData (APOT_VAR,ib:ie,jb:je,kb:ke) &
+                                             - solnData (GPOT_VAR,ib:ie,jb:je,kb:ke)
+    
+     solnData (ERRD_VAR,ib:ie,jb:je,kb:ke) =   solnData (APOT_VAR,ib:ie,jb:je,kb:ke) &
+                                             / solnData (GPOT_VAR,ib:ie,jb:je,kb:ke)
+
+     solnData (ERRN_VAR,ib:ie,jb:je,kb:ke) = abs (  solnData (ERRM_VAR,ib:ie,jb:je,kb:ke) &
+                                                  / solnData (APOT_VAR,ib:ie,jb:je,kb:ke) )
+
+
+!    if (maxval (abs (solnData (ERRN_VAR,ib:ie,jb:je,kb:ke))) > 0.0115) then
+!        write (*,*) ' blockID = ',blockID
+!    end if
+
+
+!     if (blockID == 74) then
+!         do ii = ib,ie
+!         do jj = jb,je
+!         do kk = kb,ke
+!            if (abs (solnData (ERRN_VAR,ii,jj,kk)) > 0.0115) then
+!                write (*,*) ' i,j,k,P = ',ii,jj,kk,solnData (GPOT_VAR,ii,jj,kk)
+!            end if
+!         end do
+!         end do
+!         end do
+!     end if
+
+
+     poterrorMax = max (potErrorMax, maxval (abs (solnData (ERRN_VAR,ib:ie,jb:je,kb:ke))))
+     poterrorMin = min (poterrorMin, minval (abs (solnData (ERRN_VAR,ib:ie,jb:je,kb:ke))))
+     apotAbsMax  = max (apotAbsMax , maxval (abs (solnData (APOT_VAR,ib:ie,jb:je,kb:ke))))
+     gpotAbsMax  = max (gpotAbsMax , maxval (abs (solnData (GPOT_VAR,ib:ie,jb:je,kb:ke))))
+     factorMax   = max (factorMax  , maxval (     solnData (ERRD_VAR,ib:ie,jb:je,kb:ke)) )
+     factorMin   = min (factorMin  , minval (     solnData (ERRD_VAR,ib:ie,jb:je,kb:ke)) )
+    ! write(*,*)"poterrorMAx",poterrorMax
+    ! write(*,*)"apotAbsMax",apotAbsMax
+    ! write(*,*)"gpotAbsMax",gpotAbsMax
+
+     call Grid_releaseBlkPtr (blockID, solnData)
+
+  end do
+!
+!
+!   ...Analyze the maximum absolute error obtained.
+!
+!  
+  if(potErrorMax < sim_passTolerance) then
+     perfect=.true.
+     write (*,*) 'Proc',dr_globalMe, ' Maximum potential error = ',potErrorMax,' SUCCESS! '
+     write (*,*) 'Proc',dr_globalMe, ' Minimum potential error = ',poterrorMin,' SUCCESS! '
+  else
+     perfect=.false.
+     write (*,*) 'Proc',dr_globalMe, ' Maximum potential error = ',potErrorMax,' FAILURE! '
+  end if
+!
+!
+!   ...Ready!
+!
+!
+ call MPI_Reduce(poterrorMax, all_poterrorMax, 1, MPI_DOUBLE, MPI_MAX,0, gr_meshComm,ierr) 
+ call MPI_Reduce(factorMax, all_factorMax, 1, MPI_DOUBLE, MPI_MAX,0, gr_meshComm,ierr) 
+ call MPI_Reduce(factorMin, all_factorMin, 1, MPI_DOUBLE, MPI_MIN,0, gr_meshComm,ierr) 
+ call MPI_Reduce(poterrorMin, all_poterrorMin, 1, MPI_DOUBLE, MPI_MIN,0, gr_meshComm,ierr) 
+ 
+ if(gr_meshMe .eq. 0) then
+    write (*,*)"======================================================"
+    write (*,*)"======================================================"
+    write (*,*)"======================================================"
+    write (*,*)"======================================================"
+    write (*,*)"=============Overall Errors==========================="
+    write (*,*) "PASS TOLERANCE = ",sim_passTolerance,"!"
+    if(all_potErrorMax <= sim_passTolerance) then
+    write (*,*) "Maximum potential error = ",all_potErrorMax,"SUCCESS! "
+    endif
+    if(all_potErrorMax > sim_passTolerance) then
+    write (*,*) "Maximum potential error = ",all_potErrorMax,"FAILURE! "
+    endif
+    if(all_potErrorMin <= sim_passTolerance) then
+    write (*,*) "Minimum potential error = ",all_potErrorMin,"SUCCESS! "
+    endif
+    if(all_potErrorMin > sim_passTolerance) then
+     write (*,*) "Minimum potential error = ",all_potErrorMin,"FAILURE! "
+    endif
+    write (*,*) "Maximum factorial error = ",all_factorMax," "
+    write (*,*) "Minimum factorial error = ",all_factorMin,""
+    write (*,*)"======================================================"
+    write (*,*)"======================================================"
+    write (*,*)"======================================================"
+    write (*,*)"======================================================"
+   
+ endif
+  return
+end subroutine sim_analyzePotential
Index: source/Simulation/SimulationMain/MacLaurin_GPU/Simulation_data.F90
===================================================================
--- source/Simulation/SimulationMain/MacLaurin_GPU/Simulation_data.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Simulation/SimulationMain/MacLaurin_GPU/Simulation_data.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,40 @@
+!!****if* source/Simulation/SimulationMain/unitTest/Multipole/Simulation_data
+!!
+!! NAME
+!!
+!!  Simulation_data
+!!
+!! SYNOPSIS
+!!
+!!  use Simulation_data
+!!
+!! DESCRIPTION
+!!
+!!  Stores the local data for the grid multipole solver unit test, based on
+!!  the gravitational MacLaurin spheroid problem.
+!!
+!!***
+
+module Simulation_data
+
+  implicit none
+  
+  integer, save :: sim_initGeometry
+  integer, save :: sim_globalMe
+  integer, save :: sim_nsubzones
+  real,    save :: sim_passTolerance              ! maximum percentage error allowed for success
+  real,    save :: sim_e                          ! eccentricity
+  real,    save :: sim_density
+  real,    save :: sim_xctr, sim_yctr, sim_zctr
+  real,    save :: sim_Newton, sim_pi
+  real,    save :: sim_nsubinv
+  real,    save :: sim_a1, sim_a3, sim_a1inv, sim_a3inv
+  real,    save :: sim_smallRho
+
+  integer, parameter :: GRID_3DCARTESIAN    =  1, &
+                        GRID_3DCYLINDRICAL  =  2, &
+                        GRID_2DCYLINDRICAL  =  3, &
+                        GRID_1DSPHERICAL    =  4, &
+                        GRID_2DSPHERICAL    =  5
+
+end module Simulation_data
Index: source/Simulation/SimulationMain/MacLaurin_GPU/Makefile
===================================================================
--- source/Simulation/SimulationMain/MacLaurin_GPU/Makefile	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Simulation/SimulationMain/MacLaurin_GPU/Makefile	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,5 @@
+
+Simulation += Simulation_data.o      \
+              sim_analyzePotential.o     \
+              sim_initBlockAnalytical.o
+              
Index: source/Simulation/SimulationMain/MacLaurin_GPU/flash.par
===================================================================
--- source/Simulation/SimulationMain/MacLaurin_GPU/flash.par	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Simulation/SimulationMain/MacLaurin_GPU/flash.par	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,135 @@
+# Runtime parameters for the Maclaurin spheroid problem.
+#
+#               3D cartesian version:
+#
+#     calculation domain    x = from 0. to 1.
+#                           y = from 0. to 1.
+#                           z = from 0. to 1.
+#
+#     center of spheroid at x = 0.5
+#                           y = 0.5
+#                           z = 0.5
+#
+#-------------------------------------------------------------------------------
+
+# Parameters for initial model
+
+
+density                   = 1.        # Spheroid density (rho)
+eccentricity              = 0.9       # Eccentricity of spheroid (e)
+equatorialSemimajorAxis   = 0.25      # Equatorial semimajor axis (a1)
+passTolerance             = 0.015 #= 1.5 percent error
+xctr                      = 0.5       # Coordinates of center of spheroid
+yctr                      = 0.5
+zctr                      = 0.5
+
+nsubzones                 = 2         # Number of averaging sub-zones per dimen.
+
+#-------------------------------------------------------------------------------
+
+# Computational volume parameters
+
+xmin                      = 0.        # Physical dimensions of the box
+xmax                      = 1.
+ymin                      = 0.
+ymax                      = 1.
+zmin                      = 0.
+zmax                      = 1.
+
+xl_boundary_type          = "outflow" # Boundary conditions
+xr_boundary_type          = "outflow"
+yl_boundary_type          = "outflow"
+yr_boundary_type          = "outflow"
+zl_boundary_type          = "outflow"
+zr_boundary_type          = "outflow"
+grav_boundary_type        = "isolated"
+
+#-------------------------------------------------------------------------------
+
+# Adaptive mesh parameters
+
+lrefine_max               = 4         # Maximum refinement level
+lrefine_min               = 4         # Minimum refinement level
+
+refine_var_1              = "dens"    # Refinement variables
+refine_var_2              = "pres"
+
+Nblockx                   = 1         # Number of top-level blocks
+Nblocky                   = 1
+Nblockz                   = 1
+
+convertToConsvdInMeshInterp  = .true.    # Interpolate conserved variables?
+
+#-------------------------------------------------------------------------------
+
+# Time and I/O parameters
+
+run_comment               = "Maclaurin spheroid problem"
+                                      # Comment to use in log file
+basenm                    = "maclaurin_"
+                                      # Base name for output files
+
+restart                   = .false.   # Start from checkpoint file?
+checkpointFileNumber      = 0         # Initial checkpoint file number
+
+checkpointFileIntervalTime = 0.1      # Checkpoint simulation-time interval
+checkpointFileIntervalStep = 200       # Checkpoint step interval
+wall_clock_checkpoint      = 3600.     # Checkpoint wall-clock interval
+
+dtinit                    = 1.0E-2    # Initial time step
+dtmin                     = 1.0E-3    # Minimum time step
+dtmax                     = 1.0E-1    # Maximum time step
+
+nend                      = 1     # Maximum number of steps to take
+tmax                      = 1.0       # Maximum time to run
+
+plot_var_1                = "dens"
+plot_var_2                = "pres"
+plot_var_3                = "temp"
+plot_var_4                = "ener"
+plot_var_5                = "eint"
+plot_var_5                = "gpot"
+plot_var_5                = "apot"
+
+#-------------------------------------------------------------------------------
+
+# Physics/solver parameters
+
+gamma                     = 1.6666667 # Ratio of specific heats
+cfl                       = 0.8       # Courant timestep parameter
+
+useGravity                = .true.    # Turn on gravity?
+
+
+
+#GPU_BH tree code Parameter
+grv_bh_gpu_TreeLimAngle = 0.5
+grv_bh_gpu_PinnedCells  = .false. # use CudaPinned memory for data cells  ?
+grv_bh_gpu_PinnedMPI    = .false. # use CudaPinned memory for MPI buffers ?
+
+#-------------------------------------------------------------------------------
+
+# End
+eintSwitch     = 1.e-4
+
+ 
+## -------------------------------------------------------------##
+##  SWITCHES SPECIFIC TO THE UNSPLIT HYDRO SOLVER               ##
+#	I. INTERPOLATION SCHEME:
+order		= 2      # Interpolation order (first/second/third/fifth order)
+slopeLimiter    = "mc"   # Slope limiters (minmod, mc, vanLeer, hybrid, limited)
+LimitedSlopeBeta= 1.     # Slope parameter for the "limited" slope by Toro
+charLimiting	= .true. # Characteristic limiting vs. Primitive limiting
+
+use_avisc	= .false. # use artificial viscosity (originally for PPM)
+#cvisc		= 0.1     # coefficient for artificial viscosity
+use_flattening	= .false. # use flattening (dissipative) (originally for PPM)
+use_steepening	= .false. # use contact steepening (originally for PPM)
+use_upwindTVD	= .false. # use upwind biased TVD slope for PPM (need nguard=6)
+
+#	II. RIEMANN SOLVERS:
+RiemannSolver	= "Roe"       # Roe, HLL, HLLC, LLF, Marquina
+entropy         = .false.     # Entropy fix for the Roe solver
+
+#	III. STRONG SHOCK HANDELING SCHEME:
+shockDetect	= .false.     # Shock Detect for numerical stability
Index: source/Grid/localAPI/gr_bh_gpu_Finalize.F90
===================================================================
--- source/Grid/localAPI/gr_bh_gpu_Finalize.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/localAPI/gr_bh_gpu_Finalize.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,23 @@
+!!****if* source/Grid/localAPI/gr_bh_gpu_Finalize
+!!
+!! NAME
+!!
+!!  gr_bhFinalize
+!!
+!! 
+!! SYNOPSIS
+!!
+!!  call gr_bhFinalize()
+!!
+!!
+!! DESCRIPTION
+!!
+!!  Finalizes the tree Poisson solver.  Deallocates all arrays.
+!!
+!!***
+
+subroutine gr_bh_gpu_Finalize()
+  implicit none
+  
+  return
+end subroutine gr_bh_gpu_Finalize
Index: source/Grid/localAPI/gr_bh_gpu_Init.F90
===================================================================
--- source/Grid/localAPI/gr_bh_gpu_Init.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/localAPI/gr_bh_gpu_Init.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,26 @@
+!!****if* source/Grid/localAPI/gr_bh_gpu_Init
+!!
+!! NAME
+!!
+!!  gr_bhInit
+!!
+!! 
+!! SYNOPSIS
+!!
+!!  call gr_bhInit()
+!!
+!!
+!! DESCRIPTION
+!!
+!!  Initialize the tree Poisson solver.  Read in any of the
+!!  runtime parameters for this solver.  
+!!
+!!***
+
+subroutine gr_bh_gpu_Init()
+  implicit none
+  
+  return
+end subroutine gr_bh_gpu_Init
+
+
Index: source/Grid/localAPI/Makefile
===================================================================
--- source/Grid/localAPI/Makefile	(.../tags/FLASH4.4_release)	(revision 25574)
+++ source/Grid/localAPI/Makefile	(.../branches/flash4_lukat)	(revision 25574)
@@ -72,6 +72,8 @@
 
 Grid += gr_bhInterface.o gr_bhInit.o gr_bhFinalize.o
 
+Grid += gr_bhGpuInterface.o gr_bh_gpu_Init.o gr_bh_gpu_Finalize.o
+
 Grid += gr_xyzToBlockLevel.o gr_xyzToBlock.o
 
 gr_interface.o     : gr_interfaceTypeDecl.o
Index: source/Grid/localAPI/gr_bhGpuInterface.F90
===================================================================
--- source/Grid/localAPI/gr_bhGpuInterface.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/localAPI/gr_bhGpuInterface.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,33 @@
+!!****h* source/Grid/localAPI/gr_bhGpuInterface
+!!
+!! This is the header file for the GPU implementation of the Barnes-Hut tree solver that defines its
+!! interfaces private to the Grid unit.
+!!***
+Module gr_bhGpuInterface
+ implicit none
+
+!====================================
+!INITIALIZE
+!====================================
+ interface gr_bh_gpu_init
+   subroutine gr_bh_gpu_init()
+   end subroutine gr_bh_gpu_init
+ end interface
+
+
+!====================================
+!FINALIZE 
+!====================================
+
+
+interface gr_bh_gpu_Finalize
+   subroutine gr_bh_gpu_Finalize()
+   end subroutine gr_bh_gpu_Finalize
+ end interface
+
+
+end Module  gr_bhGpuInterface
+
+
+
+
Index: source/Grid/GridSolvers/gr_solversFinalize.F90
===================================================================
--- source/Grid/GridSolvers/gr_solversFinalize.F90	(.../tags/FLASH4.4_release)	(revision 25574)
+++ source/Grid/GridSolvers/gr_solversFinalize.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -26,6 +26,7 @@
   use gr_mgInterface, ONLY : gr_mgFinalize
   use gr_bicgInterface, ONLY : gr_bicgFinalize
   use gr_bhInterface, ONLY : gr_bhFinalize
+  use gr_bhGpuInterface, ONLY : gr_bh_gpu_Finalize
 
   implicit none 
 
@@ -56,5 +57,8 @@
   ! BHTree
   call gr_bhFinalize()
 
+  ! BHTree
+  call gr_bh_gpu_Finalize()
+  
   return
 end subroutine gr_solversFinalize
Index: source/Grid/GridSolvers/gr_solversInit.F90
===================================================================
--- source/Grid/GridSolvers/gr_solversInit.F90	(.../tags/FLASH4.4_release)	(revision 25574)
+++ source/Grid/GridSolvers/gr_solversInit.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -26,6 +26,7 @@
   use gr_mgInterface, ONLY : gr_mgInit, gr_mgPfftInit
   use gr_bicgInterface, ONLY : gr_bicgInit
   use gr_bhInterface, ONLY : gr_bhInit
+  use gr_bhGpuInterface, ONLY : gr_bh_gpu_Init
 
   implicit none 
 
@@ -58,6 +59,8 @@
 
   ! Barnes-Hut Tree
   call gr_bhInit()
+ ! GPU-Barnes-Hut Tree
+  call gr_bh_gpu_Init()
 
   ! solver unit testing
   call gr_solversTestInit()
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_MakeField.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_MakeField.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_MakeField.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,160 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+
+
+// PERIODICITY AND EWALD
+#include "gr_bh_gpu_Ewald_Real_Periodic_Kernel.cuh"
+#include "gr_bh_gpu_Ewald_Rec_Periodic_Kernel.cuh"
+#include "gr_bh_gpu_Ewald_Extrapolate_Kernel.cuh"
+#include "gr_bh_gpu_Ewald_Temp_Field_Kernel.cuh"
+#include "gr_bh_gpu_Periodic_InterpolateInField.cuh"
+
+
+extern "C" {
+void gr_bh_gpu_Ewald_MakeField_(int *NX, int *NY, int *NZ, double *grid,double *x_cell,double *y_cell,double *z_cell,
+                    double *e_xmax,double *e_ymax,double *e_zmax, int *periodics)
+    {  
+    
+    double *device_grid,*t_gridx,*t_gridy,*t_gridz;
+
+    double *d_cell;
+   
+    int periodicity =*periodics;
+
+ if (cudaSuccess != cudaMemcpyToSymbol(periodic, &periodicity, sizeof(int))) 
+                     fprintf(stderr, "copying of nbodies to device failed\n"); 
+                     CudaTest( (char*)"nbody copy to device failed");
+
+    double *device_grid_res;
+
+    if (cudaSuccess != cudaMalloc((void **)&device_grid_res, sizeof(double) * ((*NX)*(*NY)*(*NZ)*8)) )
+                        fprintf(stderr, "could not allocate poszd\n");  
+                        CudaTest( (char*)"couldn't allocate poszd");
+
+    if (cudaSuccess != cudaMalloc((void **)&device_grid, sizeof(double) * ((*NX)*(*NY)*(*NZ))) )
+                        fprintf(stderr, "could not allocate poszd\n");  
+                        CudaTest( (char*)"couldn't allocate poszd");
+
+    if (cudaSuccess != cudaMalloc((void **)&t_gridx, sizeof(double) * ((*NX)*(*NY)*(*NZ))) )
+                        fprintf(stderr, "could not allocate poszd\n");  
+                        CudaTest( (char*)"couldn't allocate poszd");
+
+    if (cudaSuccess != cudaMalloc((void **)&t_gridy, sizeof(double) * ((*NX)*(*NY)*(*NZ))) )
+                        fprintf(stderr, "could not allocate poszd\n");  
+                        CudaTest( (char*)"couldn't allocate poszd");
+
+    if (cudaSuccess != cudaMalloc((void **)&t_gridz, sizeof(double) * ((*NX)*(*NY)*(*NZ))) )
+                        fprintf(stderr, "could not allocate poszd\n");  
+                        CudaTest( (char*)"couldn't allocate poszd");
+
+
+    if (cudaSuccess != cudaMalloc((void **)&d_cell, sizeof(double) ))
+                        fprintf(stderr, "could not allocate poszd\n");  
+                        CudaTest( (char*)"couldn't allocate poszd");
+
+        double x = *x_cell;
+        double y = *y_cell;
+        double z = *z_cell;
+        double rc = sqrt(x*x+y*y+z*z);
+
+         double cell = 0.0;
+
+
+                 if (cudaSuccess != cudaMemcpy(d_cell, &cell, sizeof(double), cudaMemcpyHostToDevice))
+                     fprintf(stderr, "copying of posx to device failed\n");  
+                     CudaTest( (char*)"posx copy to device failed");
+
+                    
+
+
+        dim3 grids(2, 2,1);     // Grid -- 2 x 2 blocks
+ dim3 block(16, 16,1);    // Block -- 16 x 16 threads
+ 
+     
+         gen_temp_field<<<grids,block>>>(t_gridx,t_gridy,t_gridz) ;
+         CudaTest( (char*)"gem temp launch failed");
+        
+
+       
+         ewald_real_periodic_all<<<6 * FACTOR5, THREADS7>>>(t_gridx,t_gridy,t_gridz,device_grid);
+         CudaTest( (char*)"ewald real  launch failed");
+      
+         ewald_rec_periodic_all<<<6 * FACTOR5, THREADS7>>>(t_gridx,t_gridy,t_gridz,device_grid);
+         CudaTest( (char*)"ewald real  launch failed");
+       
+   
+         ewald_extrapolate_real<<<6 * FACTOR5, THREADS7>>>(x,y,z,d_cell);
+           CudaTest( (char*)"ewald extrapolate real  launch failed");
+      
+         ewald_extrapolate_rec<<<6 * FACTOR5, THREADS7>>>(x,y,z,d_cell);
+           CudaTest( (char*)"ewald extrapolate rec  launch failed");
+      
+
+
+    if (cudaSuccess != cudaMemcpy(grid, device_grid, sizeof(double) * ((*NX)*(*NY)*(*NZ)), cudaMemcpyDeviceToHost)) 
+                          fprintf(stderr, "copying of velx to device failed\n");  
+                           CudaTest( (char*)"velx copy to device failed");
+
+   if (cudaSuccess != cudaMemcpy(&cell, d_cell, sizeof(double), cudaMemcpyDeviceToHost)) 
+                          fprintf(stderr, "copying of velx to device failed\n");  
+                           CudaTest( (char*)"velx copy to device failed");
+        double re = sqrt(
+                         (*e_xmax/(*NX -1)) * (*e_xmax/(*NX -1)) 
+                       + (*e_ymax/(*NY -1)) * (*e_ymax/(*NY -1)) 
+                       + (*e_zmax/(*NZ -1)) * (*e_zmax/(*NZ -1))
+                         );
+
+     
+
+        double p = (-rc)/(re-rc);
+        
+        grid[0] = (1.0-p) * cell + p * grid[1 + 1*(*NX) + 1 * (*NY)*(*NX) ];
+   
+
+   
+
+
+
+
+     cudaFree(device_grid);
+     cudaFree(device_grid_res);
+     cudaFree(t_gridx);
+     cudaFree(t_gridy);
+     cudaFree(t_gridz);
+     cudaFree(d_cell);
+
+    }
+}
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Rec_Periodic_Kernel.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Rec_Periodic_Kernel.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Rec_Periodic_Kernel.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,51 @@
+#include "gr_bh_gpu_Ewald_Rec_Periodic_Kernel.cuh"
+
+
+
+__global__ void ewald_rec_periodic_all(double *t_grid_x,double *t_grid_y, double *t_grid_z,double *grid)
+{
+int xx = threadIdx.x + blockIdx.x * blockDim.x;
+int inc = blockDim.x * gridDim.x;
+
+
+
+double x, y,z, k2,kx,t_rec;
+int id,ni,nj,nk;
+
+
+
+
+
+ while(xx< (d_NX * d_NY *d_NZ))       
+    {
+ 
+     id =(xx);
+     x = t_grid_x[xx];
+     y = t_grid_y[xx];
+     z = t_grid_z[xx];
+     t_rec=0;
+     
+        for( ni = -d_es_nrx; ni < d_es_nrx; ni++){
+            for( nj = -d_es_nry; nj < d_es_nry; nj++){ 
+              for( nk = -d_es_nrz; nk < d_es_nrz; nk++){
+                // terms with non-negligible contributions must lie inside ellipse 
+                // (perpendicular to the previous ellipse)
+               if ((ni*ni + nj*nj + nk*nk ) > 0.0)
+                   {
+                       k2 = 4*CUDA_PI*CUDA_PI*(ni*ni/d_Lx_per/d_Lx_per + nj*nj/d_Ly_per/d_Ly_per + nk*nk/d_Lz_per/d_Lz_per);
+                       kx = 2*CUDA_PI*(ni*x/d_Lx_per + nj*y/d_Ly_per + nk*z/d_Lz_per);
+                       t_rec += (4*CUDA_PI/(k2*d_Ln+1.0E-99)) * exp(-k2/(4*d_ewald_alpha*d_ewald_alpha)) * cos(kx);
+                     
+                   }
+              
+              }
+           }
+         }
+
+ grid[id] += t_rec;
+   xx += inc;
+     }
+    // printf("ewrec %e\n",grid[id]);
+ 
+   }
+ 
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Init_Kernel.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Init_Kernel.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Init_Kernel.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,13 @@
+#include "gr_bh_gpu_Init_Kernel.cuh"
+
+
+__launch_bounds__(THREADS0, FACTOR3)
+__global__ void InitializationKernel()
+{  
+  maxdepthd = 1;
+  blkcntd = 0;
+ }
+
+
+
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Extrapolate_Kernel.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Extrapolate_Kernel.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Extrapolate_Kernel.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,68 @@
+#include "gr_bh_gpu_Ewald_Extrapolate_Kernel.cuh"
+
+
+
+
+__device__ double atomicAdd(double* address, double val)
+{
+    unsigned long long int* address_as_ull =
+                                          (unsigned long long int*)address;
+    unsigned long long int old = *address_as_ull, assumed;
+    do {
+        assumed = old;
+        old = atomicCAS(address_as_ull, assumed, 
+                        __double_as_longlong(val + 
+                        __longlong_as_double(assumed)));
+    } while (assumed != old);
+    return __longlong_as_double(old);
+}
+
+
+__global__ void ewald_extrapolate_real(double x,double y, double z,double *cell)
+{
+int ni,nj,nk;
+double xni,yni,zni,rni;
+double t_cell;
+  
+     for( ni = -d_es_nrx + (threadIdx.x + blockIdx.x * blockDim.x); ni < d_es_nrx; ni += blockDim.x * gridDim.x ){ 
+        t_cell = 0.0;  
+      for( nj = -d_es_nry; nj < d_es_nry; nj++){
+        for( nk = -d_es_nrz; nk < d_es_nrz; nk++){
+          xni = x + ni * d_Lx_per;
+          yni = y + nj * d_Ly_per;
+          zni = z + nk * d_Lz_per;
+          rni = sqrt(xni*xni + yni*yni + zni*zni);
+          
+         
+           t_cell +=  erfc(d_ewald_alpha*rni) / (rni + 1.0E-99);
+          
+         }
+       }
+atomicAdd(cell,t_cell);
+      }
+} 
+
+
+__global__ void ewald_extrapolate_rec(double x,double y, double z,double *cell)
+{
+double k2,kx;
+int ni,nj,nk;    
+double t_cell;   
+     for( ni = -d_es_nrx + (threadIdx.x + blockIdx.x * blockDim.x); ni < d_es_nrx; ni +=blockDim.x * gridDim.x ){
+         t_cell = 0.0;
+      for( nj = -d_es_nry; nj < d_es_nry; nj++){
+        for( nk = -d_es_nrz; nk < d_es_nrz; nk++){
+            if ((ni*ni + nj*nj + nk*nk ) > 0.0)
+                   {
+                       k2 = 4*CUDA_PI*CUDA_PI*(ni*ni/d_Lx_per/d_Lx_per + nj*nj/d_Ly_per/d_Ly_per + nk*nk/d_Lz_per/d_Lz_per);
+                       kx = 2*CUDA_PI*(ni*x/d_Lx_per + nj*y/d_Ly_per + nk*z/d_Lz_per);
+                       t_cell += (4*CUDA_PI/(k2*d_Ln+1.0E-99)) * exp(-k2/(4*d_ewald_alpha*d_ewald_alpha)) * cos(kx);
+                     
+                   }
+     }
+   }
+
+atomicAdd(cell,t_cell);
+  }
+
+}
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Build_Init_Wrap.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Build_Init_Wrap.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Build_Init_Wrap.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,384 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+
+// GENERAL TREE BUILDING
+#include "gr_bh_gpu_Init_Kernel.cuh"
+#include "gr_bh_gpu_MakeRootKernel.cuh"
+#include "gr_bh_gpu_Summarization_Kernel.cuh"
+#include "gr_bh_gpu_TreeBuilding_Kernel.cuh"
+
+
+extern "C" {
+void gr_bh_gpu_Essentials_Build_Init_Wrap_(int *device_ID,int *gpu_blocks,
+                                              int *bodies_count, //
+                                              int *bodies_node_count,
+                                              double *angle,
+                                              double *domain_diagonal,
+                                              double **x_data, //
+                                              double **y_data, //
+                                              double **z_data, //
+                                              double **m_data,
+                                              double *maxx, double *minx, double *maxy, double *miny, double *maxz, double *minz,
+					      int *depth,
+                                              int *periodics,
+                                              double *roots,
+                                              int *gr_meshNumProcs,
+                                              double *memory_timings, 
+					      double *kernel_timings, 
+                                              double *tx, double *ty, double *tz, double *tm,
+					      int **essential_nodes,int **essential_counter,
+					      int **all_ints, double **all_doubles)//
+    {
+
+	double *root_maxxl;
+	double *root_maxyl;
+	double *root_maxzl;        
+	double *root_minxl;          
+	double *root_minyl;             
+	double *root_minzl;
+	double *xpos_device;//  = all_doubles + rsize + 6;
+	double *ypos_device;//  = all_doubles + rsize + 6 + (nnodes+1);
+	double *zpos_device;//  = all_doubles + rsize + 6 + (2*(nnodes+1));
+	double *mass_device;//  = all_doubles + rsize + 6 + (3*(nnodes+1));
+	double *radii_device;// = all_doubles + rsize + 6 + (4*(nnodes+1));
+        double *delta_device;
+	double *roots_device;
+
+	int *tree_device; 
+	int *essential_counter_device;
+	int *essential_nodes_devive; 
+	int *res_device;
+
+        double *x_cells = *x_data;
+	double *y_cells = *y_data;
+	double *z_cells = *z_data;
+	double *m_cells = *m_data;
+   	int nnodes = *bodies_node_count;
+   	int nbodies = *bodies_count;
+	double  itolsq = 1.0 /((*angle) * (*angle)) ;/// (0.5);
+        double bytes = 0;
+	int blocks = *gpu_blocks;
+ 	int n_blocks = blocks * FACTOR8;
+	int rsize = *gr_meshNumProcs * 4;
+	int periodicity = *periodics;
+	int dp;
+	cudaEvent_t start, stop;
+	float time;
+
+        cudaSetDevice(*device_ID);
+        CudaTest((char*)"gr_bh_gpu_Essentials_Build_Init_Wrap.cu: Could not set device ID");
+
+
+
+	cudaEventCreate(&start);
+	cudaEventCreate(&stop);
+	cudaEventRecord(start, 0);
+
+//malloc
+    
+	
+
+        cudaMallocHost((void **)essential_counter, n_blocks * sizeof(int));
+	CudaTest((char*)"gr_bh_gpu_Essentials_Build_Init_Wrap.cu: couldn't allocate essential_counter");
+
+	cudaMallocHost((void **)essential_nodes, sizeof(int)* (nnodes+1));
+	CudaTest((char*)"gr_bh_gpu_Essentials_Build_Init_Wrap.cu: couldn't allocate essentials array");
+
+
+
+        cudaMalloc((void **)all_ints, sizeof(int) * ( ((nnodes+1) * 8) + n_blocks +  (nnodes+1) + nbodies)   ); 
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: couldn't allocate all_ints");
+
+
+
+        tree_device              = (*all_ints) + 0; 
+        essential_counter_device = (*all_ints) + ((nnodes+1) * 8);
+	essential_nodes_devive   = (*all_ints) + ((nnodes+1) * 8) + n_blocks;
+        res_device               = (*all_ints) + ((nnodes+1) * 8) + n_blocks + (nnodes+1);
+ 
+
+
+        cudaMalloc((void **) all_doubles, sizeof(double) *( rsize + 6 + ((nnodes+1)*6)) );
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: couldn't allocate all");
+
+
+	
+	roots_device = (*all_doubles) + 0;
+	root_maxxl   = (*all_doubles) + rsize + 0;
+	root_maxyl   = (*all_doubles) + rsize + 1;
+	root_maxzl   = (*all_doubles) + rsize + 2;       
+	root_minxl   = (*all_doubles) + rsize + 3;        
+	root_minyl   = (*all_doubles) + rsize + 4;            
+	root_minzl   = (*all_doubles) + rsize + 5;
+	xpos_device  = (*all_doubles) + rsize + 6;
+	ypos_device  = (*all_doubles) + rsize + 6 + (nnodes+1);
+	zpos_device  = (*all_doubles) + rsize + 6 + (2*(nnodes+1));
+	mass_device  = (*all_doubles) + rsize + 6 + (3*(nnodes+1));
+	radii_device = (*all_doubles) + rsize + 6 + (4*(nnodes+1));
+        delta_device = (*all_doubles) + rsize + 6 + (5*(nnodes+1));
+
+
+	cudaEventRecord(stop, 0);
+	cudaEventSynchronize(start); 
+	cudaEventSynchronize(stop);
+	cudaEventElapsedTime(&time, start, stop);
+        memory_timings[0] += time;
+
+
+
+
+
+
+
+//cpytoSYMB
+	cudaEventRecord(start, 0);
+
+	cudaMemcpyToSymbol(periodic, &periodicity, sizeof(int)); 
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: periodics copy to Symbol failed");
+        bytes += 1 * sizeof(int);
+
+	cudaMemcpyToSymbol(nbodiesd, &nbodies, sizeof(int));
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: nbody copy to Symbol failed");
+        bytes += 1 * sizeof(int);
+
+	cudaMemcpyToSymbol(nnodesd, &nnodes, sizeof(int));
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: nnode copy to Symbol failed");
+ 	bytes += 1 * sizeof(int);
+
+	cudaMemcpyToSymbol(itolsqd, &itolsq, sizeof(double));
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: itolsq copy to Symbolfailed");
+	bytes += 1 * sizeof(double);
+
+	cudaMemcpyToSymbol(domdiag, domain_diagonal, sizeof(double));
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: domain copy to Symbol failed");
+        bytes += 1 * sizeof(double);
+
+
+
+	cudaMemcpyToSymbol(childd,&tree_device, sizeof(void*));
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: childl copy to Symbol failed");
+	
+
+
+	cudaMemcpyToSymbol(posxd, &xpos_device, sizeof(void*));
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: posx copy to Symbol failed");
+	
+	cudaMemcpyToSymbol(posyd, &ypos_device, sizeof(void*));
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: posy copy to Symbol failed");    
+	
+
+	cudaMemcpyToSymbol(poszd, &zpos_device, sizeof(void*));
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: posz copy to Symbol failed");    
+	
+
+	cudaMemcpyToSymbol(massd,&mass_device, sizeof(void*));
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: mass copy to Symbol failed");
+	
+
+	cudaMemcpyToSymbol(posrd, &radii_device, sizeof(void*)); 
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: radii copy to Symbol failed");
+
+
+	cudaMemcpyToSymbol(delta, &delta_device, sizeof(void*));
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: delta copy to Symbol failed");
+
+
+
+
+//cpy
+	cudaMemcpy(roots_device,roots, sizeof(double) * (rsize ), cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: rootsl copy to device failed");
+	bytes += rsize * sizeof(double);
+
+	cudaMemcpy(root_maxxl, maxx, sizeof(double), cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: maxx copy to device failed"); 
+	bytes += 1 * sizeof(double);
+
+	cudaMemcpy(root_maxyl, maxy, sizeof(double) , cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: maxy copy to device failed"); 
+	bytes += 1 * sizeof(double);
+
+	cudaMemcpy(root_maxzl, maxz, sizeof(double) , cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: maxz copy to device failed"); 
+	bytes += 1 * sizeof(double);
+
+	cudaMemcpy(root_minxl, minx, sizeof(double), cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: minx copy to device failed"); ;
+	bytes += 1 * sizeof(double);
+
+	cudaMemcpy(root_minyl, miny, sizeof(double) , cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: miny copy to device failed"); 
+	bytes += 1 * sizeof(double);
+
+	cudaMemcpy(root_minzl, minz, sizeof(double) , cudaMemcpyHostToDevice),
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: minz copy to device failed"); 
+	bytes += 1 * sizeof(double);
+
+	cudaMemcpy(xpos_device,x_cells, sizeof(double) * (nbodies), cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: posx copy to device failed");
+	bytes += nbodies * sizeof(double);
+
+	cudaMemcpy(ypos_device,y_cells, sizeof(double) * (nbodies), cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: posy copy to device failed");
+	bytes += nbodies * sizeof(double);
+
+	cudaMemcpy(zpos_device,z_cells, sizeof(double) * (nbodies), cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: posz copy to device failed");
+	bytes += nbodies * sizeof(double);
+
+	cudaMemcpy(mass_device,m_cells, sizeof(double) * (nbodies), cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: mass copy to device failed");
+	bytes += nbodies * sizeof(double);
+
+        cudaEventRecord(stop, 0);
+	cudaEventSynchronize(start); 
+	cudaEventSynchronize(stop);
+	cudaEventElapsedTime(&time, start, stop);
+        memory_timings[1] += time;
+        memory_timings[2] += bytes;
+        bytes = 0;
+//memset
+        cudaEventRecord(start, 0);
+
+	cudaMemset((void*) delta_device, 0, sizeof(double) * (nnodes+1));
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: couldn't initialize delta");
+
+
+
+
+
+	InitializationKernel<<<1, 1>>>();
+    	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: kernel 0 launch failed");
+    
+    	MakeRootKernel<<<1,1>>>(root_minxl,root_maxxl,root_minyl,root_maxyl,root_minzl,root_maxzl);
+    	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: kernel 1 launch failed");
+
+
+    	cudaEventRecord(stop, 0);
+    	cudaEventSynchronize(start); 
+    	cudaEventSynchronize(stop);
+    	cudaEventElapsedTime(&time, start, stop);
+
+        kernel_timings[0] += time; 
+
+        cudaEventRecord(start, 0);
+
+
+    	TreeBuildingKernel<<<blocks * FACTOR2, THREADS2>>>();
+    	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: kernel treebuild launch failed");
+       
+    	cudaEventRecord(stop, 0);
+    	cudaEventSynchronize(start); 
+    	cudaEventSynchronize(stop);
+    	cudaEventElapsedTime(&time, start, stop);
+
+        kernel_timings[1] += time;
+
+    	cudaEventRecord(start, 0);
+
+    	SummarizationKernel<<<blocks * FACTOR3, THREADS3>>>();
+    	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: kernel summit launch failed");
+   
+    	cudaEventRecord(stop, 0);
+   	cudaEventSynchronize(start); 
+   	cudaEventSynchronize(stop);
+   	cudaEventElapsedTime(&time, start, stop);
+
+     	kernel_timings[2] += time; 
+
+    	cudaEventRecord(start, 0);
+
+
+
+        bytes = 0;
+    	cudaMemcpyFromSymbol(&dp,maxdepthd, sizeof(int),0, cudaMemcpyDeviceToHost);
+    	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: maxdepthd copy to host failed");
+	bytes += 1 * sizeof(int);
+    	*depth=dp;
+
+
+      
+        cudaMemcpy(tx,xpos_device, sizeof(double) *  (nnodes+1), cudaMemcpyDeviceToHost);
+    	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: bodies_x_coords copy to host failed");
+	bytes += (nnodes+1) * sizeof(double);
+
+
+    	cudaMemcpy(ty,ypos_device, sizeof(double) *   (nnodes+1), cudaMemcpyDeviceToHost);
+    	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: bodies_y_coords copy to host failed");
+	bytes += (nnodes+1) * sizeof(double);
+
+
+	cudaMemcpy(tz,zpos_device, sizeof(double) *  (nnodes+1), cudaMemcpyDeviceToHost);
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: bodies_z_coords to host failed");
+	bytes += (nnodes+1) * sizeof(double);
+
+
+	cudaMemcpy(tm,mass_device, sizeof(double) * (nnodes+1), cudaMemcpyDeviceToHost);
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Build_Init_Wrap: bodies_mass copy to host failed");
+	bytes += (nnodes+1) * sizeof(double);
+
+
+
+	cudaEventRecord(stop, 0);
+	cudaEventSynchronize(start); 
+	cudaEventSynchronize(stop);
+	cudaEventElapsedTime(&time, start, stop);
+
+	memory_timings[3] += time;
+	memory_timings[4] += bytes;
+        bytes = 0;
+
+	cudaEventRecord(start, 0);
+/*
+    	cudaFree(root_maxxl);
+    	cudaFree(root_maxyl);
+    	cudaFree(root_maxzl);
+    	cudaFree(root_minxl);
+    	cudaFree(root_minyl);
+   	cudaFree(root_minzl);
+ */
+
+	cudaEventRecord(stop, 0);
+	cudaEventSynchronize(start); 
+	cudaEventSynchronize(stop);
+	cudaEventElapsedTime(&time, start, stop);
+
+	memory_timings[0] += time;
+	cudaEventDestroy(start);
+	cudaEventDestroy(stop);  
+
+    }
+}
+
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_ReAllocBlocks.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_ReAllocBlocks.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_ReAllocBlocks.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,50 @@
+!!****if* source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_ReAllocBlocks
+!!
+!! NAME
+!!
+!!  gr_bh_gpu_ReAllocBlocks
+!!
+!! SYNOPSIS
+!!
+!!  call gr_bh_gpu_ReAllocBlocks()
+!!
+!! DESCRIPTION
+!!
+!! Resize the cell data containers to hold the data for imported foreign subtrees.
+!!
+!! ARGUMENTS
+!!
+!!   No arguments
+!!
+!! AUTOGENROBODOC
+!!
+!!
+!!***
+
+
+subroutine gr_bh_gpu_ReAllocBlocks()
+ use Grid_data, ONLY : gr_meshMe,gr_meshNumProcs, gr_meshComm
+ use Timers_interface, ONLY : Timers_start, Timers_stop
+ use Driver_interface, ONLY : Driver_abortFlash
+
+ use gr_bh_gpu_data, ONLY :   gr_bh_gpu_essentials_count,gr_bh_gpu_local_cell_count,gr_bh_gpu_total_cell_count,&
+			     gr_bh_gpu_x_data,gr_bh_gpu_y_data,gr_bh_gpu_z_data,gr_bh_gpu_m_data,gr_bh_gpu_p_data,gr_bh_gpu_pinned_cells
+ use gr_bh_gpu_Interface , ONLY:   gr_bh_gpu_Essentials_Reallocate_CellData                         
+
+
+implicit none
+#include "Flash_mpi.h"
+#include "Flash.h"
+ 
+
+ gr_bh_gpu_total_cell_count=gr_bh_gpu_local_cell_count+(gr_bh_gpu_essentials_count/4)
+
+if(gr_bh_gpu_local_cell_count .gt. 0) then
+ call gr_bh_gpu_Essentials_Reallocate_CellData( gr_bh_gpu_x_data,gr_bh_gpu_y_data,gr_bh_gpu_z_data,gr_bh_gpu_m_data,&
+						gr_bh_gpu_local_cell_count,gr_bh_gpu_total_cell_count,gr_bh_gpu_pinned_cells,gr_meshMe)
+endif
+
+return
+
+end subroutine  gr_bh_gpu_ReAllocBlocks
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_GetGridData.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_GetGridData.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_GetGridData.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,190 @@
+!!****if* source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_GetGridData
+!!
+!! NAME
+!!  gr_bh_gpu_GetGridData
+!!
+!! SYNOPSIS
+!!
+!!  use gr_bh_gpu_GetGridData(Integer (IN)::blockid, 
+!!                            Integer (IN)::num_cells,
+!!                            Integer (IN)::counter,
+!!                            Integer (IN)::idensvar,
+!!                            Real (INOUT):: minx,
+!!                            Real (INOUT):: maxx,
+!!                            Real (INOUT):: miny,
+!!                            Real (INOUT):: maxy,
+!!                            Real (INOUT):: minz,
+!!                            Real (INOUT):: maxz,
+!!                            Real (OUT)  :: cx,
+!!                            Real (OUT)  :: cy,
+!!                            Real (OUT)  :: cz,
+!!                            Real (OUT)  :: cm)
+!!
+!! DESCRIPTION
+!!
+!!  Reads position and mass data for all cells in the paramesh leaf block with blockId "blockid". 
+!!  Calculates the center of mass for the given block (cx,cy,cz)
+!!  Calculates min and max position values.
+!!  Stores the position and mass data for the cells in pinned or paged memory allocated in "C".
+!!  
+!! ARGUMENTS
+!!  
+!!  blockid         -Paramesh blockid odf the Leaf block that will be processed
+!!
+!!  num_cells       - unused.... 
+!!
+!!  counter         - Number of the leafblock (!= blockid)
+!!
+!!  idensvar        - The index for the density variable
+!!
+!! 
+!!
+!!  minx,maxx,..minz,maxz - Minimum and maximum values of positional data. 
+!!
+!!  
+!!
+!!  cx,..,cm  - The postion and mass of the center of mass of this block.
+!!   
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!!  
+!!
+!!***
+
+subroutine gr_bh_gpu_GetGridData(blockid,num_cells,counter,idensvar,minx,maxx,miny,maxy,minz,maxz,cx,cy,cz,cm)
+
+ use Grid_interface, ONLY :  Grid_getBlkRefineLevel, Grid_getCellCoords, Grid_getBlkIndexLimits,&
+   Grid_getLocalNumBlks, Grid_getListOfBlocks, Grid_getBlkPhysicalSize, Grid_getMaxCommonRefinement, Grid_getBlkPtr, &
+   Grid_getDomainBoundBox, Grid_getBlkCenterCoords
+ use Grid_data, ONLY : gr_meshMe
+ use gr_bh_gpu_data, ONLY : gr_bh_gpu_x_data,gr_bh_gpu_y_data,gr_bh_gpu_z_data,gr_bh_gpu_m_data,gr_bh_gpu_p_data
+ use gr_bh_gpu_Interface, ONLY: gr_bh_gpu_GetGridData_SetValue
+ implicit none
+
+#include "Flash.h"
+#include "Flash_mpi.h"
+#include "constants.h"
+integer, intent(in) :: num_cells
+integer, intent(in) :: blockid
+integer, intent(in) :: idensvar
+integer, intent(in) :: counter
+
+
+real, intent(inout)               :: minx
+real, intent(inout)               :: maxx
+real, intent(inout)               :: miny
+real, intent(inout)               :: maxy
+real, intent(inout)               :: minz
+real, intent(inout)               :: maxz
+
+real, intent(out)               :: cx
+real, intent(out)               :: cy
+real, intent(out)               :: cz
+real, intent(out)               :: cm
+
+
+
+real            :: xi(1:NXB)
+real            :: yi(1:NYB)
+real            :: zi(1:NZB)
+
+integer, dimension(LOW:HIGH,MDIM) :: blkLimits,blkLimitsGC
+
+integer               :: i,j,k,cellid
+
+real :: dvol, massd
+
+real, DIMENSION(:,:,:,:), POINTER :: solnData
+
+real :: point(MDIM)
+
+
+
+
+ cx = 0
+ cy = 0
+ cz = 0 
+ cm = 0
+      
+ call Grid_getBlkPtr(blockid,solnData,CENTER)
+ call Grid_getBlkIndexLimits(blockid,blkLimits,blkLimitsGC)
+
+
+
+           call Grid_getCellCoords(IAXIS, blockid, CENTER, .false., xi, NXB)
+           call Grid_getCellCoords(JAXIS, blockid, CENTER, .false., yi, NYB)
+           call Grid_getCellCoords(KAXIS, blockid, CENTER, .false., zi, NZB)
+
+              
+
+               do k = 1,NZB
+                 point(3) = k
+                 do j = 1,NYB
+                    point(2) = j
+                    do i = 1,NXB
+                       point(1) = i
+                      
+
+                     if(NXB==8)then
+                     if (j <= 4 .and. i <= 4) then
+                        cellid = (i-1) + ((j-1)*4) + ((k-1)*4*4) + ((counter)*(NXB*NYB*NZB))
+                  
+                      else  if (j > 4 .and. i > 4) then
+                       cellid = (i-5) + ((j-5)*4) + ((k-1)*4*4) + 128 + ((counter)*(NXB*NYB*NZB))
+                    
+                      else if ( j > 4 .and. i <= 4) then
+                        cellid = (i-1) + ((j-5)*4) + ((k-1)*4*4) + 256 + ((counter)*(NXB*NYB*NZB))
+                 
+                     else if ( j <= 4 .and. i > 4) then
+                        cellid = (i-5) + ((j-1)*4) + ((k-1)*4*4) + 384 + ((counter)*(NXB*NYB*NZB))
+                    end if
+                      else
+                       cellid = (i-1) + ((j-1)*NYB) + ((k-1)*NYB*NXB) + ((counter)*(NXB*NYB*NZB))
+                       endif
+                     !  x(cellid ) = xi(i)!8*8*8 size
+                     !  y((cellid))= yi(j)
+                     !  z((cellid))= zi(k)
+                       
+                       
+                       
+                       call Grid_getSingleCellVol(blockid, INTERIOR, point, dvol)
+                       massd = solnData(idensvar, i+blkLimits(LOW,IAXIS)-1,&
+                                                           j+blkLimits(LOW,JAXIS)-1,&
+                                                           k+blkLimits(LOW,KAXIS)-1)*dvol
+                      
+                
+
+              call gr_bh_gpu_GetGridData_SetValue(gr_bh_gpu_x_data,gr_bh_gpu_y_data,gr_bh_gpu_z_data,gr_bh_gpu_m_data,&
+				 		xi(i), yi(j) , zi(k) , massd ,cellid)
+
+                cm = cm + massd
+                cx = cx +  xi(i) * massd
+                cy = cy +  yi(j)* massd
+                cz = cz +  zi(k) * massd
+
+                       maxx = max(maxx,xi(i))
+                       minx = min(minx,xi(i))
+                       
+                       maxy = max(maxy,yi(j))
+                       miny = min(miny,yi(j))
+
+                       maxz = max(maxz,zi(k))
+                       minz = min(minz,zi(k))
+                
+             
+                    end do !k
+                 end do !j
+               end do !i
+       
+             
+
+
+ call Grid_releaseBlkPtr(blockid,solnData,CENTER)
+      
+
+return
+end subroutine gr_bh_gpu_GetGridData
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Gather_Allocate_CellData.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Gather_Allocate_CellData.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Gather_Allocate_CellData.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,82 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+
+extern "C" {
+void gr_bh_gpu_Gather_Allocate_CellData_(double **x_data, 
+				   double **y_data, 
+				   double **z_data, 
+				   double **m_data,
+				   double **pot_data,
+				   int *cell_count,bool *pinned, int *rank)//
+    {
+
+
+
+        if(*pinned){
+
+#ifdef GPU_BH_DEBUG
+printf(" Rank %d will allocate %d * 4 doubles in pinned memory for cell data\n",*rank, *cell_count);
+#endif
+        cudaMallocHost((void**) x_data, sizeof(double) * *cell_count);
+	CudaTest((char *)"gr_bh_gpu_Gather_Allocate_CellData.cu : Allocating pinned Memory for x FAILED");
+
+	cudaMallocHost((void**) y_data, sizeof(double) * *cell_count);
+	CudaTest((char *)"gr_bh_gpu_Gather_Allocate_CellData.cu : Allocating pinned Memory for y FAILED");
+
+	cudaMallocHost((void**) z_data, sizeof(double) * *cell_count);
+	CudaTest((char *)"gr_bh_gpu_Gather_Allocate_CellData.cu : Allocating pinned Memory for z FAILED");
+
+	cudaMallocHost((void**) m_data, sizeof(double) * *cell_count);
+	CudaTest((char *)"gr_bh_gpu_Gather_Allocate_CellData.cu : Allocating pinned Memory for m FAILED");
+
+	cudaMallocHost((void**) pot_data, sizeof(double) * *cell_count);
+	CudaTest((char *)"gr_bh_gpu_Gather_Allocate_CellData.cu : Allocating pinned Memory for pot FAILED");
+        }
+        else{
+
+
+#ifdef GPU_BH_DEBUG
+printf(" Rank %d will allocate %d * 4 doubles in pageable memory for cell data\n",*rank, *cell_count);
+#endif
+
+                *x_data = (double *) malloc(sizeof(double) * *cell_count);
+ 		*y_data = (double *) malloc(sizeof(double) * *cell_count);
+ 		*z_data = (double *) malloc(sizeof(double) * *cell_count);
+ 		*m_data = (double *) malloc(sizeof(double) * *cell_count);
+ 		*pot_data = (double *) malloc(sizeof(double) * *cell_count);
+
+            }
+
+    }
+}
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Periodic_InterpolateInField.cuh
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Periodic_InterpolateInField.cuh	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Periodic_InterpolateInField.cuh	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,9 @@
+
+#ifndef __PER_INT_CUH__
+#define __PER_INT_CUH__
+
+
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+__device__ __forceinline__ double interp_ewald_noref(double x, double y, double z);
+
+#endif
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CalcEwaldField.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CalcEwaldField.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CalcEwaldField.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,115 @@
+!!****if* BHTree/GPU/gr_bh_gpu_CalcEwaldField
+!!
+!! NAME
+!!
+!!  gr_bh_gpu_CalcEwaldField
+!!
+!! SYNOPSIS
+!!
+!!  call gr_bh_gpu_CalcEwaldField()
+!!
+!! DESCRIPTION
+!!
+!! Calculate the Ewald field on the GPU-device.
+!!
+!! ARGUMENTS
+!!
+!!   No arguments
+!!
+!! AUTOGENROBODOC
+!!
+!!
+!!***
+
+subroutine gr_bh_gpu_CalcEwaldField()
+use RuntimeParameters_interface, ONLY : RuntimeParameters_get
+use gr_bh_gpu_data,ONLY :  gr_bh_gpu_Lx, gr_bh_gpu_Ly ,gr_bh_gpu_Lz,&
+                           gr_bh_gpu_ewald_xmax, gr_bh_gpu_ewald_ymax, gr_bh_gpu_ewald_zmax,&
+                           gr_bh_gpu_EwaldSize,&
+                           gr_bh_gpu_EwaldFieldNx, gr_bh_gpu_EwaldFieldNy, gr_bh_gpu_EwaldFieldNz,&
+                           gr_bh_gpu_EwaldSeriesN,&
+                           gr_bh_gpu_EwaldSIsoFac,&
+                           gr_bh_gpu_EwaldField,&
+                           gpu_comm,gpu_rank,gr_bh_gpu_mcs, gr_bh_gpu_periodic
+   
+  use gr_bh_gpu_Interface, ONLY: gr_bh_gpu_Ewald_Init, gr_bh_gpu_Ewald_MakeField                      
+
+  use Grid_interface, ONLY : Grid_getMinCellSizes
+  use Driver_interface, ONLY : Driver_abortFlash
+
+  implicit none
+#include "Flash.h"
+#include "Flash_mpi.h"
+#include "constants.h"
+
+  integer :: i, j, k, l, ierr, nref,id
+  integer :: ewald_periodicity
+ 
+  real :: x, y, z,dum,ewald_xmax,ewald_ymax,ewald_zmax
+
+ 
+  
+
+
+
+ ! dimensions of one grid-cell of the largest Ewald field,
+  ! used in grv_bhEwald.F90
+  gr_bh_gpu_ewald_xmax = gr_bh_gpu_Lx
+  gr_bh_gpu_ewald_ymax = gr_bh_gpu_Ly
+  gr_bh_gpu_ewald_zmax = gr_bh_gpu_Lz
+
+ 
+
+
+ call gr_bh_gpu_Ewald_Init(gr_bh_gpu_EwaldFieldNx,gr_bh_gpu_EwaldFieldNy, gr_bh_gpu_EwaldFieldNz,&
+                              gr_bh_gpu_EwaldSeriesN,&
+                              gr_bh_gpu_EwaldSIsoFac,&
+                              gr_bh_gpu_Lx,gr_bh_gpu_Ly,gr_bh_gpu_Lz,&
+                              gr_bh_gpu_ewald_xmax,gr_bh_gpu_ewald_ymax ,gr_bh_gpu_ewald_zmax, gr_bh_gpu_periodic )
+
+ if(gpu_rank .eq. 0) then
+
+     call gr_bh_gpu_Ewald_MakeField(gr_bh_gpu_EwaldFieldNx,gr_bh_gpu_EwaldFieldNy, gr_bh_gpu_EwaldFieldNz,&
+                     gr_bh_gpu_EwaldField, &
+                     gr_bh_gpu_mcs(IAXIS),gr_bh_gpu_mcs(JAXIS),gr_bh_gpu_mcs(KAXIS),&
+                     gr_bh_gpu_ewald_xmax,gr_bh_gpu_ewald_ymax ,gr_bh_gpu_ewald_zmax, gr_bh_gpu_periodic)
+
+
+
+
+
+
+
+
+  endif
+!Deliver Grid to other pids in gpu comm grp
+call MPI_BCAST (gr_bh_gpu_EwaldField,gr_bh_gpu_EwaldSize,FLASH_REAL,0,gpu_comm,ierr)
+
+
+   
+   
+ 
+
+
+
+!if(gr_meshMe .eq. 0) then
+
+ !open(unit=53, file="ewald_field_2",status="replace")
+  !    do k = 0,grv_bh_gpu_EwaldFieldNz-1
+  !      z = (k * 0.5 * grv_bh_gpu_Lz) / grv_bh_gpu_EwaldFieldNz
+  !      do j = 0,grv_bh_gpu_EwaldFieldNy-1
+  !        y = (j * 0.5 * grv_bh_gpu_Ly) / grv_bh_gpu_EwaldFieldNy
+  !        do i = 0,grv_bh_gpu_EwaldFieldNx-1
+  !          x = (i * 0.5 * grv_bh_gpu_Lx) / grv_bh_gpu_EwaldFieldNx
+           ! write(53,'(5(e17.10, 2x))') x, y, z, grid(i + j * grv_bh_gpu_EwaldFieldNx + k * grv_bh_gpu_EwaldFieldNx*grv_bh_gpu_EwaldFieldNy) &
+   !         , 1.0/(sqrt(x*x+y*y+z*z)+1d-99)
+   !       enddo
+   !       write(53,*)
+   !     enddo
+   !     write(53,*)
+   !   enddo
+   !   close(unit=53)
+!endif
+
+return
+end subroutine gr_bh_gpu_CalcEwaldField
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CalcPot.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CalcPot.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CalcPot.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,218 @@
+!!****if* BHTree/GPU/gr_bh_gpu_CalcPot
+!!
+!! NAME
+!!
+!!  gr_bh_gpu_CalcPot
+!!
+!! SYNOPSIS
+!!
+!!  call gr_bh_gpu_CalcPot()
+!!
+!! DESCRIPTION
+!!
+!! Calculate the gravitational potential on the GPU-device.
+!! 
+!! ARGUMENTS
+!!
+!!   No arguments
+!!
+!!
+!!
+!!***
+
+
+
+subroutine gr_bh_gpu_CalcPot()
+   use Driver_interface, ONLY : Driver_abortFlash
+   use Grid_data, ONLY : gr_meshMe,gr_meshNumProcs, gr_meshComm
+   use gr_bh_gpu_data, ONLY: gr_bh_gpu_Mac_Angle,gr_bh_gpu_global_domdiagsq,&
+                             gr_bh_gpu_GravFac,gr_bh_gpu_EwaldField,gr_bh_gpu_EwaldSize,&
+                             gr_bh_gpu_MAX_GPU_CELLS, &
+                             gr_bh_gpu_xmin_domain,gr_bh_gpu_ymin_domain,gr_bh_gpu_zmin_domain,&
+                             gr_bh_gpu_xmax_domain,gr_bh_gpu_ymax_domain,gr_bh_gpu_zmax_domain,&
+                             gr_bh_gpu_roots_own,gpu_blocks, gr_bh_gpu_periodic,&
+                             potential_memory_timings,&
+			     potential_kernel_timings,gr_bh_gpu_p_data,&
+			     gr_bh_gpu_essentials_count,gr_bh_gpu_local_cell_count,&
+			     gr_bh_gpu_total_cell_count,gr_bh_gpu_node_count,&
+ 			     gr_bh_gpu_x_data,gr_bh_gpu_y_data,gr_bh_gpu_z_data,&
+			     gr_bh_gpu_m_data,gr_bh_gpu_p_data,device_ID
+
+                             
+
+   use gr_bh_gpu_Interface, ONLY: gr_bh_gpu_Potential_Init,gr_bh_gpu_Potential_Subset,&
+                                  gr_bh_gpu_Potential_All,gr_bh_gpu_Potential_Copy,&
+				  gr_bh_gpu_Potential_Local,gr_bh_gpu_Potential_Clean
+
+   use, intrinsic :: iso_c_binding
+   implicit none
+
+
+   integer :: non_local_cell_count, local_cell_count,begin_count,end_count,current_size,other
+   real ::  itolsq
+   type( c_ptr ) :: xd, yd, zd, gpotd, maxxd, maxyd, maxzd, minxd, minyd, minzd, e_gridd
+   logical :: run
+   run = .true.
+
+   itolsq= 1.0 /((gr_bh_gpu_Mac_Angle) * (gr_bh_gpu_Mac_Angle)) ; 
+   other = 0
+     
+     if(gr_bh_gpu_local_cell_count .gt. 1)then
+         non_local_cell_count = gr_bh_gpu_total_cell_count - gr_bh_gpu_local_cell_count
+         current_size = non_local_cell_count
+         local_cell_count = gr_bh_gpu_local_cell_count
+
+
+!!===================GPU==GPU==GPU=====================================
+
+     !allocate and copy general data to GPU
+         call gr_bh_gpu_Potential_Init(device_ID,itolsq,&
+                                         local_cell_count,  &
+                                         gr_bh_gpu_global_domdiagsq,&
+                                         gr_bh_gpu_xmin_domain,&
+                                         gr_bh_gpu_ymin_domain,&
+                                         gr_bh_gpu_zmin_domain,&
+                                         gr_bh_gpu_xmax_domain,&
+                                         gr_bh_gpu_ymax_domain,&
+                                         gr_bh_gpu_zmax_domain, &      
+                                         gr_bh_gpu_EwaldField,&
+                                         gr_bh_gpu_EwaldSize,&
+                                         gr_bh_gpu_periodic,&
+                                         potential_memory_timings,&
+                                         gpotd,&
+                                         e_gridd,&
+                                         maxxd,&
+                                         maxyd,&
+                                         maxzd,&
+                                         minxd,&
+                                         minyd,&
+                                         minzd)
+
+!=========================================SPLIT FORCE CALCULATION==========================================!
+       if(gr_bh_gpu_total_cell_count .ge. gr_bh_gpu_MAX_GPU_CELLS)then 
+         !Calculate local potential first
+          other = 1
+          call gr_bh_gpu_Potential_Local(device_ID,gpu_blocks,&
+                                          local_cell_count,  & 
+			            gr_bh_gpu_x_data,&
+				    gr_bh_gpu_y_data,&
+			            gr_bh_gpu_z_data,&
+				    gr_bh_gpu_m_data,&
+                                          potential_memory_timings,&
+                                          potential_kernel_timings,&
+                                          maxxd,&
+                                          maxyd,&
+                                          maxzd,&
+                                          minxd,&
+                                          minyd,&
+                                          minzd)
+
+
+         !Copy and allocate locals on GPU
+         call gr_bh_gpu_Potential_Copy(device_ID,local_cell_count,  &  
+			            gr_bh_gpu_x_data,&
+				    gr_bh_gpu_y_data,&
+			            gr_bh_gpu_z_data,&
+                                                xd,&
+                                                yd,& 
+                                                zd,& 
+                                                potential_memory_timings)
+
+         begin_count = local_cell_count
+
+        do while((current_size  + local_cell_count) .ge.  gr_bh_gpu_MAX_GPU_CELLS )           
+               current_size  = (current_size * 0.5)
+         end do
+
+
+         end_count = current_size  + local_cell_count
+         do while(end_count .le. gr_bh_gpu_total_cell_count)
+  
+               call gr_bh_gpu_Potential_Subset( device_ID,gpu_blocks,&
+                                           local_cell_count,&
+                                           end_count-begin_count,&
+					   begin_count,&
+			            gr_bh_gpu_x_data,&
+				    gr_bh_gpu_y_data,&
+			            gr_bh_gpu_z_data,&
+				    gr_bh_gpu_m_data,&
+                                           maxxd,&
+                                           maxyd,&
+                                           maxzd,&
+                                           minxd,&
+                                           minyd,&
+                                           minzd,&
+                                           xd,& 
+                                           yd,&
+                                           zd,&
+                                          potential_memory_timings,&
+                                          potential_kernel_timings)
+
+
+
+               
+
+               if(end_count .eq. gr_bh_gpu_total_cell_count)then
+                  exit
+               endif
+
+               begin_count = end_count
+               end_count = end_count + current_size
+
+                  if((end_count) .ge. gr_bh_gpu_total_cell_count .or. ((end_count) + 32) .ge. gr_bh_gpu_total_cell_count)then
+                     end_count = gr_bh_gpu_total_cell_count
+                  endif
+              
+
+              
+
+         end do
+    
+
+       else
+!=========================================UNSPLIT FORCE CALCULATION==========================================!
+       !Calculate potential with all data at once
+       call gr_bh_gpu_Potential_All(device_ID,gpu_blocks,&
+                                    local_cell_count,&
+                                    gr_bh_gpu_total_cell_count, & 
+			            gr_bh_gpu_x_data,&
+				    gr_bh_gpu_y_data,&
+			            gr_bh_gpu_z_data,&
+				    gr_bh_gpu_m_data,&
+                                    potential_memory_timings,&
+                                    potential_kernel_timings,&
+                                      maxxd,&
+                                      maxyd,&
+                                      maxzd,&
+                                      minxd,&
+                                      minyd,&
+                                      minzd)
+
+
+       endif 
+
+        call gr_bh_gpu_Potential_Clean(device_Id,local_cell_count, &
+                                       other,&
+                                       gr_bh_gpu_periodic,&
+                                       potential_memory_timings,&
+                                       gr_bh_gpu_p_data,&
+                                       maxxd,&
+                                       maxyd,&
+                                       maxzd,&
+                                       minxd,&
+                                       minyd,&
+                                       minzd,&
+                                       xd,&
+                                       yd,&
+                                       zd,&
+                                       gpotd,&
+                                       e_gridd)
+ 
+   endif
+        
+
+     
+
+return
+end subroutine gr_bh_gpu_CalcPot
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Calculate_Direkt_Kernel.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Calculate_Direkt_Kernel.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Calculate_Direkt_Kernel.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,64 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+#include "gr_bh_gpu_Potential_Calculate_Direkt_Kernel.cuh"
+
+__global__
+void GravPotential_Direkt_Summ_Kernel(int n_old,int from, int till, double *posx, double *posy, double *posz, int ind)
+{
+ double dx,dy,dz,px,py,pz,ax,tmp;
+ int k,n;
+ for (k = threadIdx.x + blockIdx.x * blockDim.x; k < n_old; k += blockDim.x * gridDim.x) 
+    {
+      px = posx[k];
+      py = posy[k];
+      pz = posz[k];
+
+      ax = 0.0;
+      for(n = 0 ; n <  n_old; n++)
+         {
+            dx = posxd[n] - px;
+            dy = posyd[n] - py;
+            dz = poszd[n] - pz;
+            
+            if(n != k)
+              {
+                   tmp = dx*dx + (dy*dy + (dz*dz));  
+                   tmp = rsqrt(tmp); 
+                   ax += massd[n]*tmp;
+              }
+         }
+grv_potd[k] += ax;
+
+    }
+}
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Init.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Init.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Init.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,176 @@
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+
+
+// PERIODICITY AND EWALD
+#include "gr_bh_gpu_Ewald_Real_Periodic_Kernel.cuh"
+#include "gr_bh_gpu_Ewald_Rec_Periodic_Kernel.cuh"
+#include "gr_bh_gpu_Ewald_Extrapolate_Kernel.cuh"
+#include "gr_bh_gpu_Ewald_Temp_Field_Kernel.cuh"
+#include "gr_bh_gpu_Periodic_InterpolateInField.cuh"
+
+extern "C" {
+void gr_bh_gpu_Ewald_Init_(int *NX,int *NY, int *NZ,
+                               int *grv_bhEwaldSeriesN,
+                               int *gr_bhEwaldIsoFac,
+                               double *grv_bhLx,double *grv_bhLy,double *grv_bhLz,
+                               double *e_xmax,double *e_ymax,double *e_zmax, int *periodics)
+    {
+ int periodicity = *periodics;
+ int es_nrx = *grv_bhEwaldSeriesN;
+ int es_nry = *grv_bhEwaldSeriesN;
+ int es_nrz = *grv_bhEwaldSeriesN;
+ double Lx_per = *gr_bhEwaldIsoFac * *grv_bhLx;
+ double Ly_per = *gr_bhEwaldIsoFac * *grv_bhLy;
+ double Lz_per = *gr_bhEwaldIsoFac * *grv_bhLz;
+
+
+ double Lav, ewald_alpha,Ln;
+ //int es_radius2 = *grv_bhEwaldSeriesN * *grv_bhEwaldSeriesN;
+
+//! axis ratio of ellipsis
+
+
+ 
+
+
+
+//! particular values for problem with given kind of boundary conditions and orientation
+//! note ewald_periodicity = 1,2,4 means 1 direction periodic and 2 isolated,
+//! ewald_periodicity = 3,5,6 means 2 directions periodic and 1 isolated,
+//! ewald_periodicity = 7 means boundary conditions isolated in tree directions 
+
+         
+               /*
+        ewald 0 = no periodic
+        ewald 1 = X periodic
+        ewald 2 = y periodic
+        ewald 3 = x & y periodic
+        ewald 4 = z periodic
+        ewald 5 = x & z periodic
+        ewald 6 = y & z periodic
+        ewald 7 = x & y & z periodic
+       */
+
+
+if( periodicity == 1 ||  periodicity == 3 || periodicity == 5 || periodicity == 7){ // xperiodic
+
+ 
+    Lx_per = *grv_bhLx ;
+    *e_xmax = 0.5 * (*grv_bhLx);
+}
+
+if( periodicity == 2 ||  periodicity == 3 || periodicity == 6 ||periodicity == 7 ){//yperiodic
+ 
+    Ly_per =  *grv_bhLy; 
+    *e_ymax = 0.5 * (*grv_bhLy);
+}
+
+
+if( periodicity == 4 ||  periodicity == 5 || periodicity== 6 || periodicity == 7){ //zperiodic
+  
+    Lz_per = *grv_bhLz;
+    *e_zmax = 0.5 * (*grv_bhLz);
+}
+
+  Lav = (Lx_per+Ly_per+Lz_per)/3.0;
+  ewald_alpha = 2.0/Lav;
+  Ln = Lx_per*Ly_per*Lz_per;
+  
+
+
+
+
+
+                     if (cudaSuccess != cudaMemcpyToSymbol(ewald_xmax, e_xmax, sizeof(double))) 
+                     fprintf(stderr, "copying of cf1 to device failed\n"); 
+                     CudaTest( (char*)"cf1 copy to device failed");
+
+                     if (cudaSuccess != cudaMemcpyToSymbol(ewald_ymax, e_ymax, sizeof(double))) 
+                     fprintf(stderr, "copying of cf2 to device failed\n"); 
+                     CudaTest( (char*)"cf2 copy to device failed");
+   
+                     if (cudaSuccess != cudaMemcpyToSymbol(ewald_zmax, e_zmax, sizeof(double))) 
+                     fprintf(stderr, "copying of cf3 to device failed\n"); 
+                     CudaTest( (char*)"cf3 copy to device failed");
+
+                     //__constant__ int d_es_nrx, d_es_nry, d_es_nrz;
+                     if (cudaSuccess != cudaMemcpyToSymbol(d_es_nrx, &es_nrx, sizeof(int))) 
+                     fprintf(stderr, "copying of es_nrx to device failed\n"); 
+                     CudaTest( (char*)"es_nrx copy to device failed");
+
+                     if (cudaSuccess != cudaMemcpyToSymbol(d_es_nry, &es_nry, sizeof(int))) 
+                     fprintf(stderr, "copying of es_nry to device failed\n"); 
+                     CudaTest( (char*)"es_nry copy to device failed");
+   
+                     if (cudaSuccess != cudaMemcpyToSymbol(d_es_nrz, &es_nrz, sizeof(int))) 
+                     fprintf(stderr, "copying of es_nrz to device failed\n"); 
+                     CudaTest( (char*)"es_nrz copy to device failed");
+
+
+                     //__constant__ double d_grv_bhLx, d_grv_bhLy,d_grv_bhLz;
+                     if (cudaSuccess != cudaMemcpyToSymbol(d_grv_bhLx, grv_bhLx, sizeof(double))) 
+                     fprintf(stderr, "copying of grv_bhLx to device failed\n"); 
+                     CudaTest( (char*)"grv_bhLx copy to device failed");
+
+                     if (cudaSuccess != cudaMemcpyToSymbol(d_grv_bhLy, grv_bhLy, sizeof(double))) 
+                     fprintf(stderr, "copying of grv_bhLy to device failed\n"); 
+                     CudaTest( (char*)"grv_bhLx copy to device failed");
+   
+                     if (cudaSuccess != cudaMemcpyToSymbol(d_grv_bhLz, grv_bhLz, sizeof(double))) 
+                     fprintf(stderr, "copying of grv_bhLz to device failed\n"); 
+                     CudaTest( (char*)"grv_bhLz copy to device failed");
+
+
+
+
+                     //__constant__ int d_NX,d_NY,d_NZ;
+                     if (cudaSuccess != cudaMemcpyToSymbol(d_NX, NX, sizeof(int))) 
+                     fprintf(stderr, "copying of NX to device failed\n"); 
+                     CudaTest( (char*)"NX copy to device failed");
+
+                     if (cudaSuccess != cudaMemcpyToSymbol(d_NY, NY, sizeof(int))) 
+                     fprintf(stderr, "copying of NY to device failed\n"); 
+                     CudaTest( (char*)"NY copy to device failed");
+   
+                     if (cudaSuccess != cudaMemcpyToSymbol(d_NZ, NZ, sizeof(int))) 
+                     fprintf(stderr, "copying of NZ to device failed\n"); 
+                     CudaTest( (char*)"NZ copy to device failed");
+
+
+                   //__constant__ double d_ewald_alpha;
+                     if (cudaSuccess != cudaMemcpyToSymbol(d_ewald_alpha, &ewald_alpha, sizeof(double))) 
+                     fprintf(stderr, "copying of ewald_alpha to device failed\n"); 
+                     CudaTest( (char*)"ewald_alpha copy to device failed");
+                      
+
+
+
+
+
+                     //__constant__ double d_grv_bhDxI,d_grv_bhDyI,d_grv_bhDzI;
+                     if (cudaSuccess != cudaMemcpyToSymbol(d_Lx_per, &Lx_per, sizeof(double))) 
+                     fprintf(stderr, "copying of grv_bhDxIto device failed\n"); 
+                     CudaTest( (char*)"grv_bhDxI copy to device failed");
+
+                     if (cudaSuccess != cudaMemcpyToSymbol(d_Ly_per,&Ly_per, sizeof(double))) 
+                     fprintf(stderr, "copying of grv_bhDyI to device failed\n"); 
+                     CudaTest( (char*)"grv_bhDyI copy to device failed");
+
+                     if (cudaSuccess != cudaMemcpyToSymbol(d_Lz_per, &Lz_per, sizeof(double))) 
+                     fprintf(stderr, "copying of grv_bhDzI to device failed\n"); 
+                     CudaTest( (char*)"grv_bhDzI copy to device failed");
+
+                     if (cudaSuccess != cudaMemcpyToSymbol(d_Ln, &Ln, sizeof(double))) 
+                     fprintf(stderr, "copying of grv_bhDzI to device failed\n"); 
+                     CudaTest( (char*)"grv_bhDzI copy to device failed");
+
+
+
+
+//__constant__ volatile double *grid;
+
+
+    }
+}
+
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Init.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Init.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Init.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,207 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+
+// GENERAL TREE BUILDING
+#include "gr_bh_gpu_Init_Kernel.cuh"
+#include "gr_bh_gpu_MakeRootKernel.cuh"
+#include "gr_bh_gpu_Summarization_Kernel.cuh"
+#include "gr_bh_gpu_TreeBuilding_Kernel.cuh"
+
+
+extern "C" {
+void gr_bh_gpu_Potential_Init_(int *device_ID,double *itolsq,
+                                   int *local_cell_count,  
+                                   double *domain_diagonal,
+                                   double *minx,  double *miny, double *minz, double *maxx, double *maxy, double *maxz, 
+                                   double *e_grid, int *ewald_size, 
+                                   int *periodicity,
+                                   double *memory_time,
+                                   double **gpotd, double **gridl,
+                                   double **root_maxxd,
+                                   double **root_maxyd,
+                                   double **root_maxzd,
+                                   double **root_minxd,
+                                   double **root_minyd,
+                                   double **root_minzd
+                                  )
+    {
+
+
+
+
+
+
+
+  float time; 
+  double bytes = 0;
+  cudaEvent_t start, stop;
+
+  cudaSetDevice(*device_ID);
+ //start event timers to measure memory timings
+  cudaEventCreate(&start);
+  cudaEventCreate(&stop);
+  cudaEventRecord(start, 0);
+
+//alloc
+        
+	cudaMalloc((void **)root_minxd, sizeof(double)); 
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init: couldn't allocate minxd");
+
+	cudaMalloc((void **)root_minyd, sizeof(double)); 
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init: couldn't allocate minyd");
+
+	cudaMalloc((void **)root_minzd, sizeof(double));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init: couldn't allocate minzd");
+
+	cudaMalloc((void **)root_maxxd, sizeof(double));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init: couldn't allocate maxxd");
+
+	cudaMalloc((void **)root_maxyd, sizeof(double));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init: couldn't allocate maxyd");
+
+	cudaMalloc((void **)root_maxzd, sizeof(double));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init: couldn't allocate maxzd");
+ 
+
+	cudaMalloc((void **)gpotd, sizeof(double) * (*local_cell_count));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init: couldn't allocate grv_potd");
+
+
+	cudaEventRecord(stop, 0);
+	cudaEventSynchronize(start); 
+	cudaEventSynchronize(stop);
+	cudaEventElapsedTime(&time, start, stop);
+
+        memory_time[0] += time; 
+
+//cpytosym
+        cudaEventRecord(start, 0);
+
+	cudaMemcpyToSymbol(periodic, periodicity, sizeof(int)); 
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init: nbody copy to device failed");
+        bytes += 1 * sizeof(int);
+
+	cudaMemcpyToSymbol(grv_potd, gpotd, sizeof(void*)); 
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init: gpot copy to device symbol failed");
+  
+
+	cudaMemcpyToSymbol(domdiag, domain_diagonal, sizeof(double)); 
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init: domdiag copy to device symbol failed");
+        bytes += 1 * sizeof(double);
+
+	cudaMemcpyToSymbol(itolsqd, itolsq, sizeof(double));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init: itolsq copy to device failed");
+        bytes += 1 * sizeof(double);
+     
+
+
+//cpy
+
+
+	cudaMemcpy(*root_minxd, minx, sizeof(double), cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init: minx copy to device failed");
+        bytes += 1 * sizeof(double);
+
+	cudaMemcpy(*root_minyd, miny, sizeof(double), cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init: miny copy to device failed");   
+        bytes += 1 * sizeof(double);
+     
+	cudaMemcpy(*root_minzd, minz, sizeof(double), cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init: minz copy to device failed");
+        bytes += 1 * sizeof(double);
+
+	cudaMemcpy(*root_maxxd, maxx, sizeof(double), cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init: maxx copy to device failed");
+        bytes += 1 * sizeof(double);
+
+	cudaMemcpy(*root_maxyd, maxy, sizeof(double), cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init: maxy copy to device failed");
+        bytes += 1 * sizeof(double);
+
+	cudaMemcpy(*root_maxzd, maxz, sizeof(double), cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init: maxz copy to device failed");
+        bytes += 1 * sizeof(double);
+
+
+       cudaEventRecord(stop, 0);
+       cudaEventSynchronize(start); 
+       cudaEventSynchronize(stop);
+       cudaEventElapsedTime(&time, start, stop);
+      // printf("  Device to Host bandwidth (GB/s): %fn \n", bytes * 1e-6 / time);
+       memory_time[1] += time; 
+       memory_time[2] += bytes; 
+
+       
+
+/******************Allocate fields for periodicity***************/             
+
+
+ if (*periodicity != 0){
+
+
+	cudaMalloc((void **)gridl, sizeof(double) * (*ewald_size));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init:couldn't allocate gridl");
+
+
+	cudaMemcpyToSymbol(d_grid, gridl, sizeof(void*)); 
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init: childl copy to device failed");
+
+
+
+	cudaMemcpy(*gridl, e_grid, sizeof(double) * (*ewald_size), cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Potential_Init: posy copy to device failed");
+
+}
+
+
+
+
+    
+
+//memset
+
+   cudaMemset((void*) *gpotd, 0, sizeof(double) * (*local_cell_count));
+   CudaTest( (char*)"gr_bh_gpu_Potential_Init: couldn't initialize gpot");
+
+    cudaEventDestroy(start);
+    cudaEventDestroy(stop);  
+
+     
+
+
+
+  }
+}
+
+
Index: source/Grid/GridSolvers/BHTree/GPU/Config
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/Config	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/Config	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,18 @@
+#LIBRARY CUDA
+#REQUIRES hwaccel/GpuSupport/GpuSupportMain/CUDA
+#REQUIRES ./KernelCode
+
+DEFAULT KernelCode
+# Parameters:
+
+
+PARAMETER grv_bh_gpu_TreeLimAngle        REAL        0.5 [ 0 ... ]
+PARAMETER grv_bh_gpu_EwaldFieldNx        INTEGER     64	 [ 1 ... ]
+PARAMETER grv_bh_gpu_EwaldFieldNy        INTEGER     64	 [ 1 ... ]
+PARAMETER grv_bh_gpu_EwaldFieldNz        INTEGER     64	 [ 1 ... ]
+PARAMETER grv_bh_gpu_EwaldSeriesN        INTEGER     10
+PARAMETER grv_bh_gpu_EwaldNRef           INTEGER     -1  [ -1 ... ]
+PARAMETER grv_bh_gpu_PinnedCells         BOOLEAN     false [false, true]
+PARAMETER grv_bh_gpu_PinnedMPI           BOOLEAN     false [false, true]
+
+
Index: source/Grid/GridSolvers/BHTree/GPU/Grid_solvePoisson.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/Grid_solvePoisson.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/Grid_solvePoisson.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,350 @@
+!!****if* source/Grid/GridSolvers/BHTree/GPU/Grid_solvePoisson.F90
+!!
+!! NAME
+!!
+!!  Grid_solvePoisson
+!!
+!! SYNOPSIS
+!!
+!!  call Grid_solvePoisson(integer(IN) :: iSoln,
+!!                         integer(IN) :: iSrc, 
+!!                         integer(IN) :: bcTypes(6),
+!!                         real(IN)    :: bcValues(2,6),
+!!                         real(INOUT) :: poisfact)
+!!
+!! DESCRIPTION
+!!
+!!  This routine solves Poisson's using a BH tree on GPUs.
+!!
+!! AUTHOR
+!!
+!!  (c) 2016 Gunther Lukat 
+!!  
+!!  
+!! ARGUMENTS
+!!
+!!  iSoln    - the index for the solution variable (potential when used for self-gravity)
+!!  iSrc     - the index of the source variable (density when used for self-gravity)
+!!  bcTypes  - the boundary condition type; only the first entry is used to determine
+!!             whether all boundaries should be handled as isolated.
+!!                    the following conditions are presently supported:
+!!                     GRID_PDE_BND_PERIODIC (1)
+!!                     GRID_PDE_BND_ISOLATED (0)
+!!             The current implementation ignores this argument and uses runtime parameters
+!!             directly instead.
+!!  bcValues - an unused argument used to keep the interface standard.
+!!             The current implementation ignores this argument.
+!!  poisfact      - the scaling factor for the eventual solution
+!!                  for gravity this is 4*pi*G
+!!
+!! RESULT
+!!  
+!!  The variable at iSoln contains the solution to L(u) = rho for the
+!!  given boundary value setup.
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!  The symbols listed above for bcTypes are declared as FORTRAN PARAMETERS in
+!!  the module Grid_interfaces.  Code using this interface should refer to that
+!!  module with a USE statement, like this:
+!!
+!!    use Grid_interface, ONLY : GRID_PDE_BND_PERIODIC, GRID_PDE_BND_NEUMANN, &
+!!       GRID_PDE_BND_ISOLATED, GRID_PDE_BND_DIRICHLET, &
+!!       Grid_solvePoisson
+!! SEE ALSO
+!! Gunther Lukat, Robi Banerjee, A GPU accelerated Barnes Hut tree code for FLASH4, New Astronomy, Volume 45, May 2016, 
+!! Pages 14-28, ISSN 1384-1076, http://dx.doi.org/10.1016/j.newast.2015.10.007.
+!!  
+!!
+!!***
+
+subroutine Grid_solvePoisson (iSoln, iSrc, bcTypes, bcValues, poisfact)
+
+ use gr_bh_gpu_Interface, ONLY : gr_bh_gpu_GatherCells,gr_bh_gpu_CommLocalRoots,& 
+                                 gr_bh_gpu_IrecvEssentials,gr_bh_gpu_CalcPot,gr_bh_gpu_CalcPotSelf,&
+                                 gr_bh_gpu_SavePot,gr_bh_gpu_CleanUp, gr_bh_gpu_CalcExportTree,&
+                                 gr_bh_gpu_WaitForEssentialSizes,gr_bh_gpu_WaitForEssentialData,&
+                                 gr_bh_gpu_ReAllocBlocks,gr_bh_gpu_Essentials_Probe_Messages,gr_bh_gpu_Essentials_Receive_Message
+
+ use Grid_data, ONLY : gr_meshMe,gr_meshNumProcs, gr_meshComm
+ use gr_bh_gpu_Data, ONLY : gr_bh_gpu_GravFac,gr_bh_gpu_idensvar,gr_bh_gpu_ipoisvar,&
+                            gpu_comm,gpu_rank,gpu_commSize,&
+			    gr_bh_gpu_message_count,recv_buffer,recv_request,&
+                            gr_bh_gpu_essentials_count,gr_bh_gpu_local_cell_count,gr_bh_gpu_total_cell_count,&
+			    gr_bh_gpu_pinned_mpi,gr_bh_gpu_pinned_cells
+			
+
+
+
+ use Timers_interface, ONLY : Timers_start, Timers_stop
+ use tree, ONLY: grid_changed
+ use, intrinsic :: iso_c_binding
+ implicit none
+
+#include "Flash.h"
+#include "constants.h"
+#include "Flash_mpi.h"
+#ifdef USE_VT
+include "VT.inc"
+#endif
+
+
+  integer, intent(IN)               :: iSoln, iSrc
+  integer, dimension(6), intent(IN) :: bcTypes
+  real, dimension(2,6), intent(IN)  :: bcValues
+  real, intent(INOUT)               :: poisfact !DEV: NOT intent(IN) because some implementation actually changes it? - KW
+  integer :: ierr, i, vt_err,xx
+  integer :: mstat(MPI_STATUS_SIZE)
+  integer :: finished
+  integer :: sender, istat,req
+  logical :: test_lock,recvflag
+  logical :: done_essentials
+  logical :: lock_flag
+  logical :: essentials_flag
+  integer :: elements
+  integer :: GPU_LOCK,TEMP_LOCK, GPU_STAT
+  integer :: GPU_LOCK_MPI_REQ= MPI_REQUEST_NULL
+  logical :: GPU_LOCK_FLAG 
+!=======================================================================
+#ifdef USE_VT
+call VTTRACEON()
+#endif
+
+gr_bh_gpu_GravFac =poisfact / (4*PI)
+gr_bh_gpu_idensvar = iSrc
+gr_bh_gpu_ipoisvar = iSoln
+gr_bh_gpu_essentials_count = 0
+gr_bh_gpu_local_cell_count = 0
+gr_bh_gpu_total_cell_count = 0
+
+
+
+ call Timers_start("gravity_tree")
+
+
+
+ call Timers_start("GatherCells")
+ call gr_bh_gpu_GatherCells()
+ call Timers_stop("GatherCells")
+
+
+!======THERE IS ONLY ONE PROCESS RUNNING => NO COMMUNICATION === !
+if(gr_meshNumProcs .eq. 1) then
+
+  !!CALCULATE  POTENTIAL
+  call Timers_start("CalcPot")
+    call gr_bh_gpu_CalcPotSelf()
+  call Timers_stop("CalcPot")
+
+  !!SAVE POTENTIAL
+  call Timers_start("SavePot")
+   call gr_bh_gpu_SavePot()
+  call Timers_stop("SavePot")
+
+  !!SAVE CLEANUP
+  call Timers_start("CleanUp")
+   call gr_bh_gpu_CleanUp()
+  call Timers_stop("CleanUp")
+
+endif!=END => NO COMMUNICATION === !
+
+
+
+
+
+
+
+
+
+!======COMMUNICATION REQUIRED === ! 
+if(gr_meshNumProcs .gt. 1)then
+
+
+
+
+
+
+
+!=========== NO CONCATENATION ===================!
+
+  call Timers_start("CommLocalRoots")
+     call gr_bh_gpu_CommLocalRoots()
+  call Timers_stop("CommLocalRoots")
+
+#ifdef GPU_DEBUG
+
+   write(*,*) "finished Commroots"
+
+#endif
+
+  done_essentials = .false.
+  gr_bh_gpu_message_count = 1
+
+GPU_LOCK = 0
+if(gpu_rank .eq. 0)GPU_LOCK = 1
+
+
+
+ call Timers_start("CalcExportTree")
+
+  do while (.not. done_essentials)
+     if(GPU_LOCK .eq. 1)then
+        call gr_bh_gpu_CalcExportTree()
+
+        done_essentials = .true.
+	TEMP_LOCK = 1
+        GPU_LOCK_MPI_REQ= MPI_REQUEST_NULL 
+
+        if(gpu_rank+1 .le. gpu_commSize-1)then
+            call MPI_ISEND(TEMP_LOCK,1,MPI_INTEGER,gpu_rank+1,gr_meshNumprocs*2,gpu_comm,GPU_LOCK_MPI_REQ,ierr)
+        endif
+        GPU_LOCK = 0 
+     else if(gpu_rank .ge. 1 .and. .not. done_essentials)then
+        GPU_LOCK_FLAG = .false.
+        GPU_LOCK_MPI_REQ= MPI_REQUEST_NULL  
+        call MPI_IRECV(GPU_LOCK_FLAG,1,MPI_INTEGER,gpu_rank-1,gr_meshNumprocs*2,gpu_comm,GPU_LOCK_MPI_REQ,ierr)
+        call MPI_Test(GPU_LOCK_MPI_REQ, GPU_LOCK_FLAG, MPI_STATUS_IGNORE, ierr)
+        do while(.not. GPU_LOCK_FLAG )
+
+         call gr_bh_gpu_Essentials_Probe_Messages(essentials_flag,sender,elements)
+         
+          if(essentials_flag .and. sender .ge. 0)then
+
+             gr_bh_gpu_essentials_count = gr_bh_gpu_essentials_count + elements                                   !save number of essentials
+
+  	   call gr_bh_gpu_Essentials_Receive_Message(recv_buffer,recv_request,&
+						    elements,&
+						    sender,&
+						    gr_meshMe,&
+						    gr_meshComm,gr_bh_gpu_pinned_mpi)
+            gr_bh_gpu_message_count = gr_bh_gpu_message_count +1 
+          endif
+          call MPI_Test(GPU_LOCK_MPI_REQ,  GPU_LOCK_FLAG, MPI_STATUS_IGNORE, ierr)
+        end do
+        GPU_LOCK_FLAG = .true.
+        GPU_LOCK = 1
+     endif
+  end do
+
+!	if(gpu_rank .eq. gpu_commSize-1)then
+!		write(*,*) "will send", gpu_rank, gr_meshMe
+!		GPU_LOCK_MPI_REQ = MPI_REQUEST_NULL
+!		call MPI_ISEND(GPU_LOCK,1,MPI_INTEGER,0,0,gpu_comm,GPU_LOCK_MPI_REQ,ierr)
+!        endif 
+
+ call Timers_stop("CalcExportTree")
+
+#ifdef GPU_DEBUG
+
+   write(*,*) "finished Calcexport"
+
+#endif
+
+
+
+
+  call Timers_start("WaitForEssentialSizes")
+    call gr_bh_gpu_WaitForEssentialSizes()
+  call Timers_stop("WaitForEssentialSizes")
+
+#ifdef GPU_DEBUG
+
+   write(*,*) "finished wait for essential sizes"
+
+#endif
+
+
+  
+  call Timers_start("ReAllocBlocks")
+     call gr_bh_gpu_ReAllocBlocks()
+  call Timers_stop("ReAllocBlocks")
+
+#ifdef GPU_DEBUG
+
+   write(*,*) "finished reallocation"
+
+#endif
+
+
+
+  call Timers_start("WaitForEssentialData")
+      call gr_bh_gpu_WaitForEssentialData()
+  call Timers_stop("WaitForEssentialData")
+
+#ifdef GPU_DEBUG
+
+   write(*,*) "finished waiting for data"
+
+#endif
+
+
+ 
+!	if(gpu_rank .eq. 0)then
+!  		call MPI_Recv(GPU_LOCK, 1, MPI_INTEGER, gpu_commSize-1, 0, gpu_comm, GPU_LOCK_MPI_REQ,ierr)
+!	endif
+
+!	if(gpu_rank .eq. gpu_commSize-1)then
+!                write(*,*) "WILL WAIT",gpu_rank, gr_meshMe
+!		call MPI_Wait(GPU_LOCK_MPI_REQ,GPU_STAT, ierr)
+!	endif
+
+
+
+   call Timers_start("CalcPot")
+      do i = 0,gpu_commSize-1
+       if(gpu_rank .eq. i) then
+        call gr_bh_gpu_CalcPot()
+       else
+        call Mpi_Barrier(gpu_comm,ierr)
+       end if
+     end do
+   call Timers_stop("CalcPot")
+
+#ifdef GPU_DEBUG
+
+   write(*,*) "finished Calc POT"
+
+#endif
+
+
+
+
+  call Timers_start("SavePot")
+   call gr_bh_gpu_SavePot()
+  call Timers_stop("SavePot")
+
+
+  call Timers_start("CleanUp")
+   call gr_bh_gpu_CleanUp()
+  call Timers_stop("CleanUp")
+
+ 
+
+endif!======COMMUNICATION REQUIRED === ! 
+
+
+
+
+
+ call Timers_stop("gravity_tree")
+ 
+#ifdef USE_VT
+call VTTRACEOFF()
+#endif
+
+
+return
+end subroutine Grid_solvePoisson
+
+
+
+
+
+
+
+
+
+
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Kernel_Constants.cuh
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Kernel_Constants.cuh	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Kernel_Constants.cuh	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,79 @@
+
+#ifndef __KERNEL_CONSTANTS_CUH__
+#define __KERNEL_CONSTANTS_CUH__
+
+#define MAXDEPTH 32
+#define WARPSIZE 32
+#define FACTOR5 5
+#define FACTOR8 4
+#define FACTOR1 3
+#define FACTOR2 1
+#define FACTOR3 1
+#define THREADS0 1
+#define THREADS2 1024
+#define THREADS3 1024
+#define THREADS4 512
+#define THREADS1 256
+#define THREADS7 192
+#define CUDA_PI 3.1415926535897930
+
+
+#include <stdio.h> 
+#include <cuda.h>
+#include <cuda_runtime.h>
+#include <thrust/device_ptr.h>
+#include <thrust/sort.h>
+#include <thrust/reduce.h>
+#include <thrust/copy.h>
+#include <thrust/execution_policy.h>
+
+
+#ifdef TRACE
+ #include <cuda_profiler_api.h>
+#endif
+
+#include "CudaError.cuh"
+#include "Flash.h"
+#include "constants.h"
+
+extern __device__ volatile int bottomd;
+extern __device__ volatile int maxdepthd;
+extern __device__ volatile int blkcntd;
+extern __constant__ int nnodesd,nbodiesd;
+extern __constant__ volatile double *posxd, *posyd, *poszd;
+extern __constant__ volatile double *posrd;
+extern __constant__ volatile double *massd;
+extern __constant__ volatile double *grv_potd;
+extern __constant__ volatile double *delta;
+extern __constant__ volatile int   *childd;
+extern __constant__ double  itolsqd;
+extern __constant__ double  G;
+extern __device__ volatile double domdiag;
+extern __device__ volatile double radiusd, radiusd2;
+extern __constant__ volatile double *maxxd, *maxyd, *maxzd, *minxd, *minyd, *minzd;
+/********EWALD CONSTANTS************/
+extern __constant__ int periodic;
+extern __constant__ double d_ewald_alpha;
+extern __constant__ double d_Ln;
+extern __constant__ volatile double d_grv_bhLx, d_grv_bhLy,d_grv_bhLz;
+extern __constant__ double *d_grid;
+extern __constant__ double ewald_xmax,ewald_ymax,ewald_zmax;
+extern __constant__ int d_NX,d_NY,d_NZ;
+extern __constant__ int d_es_nrx, d_es_nry, d_es_nrz;
+extern __constant__ double d_Lx_per,d_Ly_per,d_Lz_per;
+
+
+
+ struct cp
+  {
+  const int a;
+  cp(int _a) : a(_a){}
+    __host__ __device__
+    bool operator()(const int &x)
+    {
+      return (x >= 0);
+    }
+  };
+
+#endif
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_GatherCells.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_GatherCells.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_GatherCells.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,160 @@
+!!****if* source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_GatherCells
+!!
+!! NAME
+!!
+!!  gr_bh_gpu_GatherCells
+!!
+!! SYNOPSIS
+!!
+!!  call gr_bh_gpu_GatherCells()
+!!
+!! DESCRIPTION
+!!
+!! This routine sets the number of nodes for the GPU tree, allocates pinned/paged meory in "C",
+!! calculates the center of mass for the PIDs domain and copies postion and mass data to local memory.
+!!
+!! ARGUMENTS
+!!
+!!   No arguments
+!!
+!! 
+!!
+!!
+!!***
+
+
+subroutine gr_bh_gpu_GatherCells()
+  
+   use Driver_interface, ONLY : Driver_abortFlash
+   use Grid_data, ONLY : gr_meshMe
+   use Grid_Interface, ONLY: Grid_getListOfBlocks,Grid_getCellCoords
+   use gr_bh_gpu_data, ONLY: gr_bh_gpu_local_domdiagsq, gr_bh_gpu_roots_own,gr_bh_gpu_idensvar, &
+                             gr_bh_gpu_MAX_GPU_CELLS,gpu_blocks, &
+                             gr_bh_gpu_local_xmax,gr_bh_gpu_local_xmin,&
+                             gr_bh_gpu_local_ymax,gr_bh_gpu_local_ymin,&
+                             gr_bh_gpu_local_zmax,gr_bh_gpu_local_zmin,&
+                             gpu_rank,gpu_commSize,gr_bh_gpu_pinned_cells,&
+			     gr_bh_gpu_local_cell_count,gr_bh_gpu_node_count,gr_bh_gpu_essentials_count,&
+                             gr_bh_gpu_x_data,gr_bh_gpu_y_data,gr_bh_gpu_z_data,gr_bh_gpu_m_data,gr_bh_gpu_p_data
+
+  
+   use gr_bh_gpu_Interface, ONLY: gr_bh_gpu_GetGridData,gr_bh_gpu_Gather_Allocate_CellData,gr_bh_gpu_set_nodes
+   use iso_c_binding
+   implicit none
+
+#include "Flash.h"
+#include "constants.h"
+
+   integer :: l,ierr,i
+   integer :: gr_bh_gpu_LeafblockCount 
+   integer :: gr_bh_gpu_LeafblockList(MAXBLOCKS)
+   integer :: istat
+   real    :: roots_minx,roots_maxx,roots_miny,roots_maxy,roots_minz,roots_maxz,roots_radius
+   real    :: lx,ly,lz,lm
+
+
+   real            :: xi(1:NXB)
+   real            :: yi(1:NYB)
+   real            :: zi(1:NZB)
+
+    gr_bh_gpu_essentials_count = 0
+    call Grid_getListOfBlocks(LEAF,gr_bh_gpu_LeafblockList,gr_bh_gpu_LeafblockCount)
+
+
+ 
+
+    gr_bh_gpu_local_cell_count = gr_bh_gpu_LeafblockCount*NXB*NYB*NZB
+
+    if(gr_bh_gpu_local_cell_count .ge. gr_bh_gpu_MAX_GPU_CELLS) then 
+         call Driver_abortFlash("Reached the maximum cell number of 90 MIO  in GPU_TREE_CODE. Use less Blocks or More Processors")
+   endif
+
+
+    gr_bh_gpu_roots_own(0)=0 !X
+    gr_bh_gpu_roots_own(1)=0 !Y
+    gr_bh_gpu_roots_own(2)=0 !Z
+    gr_bh_gpu_roots_own(3)=0 !Mass
+    gr_bh_gpu_roots_own(4)=0 !Radius
+
+    if(gr_bh_gpu_local_cell_count .gt. 1) then 
+
+    call gr_bh_gpu_set_nodes(gpu_blocks,gr_bh_gpu_local_cell_count,gr_bh_gpu_node_count)
+
+
+
+    call gr_bh_gpu_Gather_Allocate_CellData(gr_bh_gpu_x_data,gr_bh_gpu_y_data,gr_bh_gpu_z_data,gr_bh_gpu_m_data,gr_bh_gpu_p_data,&
+					    gr_bh_gpu_local_cell_count,gr_bh_gpu_pinned_cells,gr_meshMe)
+
+    call Grid_getCellCoords(IAXIS, gr_bh_gpu_LeafblockList(1), CENTER, .false., xi, NXB)
+    call Grid_getCellCoords(JAXIS, gr_bh_gpu_LeafblockList(1), CENTER, .false., yi, NYB)
+    call Grid_getCellCoords(KAXIS, gr_bh_gpu_LeafblockList(1), CENTER, .false., zi, NZB)
+
+    roots_minx = xi(1)
+    roots_maxx = xi(1)
+    roots_miny = yi(1)
+    roots_maxy = yi(1)
+    roots_minz = zi(1)
+    roots_maxz = zi(1)
+    roots_radius = max(roots_maxx - roots_minx, roots_maxy - roots_miny,roots_maxz-roots_minz) * 0.5
+
+
+
+
+    do l = 1 , gr_bh_gpu_LeafblockCount
+
+    call gr_bh_gpu_GetGridData(gr_bh_gpu_LeafblockList(l),&
+                               gr_bh_gpu_node_count,&
+                               l-1,&
+                               gr_bh_gpu_idensvar,&
+                               roots_minx,roots_maxx,roots_miny,roots_maxy,roots_minz,roots_maxz,&
+                               lx,ly,lz,lm)
+
+    !center of MASS
+    gr_bh_gpu_roots_own(0)=gr_bh_gpu_roots_own(0) + lx !X
+    gr_bh_gpu_roots_own(1)=gr_bh_gpu_roots_own(1) + ly !Y
+    gr_bh_gpu_roots_own(2)=gr_bh_gpu_roots_own(2) + lz !Z
+    gr_bh_gpu_roots_own(3)=gr_bh_gpu_roots_own(3) + lm !Mass
+    
+
+   end do
+   
+
+
+
+ 
+ 
+ 
+ 
+ 
+  !use  center of mass
+  gr_bh_gpu_roots_own(0)  = gr_bh_gpu_roots_own(0) / gr_bh_gpu_roots_own(3)
+  gr_bh_gpu_roots_own(1)  = gr_bh_gpu_roots_own(1) / gr_bh_gpu_roots_own(3)
+  gr_bh_gpu_roots_own(2)  = gr_bh_gpu_roots_own(2) / gr_bh_gpu_roots_own(3)
+  gr_bh_gpu_roots_own(3)  = gr_bh_gpu_roots_own(3)
+  !RADIUS
+  
+  gr_bh_gpu_roots_own(4)  = max(abs(roots_maxx - roots_minx),&
+                                abs(roots_maxy - roots_miny),&
+                                abs(roots_maxz-roots_minz)) * 0.5
+  !DIAGONAL
+  gr_bh_gpu_roots_own(5) = (abs(roots_maxx - roots_minx)**2 + &
+                              &  abs(roots_maxy - roots_miny)**2 + &
+                              &  abs(roots_maxz-roots_minz)**2 )
+  
+  gr_bh_gpu_roots_own(6)  =roots_maxx
+  gr_bh_gpu_roots_own(7)  =roots_minx
+  gr_bh_gpu_roots_own(8)  =roots_maxy
+  gr_bh_gpu_roots_own(9)  =roots_miny
+  gr_bh_gpu_roots_own(10) =roots_maxz
+  gr_bh_gpu_roots_own(11) =roots_minz
+  !use  center of GEOMETRY
+  gr_bh_gpu_roots_own(0)  = roots_minx + gr_bh_gpu_roots_own(4)
+  gr_bh_gpu_roots_own(1)  = roots_miny + gr_bh_gpu_roots_own(4)
+  gr_bh_gpu_roots_own(2)  = roots_minz + gr_bh_gpu_roots_own(4)
+
+
+ endif
+  
+return
+end subroutine gr_bh_gpu_GatherCells
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Real_Periodic_Kernel.cuh
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Real_Periodic_Kernel.cuh	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Real_Periodic_Kernel.cuh	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,12 @@
+#ifndef __EWALD_REAL_PERIODIC_CUH__
+#define __EWALD_REAL_PERIODIC_CUH__
+
+
+#include "gr_bh_gpu_Kernel_Constants.cuh"       
+
+
+
+
+__global__ void ewald_real_periodic_all(double *t_grid_x,double *t_grid_y,double *t_grid_z,double *grid);
+
+#endif
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Calculate_Direkt_Kernel.cuh
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Calculate_Direkt_Kernel.cuh	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Calculate_Direkt_Kernel.cuh	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,41 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+#ifndef __POT_DIREKT_KERNEL_CUH__
+#define __POT_DIREKT_KERNEL_CUH__
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+#include "gr_bh_gpu_Periodic_InterpolateInField.cuh"
+
+__global__
+void GravPotential_Direkt_Summ_Kernel(int n_old,int from, int till, double *posx, double *posy, double *posz, int ind);
+
+#endif
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_GetGridData_SetValue.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_GetGridData_SetValue.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_GetGridData_SetValue.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,56 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+
+
+
+extern "C" {
+void gr_bh_gpu_GetGridData_SetValue_(double **x_data, 
+				double **y_data, 
+				double **z_data, 
+				double **m_data,
+				double *x,
+				double *y,
+				double *z,
+				double *m,
+				int *idx)
+    {
+	(*x_data)[*idx]=*x;
+	(*y_data)[*idx]=*y;
+	(*z_data)[*idx]=*z;
+	(*m_data)[*idx]=*m;
+ 
+
+
+    }
+}
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Calculate_Nodes_Kernel.cuh
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Calculate_Nodes_Kernel.cuh	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Calculate_Nodes_Kernel.cuh	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,50 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+#ifndef __ESS_CALC_NODES_CUH__
+#define __ESS_CALC_NODES_CUH__
+
+
+#include "gr_bh_gpu_Kernel_Constants.cuh"   
+
+  
+__global__
+void NeighborsKernel_Force_assure(int *essentials);
+
+
+__global__    
+void NeighborsKernel_Force_copy(int *essentials, int  *mediate, int count);
+
+__global__
+void NeighborsKernel_Force_nodes(int depth,double nodediag,double *roots,int offset,int *essentials, int *cnt);
+
+#endif
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_WaitForEssentialSizes.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_WaitForEssentialSizes.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_WaitForEssentialSizes.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,84 @@
+!!****if* BHTree/GPU/gr_bh_gpu_WaitForEssentialSizes
+!!
+!! NAME
+!!
+!!  gr_bh_gpu_WaitForEssentialSizes
+!!
+!! SYNOPSIS
+!!
+!!  call gr_bh_gpu_WaitForEssentialSizes()
+!!
+!! DESCRIPTION
+!!  
+!!  This routine implements a busy waiting for MPI messages. Here, the sizes of the foreign subtrees are recieved. 
+!!
+!! ARGUMENTS
+!!
+!!   No arguments
+!!
+!! 
+!!
+!!
+!!***
+
+
+
+subroutine gr_bh_gpu_WaitForEssentialSizes()
+ use Grid_data, ONLY : gr_meshNumProcs,gr_meshMe,gr_meshComm
+
+
+
+ use gr_bh_gpu_data, ONLY : gr_bh_gpu_essentials_count,gr_bh_gpu_message_count,recv_buffer,recv_request,&
+				gr_bh_gpu_pinned_mpi,gr_bh_gpu_pinned_cells
+                            
+                           
+
+ use gr_bh_gpu_Interface, ONLY: gr_bh_gpu_Essentials_Probe_Messages,gr_bh_gpu_Essentials_Receive_Message
+ use, intrinsic :: iso_c_binding
+
+implicit none
+#include "Flash_mpi.h"
+#include "Flash.h"
+ 
+integer :: sender_ID,ierr,m,istat,recv_msg_size
+integer :: mstat(MPI_STATUS_SIZE)
+integer ::i
+logical :: essentials_flag
+
+ m = 1
+ sender_ID =0
+
+
+do while (gr_bh_gpu_message_count .lt. gr_meshNumprocs)
+ essentials_flag = .false.
+ call gr_bh_gpu_Essentials_Probe_Messages(essentials_flag,sender_ID,recv_msg_size)
+
+
+ if(essentials_flag .and. sender_ID .ge. 0)then
+                                                !who is sending the message
+  gr_bh_gpu_essentials_count = gr_bh_gpu_essentials_count + recv_msg_size                                !save number of essentials
+ 
+  call gr_bh_gpu_Essentials_Receive_Message(recv_buffer,recv_request,&
+						    recv_msg_size,&
+						    sender_ID,&
+						    gr_meshMe,&
+						    gr_meshComm,gr_bh_gpu_pinned_mpi)
+             gr_bh_gpu_message_count = gr_bh_gpu_message_count +1 
+
+ endif
+
+end do
+ 
+
+
+
+
+
+
+
+
+ 
+return
+
+end subroutine  gr_bh_gpu_WaitForEssentialSizes
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Copy.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Copy.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Copy.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,97 @@
+
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+
+
+extern "C" {
+void gr_bh_gpu_Potential_Copy_(int *device_ID,int *local_cell_count, 
+                                       double **xp, double **yp, double **zp, 
+                                       double **xd, double **yd, double **zd, 
+                                       double *memory_time)
+    {
+
+
+double *x = *xp;
+double *y = *yp;
+double *z = *zp;              
+cudaEvent_t start, stop;
+float time;    
+cudaSetDevice(*device_ID);
+cudaEventCreate(&start);
+cudaEventCreate(&stop);
+cudaEventRecord(start, 0);
+
+
+cudaMalloc((void **)xd, sizeof(double) * (*local_cell_count) );
+CudaTest( (char*)"gr_bh_gpu_Potential_Copy: couldn't allocate selfx"); 
+
+cudaMalloc((void **)yd, sizeof(double) * (*local_cell_count));
+CudaTest( (char*)"gr_bh_gpu_Potential_Copy: couldn't allocate selfy"); 
+
+cudaMalloc((void **)zd, sizeof(double) * (*local_cell_count));
+CudaTest( (char*)"gr_bh_gpu_Potential_Copy: couldn't allocate selfz");
+
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+memory_time[0] += time;
+
+cudaEventRecord(start, 0);                   
+cudaMemcpy(*xd, x, sizeof(double) * (*local_cell_count), cudaMemcpyHostToDevice);
+CudaTest( (char*)"gr_bh_gpu_Potential_Copy: selfx copy to device failed");  
+                    
+cudaMemcpy(*yd, y, sizeof(double) * (*local_cell_count), cudaMemcpyHostToDevice);
+CudaTest( (char*)"gr_bh_gpu_Potential_Copy: selfy copy to device failed");  
+                    
+cudaMemcpy(*zd, z, sizeof(double) * (*local_cell_count), cudaMemcpyHostToDevice);
+CudaTest( (char*)"gr_bh_gpu_Potential_Copy: selfz copy to device failed");  
+         
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+
+memory_time[1] += time;
+memory_time[2] += 3 * (*local_cell_count) * sizeof(double);         
+ 
+
+
+       
+cudaEventDestroy(start);
+cudaEventDestroy(stop); 
+
+ }
+}
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Init.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Init.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Init.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,251 @@
+!!****if* source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Init.F90
+!!
+!! NAME
+!!  gr_bh_gpu_Init
+!!
+!! SYNOPSIS
+!!
+!!  call gr_bh_gpu_Init()
+!!
+!! DESCRIPTION
+!!
+!!  This routine initializes data fields and MPI Communicators for the GPU accelerated BH tree.
+!!  
+!!  
+!! ARGUMENTS
+!!
+!!
+!!***
+
+
+subroutine gr_bh_gpu_Init()
+   use Driver_interface, ONLY : Driver_abortFlash
+   use RuntimeParameters_interface, ONLY : RuntimeParameters_get
+   use Logfile_interface, ONLY : Logfile_stamp
+   use Grid_data, ONLY : gr_meshMe,gr_meshNumProcs, gr_meshComm,gr_geometry
+   use GpuSupport_cudaFortInterface, ONLY : GpuSupport_setDeviceMPIgroup, GpuSupport_getDeviceProperties
+
+   use gr_bh_gpu_Data, ONLY : gr_bh_gpu_GravFac,gr_bh_gpu_idensvar,gr_bh_gpu_ipoisvar,gr_bh_gpu_Mac_Angle,&
+                           gr_bh_gpu_EwaldFieldNx,gr_bh_gpu_EwaldFieldNy,gr_bh_gpu_EwaldFieldNz,&
+                           gr_bh_gpu_EwaldSIsoFac,gr_bh_gpu_EwaldSeriesN ,&
+                           gr_bh_gpu_mcs, &
+                           gr_bh_gpu_xmin_domain,gr_bh_gpu_xmax_domain,&
+                           gr_bh_gpu_ymin_domain,gr_bh_gpu_ymax_domain,&
+                           gr_bh_gpu_zmin_domain,gr_bh_gpu_zmax_domain,&
+                           gr_bh_gpu_MAX_GPU_CELLS,gr_bh_gpu_EwaldField, &
+                           gr_bh_gpu_Lx,gr_bh_gpu_Ly,gr_bh_gpu_Lz,&
+                           gr_bh_gpu_EwaldSize,gr_bh_gpu_global_domdiagsq,gr_bh_gpu_roots_received,&
+                           gpu_blocks,gpu_commSize,&
+                           gpu_comm,gpu_rank,gr_bh_gpu_periodic,first_call,&                         
+                           essentials_memory_timings,potential_memory_timings,&
+			   essentials_kernel_timings,potential_kernel_timings,&                             
+			   send_buffer, recv_buffer, send_request, recv_request,&
+			   gr_bh_gpu_pinned_mpi,gr_bh_gpu_pinned_cells,&
+			   DeviceCount,device_ID
+
+   use gr_bh_gpu_interface, ONLY :  gr_bh_gpu_CalcEwaldField,&
+	                            gr_bh_gpu_Init_Allocate_MPI_Buffer,&
+				    gr_bh_gpu_Ewald_MakeField
+
+  implicit none
+#include "Flash.h"
+#include "Flash_mpi.h"
+#include "constants.h"
+
+#ifdef USE_VT
+include 'VT.inc'
+#endif
+
+
+   integer :: stat,istat,ierr,i,j, namelen
+   real :: comm_test, comm_test_recv
+
+   !character(len=1024) :: filename
+   character(len=MPI_MAX_PROCESSOR_NAME) :: host_name
+   character(len=256) :: gpu_name
+   character(len=MAX_STRING_LENGTH) ::grav_boundary_type,&
+                                      grav_boundary_type_x,&
+				      grav_boundary_type_y,&
+				      grav_boundary_type_z
+
+   integer ::len_name, minor, major, multiProcessorCount, totalGlobalMem
+   len_name = 256
+
+   
+
+!================================================================================! 
+!========================= Check if BH-Code can be used =========================!
+  !call MPI_Get_processor_name(host_name, namelen,ierr);
+#ifdef USE_VT  
+call VTINIT(ierr)
+call VTTRACEON()
+#endif
+ if ((NDIM /= 3) .or. (gr_geometry /= CARTESIAN)) then
+     call Driver_abortFlash('[gr_bh_gpu_init] ERROR: tree Poisson solver works only in 3D Cartesian geometry')
+  endif
+
+  ! check if NBX = NBY = NBZ
+  if ((NXB /= NYB) .or. (NXB /= NZB) .or. (NYB /= NZB)) then
+     call Driver_abortFlash('[gr_bh_gpu_init] ERROR: must be NXB = NYB = NZB')
+  endif
+
+
+
+
+!=====================================================================! 
+!========================= Set Default Values=========================!
+
+gr_bh_gpu_Mac_Angle    = 0.5
+gr_bh_gpu_EwaldFieldNx = 32
+gr_bh_gpu_EwaldFieldNy = 32
+gr_bh_gpu_EwaldFieldNz = 32
+gr_bh_gpu_EwaldSeriesN = 10
+gr_bh_gpu_EwaldSIsoFac = 1000
+gr_bh_gpu_Periodic     = 0
+gr_bh_gpu_pinned_mpi   = .false.
+gr_bh_gpu_pinned_cells = .false.
+device_ID = 0
+
+
+!============== Allocate communication Fields=========================!
+if(gr_meshNumProcs >= 2) then
+
+
+
+ call gr_bh_gpu_Init_Allocate_MPI_Buffer(recv_buffer,send_buffer,recv_request,send_request,gr_meshNumProcs)
+
+
+ allocate(gr_bh_gpu_roots_received(0:gr_meshNumProcs *12),stat = istat)
+ if (istat /= 0)call Driver_abortFlash("could not allocate gr_bh_gpu_roots_received in file: gr_bh_gpu_init.F90")
+
+
+end if
+
+  
+
+
+!=====================================================================!
+!============== Get values from par file =============================!
+
+ call Grid_getMinCellSizes(gr_bh_gpu_mcs)
+
+ call RuntimeParameters_get("grv_bh_gpu_TreeLimAngle", gr_bh_gpu_Mac_Angle)
+ call RuntimeParameters_get("grv_bh_gpu_EwaldFieldNx", gr_bh_gpu_EwaldFieldNx)
+ call RuntimeParameters_get("grv_bh_gpu_EwaldFieldNy", gr_bh_gpu_EwaldFieldNy)
+ call RuntimeParameters_get("grv_bh_gpu_EwaldFieldNz", gr_bh_gpu_EwaldFieldNz)
+ call RuntimeParameters_get("grv_bh_gpu_EwaldSeriesN", gr_bh_gpu_EwaldSeriesN)
+
+
+ call RuntimeParameters_get("xmin", gr_bh_gpu_xmin_domain)
+ call RuntimeParameters_get("xmax", gr_bh_gpu_xmax_domain)
+ call RuntimeParameters_get("ymin", gr_bh_gpu_ymin_domain)
+ call RuntimeParameters_get("ymax", gr_bh_gpu_ymax_domain)
+ call RuntimeParameters_get("zmin", gr_bh_gpu_zmin_domain)
+ call RuntimeParameters_get("zmax", gr_bh_gpu_zmax_domain)
+
+ call RuntimeParameters_get("grav_boundary_type",  grav_boundary_type)
+ call RuntimeParameters_get("grav_boundary_type_x",grav_boundary_type_x)
+ call RuntimeParameters_get("grav_boundary_type_y",grav_boundary_type_y)
+ call RuntimeParameters_get("grav_boundary_type_z",grav_boundary_type_z)
+
+
+
+ call RuntimeParameters_get("grv_bh_gpu_PinnedCells",gr_bh_gpu_pinned_cells)
+ call RuntimeParameters_get("grv_bh_gpu_PinnedMPI",gr_bh_gpu_pinned_mpi)
+!=====================================================================!
+!============== Set Values depending on domain size ==================!
+
+
+  gr_bh_gpu_Lx = gr_bh_gpu_xmax_domain - gr_bh_gpu_xmin_domain
+  gr_bh_gpu_Ly = gr_bh_gpu_ymax_domain - gr_bh_gpu_ymin_domain
+  gr_bh_gpu_Lz = gr_bh_gpu_zmax_domain - gr_bh_gpu_zmin_domain
+
+
+  gr_bh_gpu_global_domdiagsq = (ABS(gr_bh_gpu_Lx)**2 + ABS( gr_bh_gpu_Ly)**2 + ABS( gr_bh_gpu_Lz)**2 )
+  gr_bh_gpu_EwaldSize = 1 ! ((gr_bh_gpu_EwaldFieldNx) * (gr_bh_gpu_EwaldFieldNy) *(gr_bh_gpu_EwaldFieldNz))
+
+!=====================================================================!
+!============== Create GPU MPI_Communicator ==========================!
+
+! do i = 0, gr_meshNumProcs-1
+ ! if(gr_meshMe .eq. i)then
+  !   call get_device_count(DeviceCount)
+  ! endif
+  ! call MPI_BARRIER(gr_meshComm,ierr)
+ !end do
+ if(gr_meshNumProcs .gt. 1) then
+  call GpuSupport_setDeviceMPIgroup(gr_meshNumProcs, gr_meshMe, gr_meshComm,&
+	                 gpu_commSize,gpu_rank,gpu_comm,&
+			 device_ID,DeviceCount,host_name)
+
+
+ 
+ endif
+
+ call MPI_BARRIER(gr_meshComm,ierr)
+ if(gr_meshMe .eq. 0 )write(*,*)"Communication group for GPU devices created!"
+
+ if(gr_meshNumProcs .gt. 1) then
+    do i = 0, gpu_commSize-1
+        if(gpu_rank .eq. i)then     
+             call GpuSupport_getDeviceProperties(device_ID,multiProcessorCount)
+   
+               gpu_blocks = multiProcessorCount
+
+         endif
+         call MPI_BARRIER(gpu_comm,ierr)
+     end do
+ else
+ device_ID = 0
+ call GpuSupport_getDeviceProperties(device_ID,multiProcessorCount)
+ gpu_blocks = multiProcessorCount
+ endif
+
+ call MPI_BARRIER(gr_meshComm,ierr)
+ if(gr_meshMe .eq. 0 )write(*,*)"GPU devices assigned to ranks!"
+
+ 
+!=====================================================================================!
+!============== Create Ewald Field if Periodicity is present==========================!
+ if(grav_boundary_type == "periodic") then
+  gr_bh_gpu_EwaldSize = ((gr_bh_gpu_EwaldFieldNx) * (gr_bh_gpu_EwaldFieldNy) *(gr_bh_gpu_EwaldFieldNz))
+  if (grav_boundary_type_x == "periodic")then
+      gr_bh_gpu_Periodic = gr_bh_gpu_Periodic + 1
+  endif
+  if (grav_boundary_type_y == "periodic")then
+     gr_bh_gpu_Periodic = gr_bh_gpu_Periodic + 2
+   endif
+  if (grav_boundary_type_z == "periodic")then
+     gr_bh_gpu_Periodic = gr_bh_gpu_Periodic + 4
+  endif
+ endif
+
+         
+
+
+   
+ allocate(gr_bh_gpu_EwaldField(0:gr_bh_gpu_EwaldSize-1),stat=ierr)
+ if (ierr /= 0) call Driver_abortFlash("could not allocate EwaldField in gr_bh_gpu_init.F90")
+  
+ if (gr_bh_gpu_periodic .gt. 0)then
+  call gr_bh_gpu_CalcEwaldField()
+ endif 
+
+
+
+ essentials_memory_timings=0
+ potential_memory_timings=0
+ essentials_kernel_timings=0
+ potential_kernel_timings=0
+
+ first_call = .true.
+
+
+#ifdef USE_VT
+ call VTTRACEOFF( )
+#endif
+
+return
+
+
+end subroutine gr_bh_gpu_Init
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Finalize_Buffer.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Finalize_Buffer.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Finalize_Buffer.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,47 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+#include "mpi.h"
+
+extern "C" {
+void gr_bh_gpu_Finalize_Buffer_(double ***recvbuff,double ***sendbuff,MPI_Request **R_request,MPI_Request **S_request)//
+    {
+        free(*sendbuff);
+	free(*recvbuff);
+	free(*S_request);
+	free(*R_request);
+
+	
+
+    }
+}
Index: source/Grid/GridSolvers/BHTree/GPU/Makefile
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/Makefile	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/Makefile	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,33 @@
+# Makefile for GPU-accelerated tree Poisson solver: objects from Fortran code
+#============================================================================
+#=================== BH-GPU Tree code makefile ==============================
+#============================================================================
+
+
+#=========== BH-GPU Tree code FORTRAN modules and routines===================
+Grid += gr_bh_gpu_Data.o \
+        gr_bh_gpu_CalcEwaldField.o\
+        gr_bh_gpu_GatherCells.o\
+        gr_bh_gpu_CommLocalRoots.o\
+        gr_bh_gpu_CalcExportTree.o\
+        gr_bh_gpu_CalcPot.o\
+        gr_bh_gpu_CalcPotSelf.o\
+        gr_bh_gpu_SavePot.o\
+        gr_bh_gpu_GetGridData.o\
+        gr_bh_gpu_SetGridPot.o\
+        gr_bh_gpu_CleanUp.o\
+        gr_bh_gpu_WaitForEssentialSizes.o\
+        gr_bh_gpu_ReAllocBlocks.o\
+	gr_bh_gpu_Essentials_Build_Local_Tree.o\
+	gr_bh_gpu_Essentials_Calculate_Essentials.o\
+	gr_bh_gpu_Essentials_Probe_Messages.o\
+        gr_bh_gpu_WaitForEssentialData.o
+
+
+Grid += gr_bh_gpu_Interface.o
+
+
+  
+#============================================================================      
+        
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Simple.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Simple.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Simple.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,465 @@
+/*
+!!****if* source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Simple.cu
+!!
+!! NAME
+!!  gr_bh_gpu_Potential_Simple
+!!
+!! SYNOPSIS
+!!
+!!  call gr_bh_gpu_Potential_Simple(integer(IN) :: device_ID,
+                                    integer(IN) :: gpu_blocks,
+                                    real(IN)    :: angle,
+                                    integer(IN) :: local_cell_count,  
+                                    real(IN)    :: Px_data(local_cell_count),
+                                    real(IN)    :: Py_data(local_cell_count),
+                                    real(IN)    :: Pz_data(local_cell_count),
+                                    real(IN)    :: Pm_data(local_cell_count),
+                                    real(INOUT) :: Ppot_data(local_cell_count),
+                                    real(IN)    :: domain_diagonal,
+                                    real(IN)    :: minx, 
+                                    real(IN)    :: miny, 
+                                    real(IN)    :: minz, 
+                                    real(IN)    :: maxx, 
+                                    real(IN)    :: maxy, 
+                                    real(IN)    :: maxz, 
+                                    real(IN)    :: e_grid(ewald_size)
+                                    integer(IN) :: ewald_size, 
+                                    integer(IN) :: periodics,
+                                    real(INOUT) :: memory_time, 
+                                    real(INOUT) :: kernel_time)
+!!
+!! DESCRIPTION
+!!
+!!  This routine calculates the gravitational potential for a local set of cell data. 
+!!
+!! AUTHOR
+!!
+!!  (c) 2016 Gunther Lukat 
+!!
+!!  
+!!  
+!! ARGUMENTS
+!!
+!!    device_ID        - GPU device id to use  
+!!    gpu_blocks       - number of SMPs on the device 
+!!    angle            - BH opening angle(0.1-0.9) 
+!!    local_cell_count - count of local paramesh cells   
+!!    Px_data          - X position data of local cells
+!!    Py_data          - Y position data of local cells
+!!    Pz_data          - Z position data of local cells 
+!!    Pm_data          - mass of local cells
+!!    Ppot_data        - potential of local cells (out) 
+!!    domain_diagonal  - diagonal size of the local domain
+!!    minx, miny, minz - local domain lower left corner
+!!    maxx, maxy, maxz - local domain upper right corner
+!!    e_grid           - ewald grid for periodic boundaries
+!!    ewald_size       - size of the ewald grid
+!!    periodics        - use periodic boundaries?
+!!    memory_time      - timers for memory operations
+!!    kernel_time      - timers for kernel operations
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!!
+!! Gunther Lukat, Robi Banerjee, A GPU accelerated Barnes Hut tree code for FLASH4, New Astronomy, Volume 45, May 2016, 
+!! Pages 14-28, ISSN 1384-1076, http://dx.doi.org/10.1016/j.newast.2015.10.007.
+!!  
+!!
+!!***
+*/
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+
+// GENERAL TREE BUILDING
+#include "gr_bh_gpu_Init_Kernel.cuh"
+#include "gr_bh_gpu_MakeRootKernel.cuh"
+#include "gr_bh_gpu_Summarization_Kernel.cuh"
+#include "gr_bh_gpu_TreeBuilding_Kernel.cuh"
+
+
+// FORCE CALCULATION
+#include "gr_bh_gpu_Potential_Calculate_Direkt_Kernel.cuh"
+#include "gr_bh_gpu_Potential_Calculate_ForeignTree_Kernel.cuh"
+#include "gr_bh_gpu_Potential_Calculate_Local_Kernel.cuh"
+
+
+//ESSENTIAL NODES
+#include "gr_bh_gpu_Essentials_Calculate_Nodes_Kernel.cuh"
+
+#include <thrust/device_ptr.h>
+#include <thrust/fill.h> 
+
+
+extern "C" {
+void gr_bh_gpu_Potential_Simple_(int *device_ID,int *gpu_blocks,double *angle,
+                           int *local_cell_count,  
+                           double **Px_data,
+                           double **Py_data, 
+                           double **Pz_data, 
+                           double **Pm_data,
+                           double **Ppot_data, 
+                           double *domain_diagonal,
+                           double *minx,  double *miny, double *minz, double *maxx, double *maxy, double *maxz, 
+                           double *e_grid, int *ewald_size, int *periodics,
+                           double *memory_time, double *kernel_time)
+    {
+
+  double *x_data = *Px_data;
+  double *y_data = *Py_data;
+  double *z_data = *Pz_data;
+  double *m_data = *Pm_data;
+  double *pot_data = *Ppot_data;
+ 
+	int  blocks = *gpu_blocks;
+	double *d_delta;
+	
+	int cells = *local_cell_count;
+	int periodicity = *periodics;
+	int nnodes= cells * 2;
+
+	 if (nnodes < 1024*blocks) nnodes = 1024*blocks;
+		while ((nnodes & (WARPSIZE-1)) != 0) nnodes++;
+			nnodes--;
+
+
+	double  itolsq = 1.0 /((*angle) * (*angle)) ;/// (0.5);
+	int esize =0;
+	double *gridl;
+	int  *childl;
+	double *posxl; 
+	double *posyl;     
+	double *poszl;
+    double *massl;
+    double *radii;  
+	double *gpotl;
+	
+	double *root_maxxl;
+	double *root_maxyl;
+	double *root_maxzl;
+	double *root_minxl;
+	double *root_minyl;      
+	double *root_minzl;   
+    double *all;
+
+	cudaEvent_t start, stop;
+
+	float time;
+    double bytes = 0;  
+
+    cudaSetDevice(*device_ID);
+	cudaEventCreate(&start);
+	cudaEventCreate(&stop);
+	cudaEventRecord(start, 0);
+
+
+
+//allocate
+    cudaMalloc((void **)&childl, sizeof(int) * (nnodes+1) * 8);
+    CudaTest( (char*)"gr_bh_gpu_Potential_Simple : couldn't allocate childd");
+
+	cudaMalloc((void **)&all, (sizeof(double) * (((nnodes+1)*6) + cells + 6 )));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : couldn't allocate cell arr");
+
+    cudaEventRecord(stop, 0);
+    cudaEventSynchronize(start); 
+    cudaEventSynchronize(stop);
+    cudaEventElapsedTime(&time, start, stop);
+
+    memory_time[0] += time; 
+
+
+
+	posxl 	  = all + 0 ; 
+	posyl 	  = all + (nnodes+1);     
+	poszl	  = all + ((nnodes+1)*2);  
+	massl	  = all + ((nnodes+1)*3);  
+	radii 	  = all + ((nnodes+1)*4); 
+    d_delta   = all + ((nnodes+1)*5);
+    gpotl	  = all + ((nnodes+1)*6);
+	root_maxxl= all + ((nnodes+1)*6)+cells;
+	root_maxyl= all + ((nnodes+1)*6)+cells+1;
+	root_maxzl= all + ((nnodes+1)*6)+cells+2;
+	root_minxl= all + ((nnodes+1)*6)+cells+3;
+	root_minyl= all + ((nnodes+1)*6)+cells+4;
+	root_minzl= all + ((nnodes+1)*6)+cells+5;
+
+
+
+    cudaEventRecord(start, 0);
+
+//copy to SYMBOL
+
+
+
+
+
+
+	cudaMemcpyToSymbol(posxd, &posxl, sizeof(void*));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : posxl copy to device failed");
+
+
+	cudaMemcpyToSymbol(poszd, &poszl, sizeof(void*));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : posyl copy to device failed");
+
+
+	cudaMemcpyToSymbol(posyd, &posyl, sizeof(void*));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : posyl copy to device failed");
+
+
+	cudaMemcpyToSymbol(massd, &massl, sizeof(void*));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : massl copy to device failed");
+
+
+	cudaMemcpyToSymbol(posrd, &radii, sizeof(void*));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : radii copy to device failed");
+
+
+	cudaMemcpyToSymbol(nbodiesd, &cells, sizeof(int)); 
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : nbody copy to device failed");
+	bytes += 1 * sizeof(int);
+
+	cudaMemcpyToSymbol(periodic, &periodicity, sizeof(int)); 
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : periodic copy to device failed");
+	bytes += 1 * sizeof(int);
+
+	cudaMemcpyToSymbol(nnodesd, &nnodes, sizeof(int));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : nnode copy to device failed");
+	bytes += 1 * sizeof(int);
+
+	cudaMemcpyToSymbol(itolsqd, &itolsq, sizeof(double));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : itolsq copy to device failed");
+	bytes += 1 * sizeof(double);
+
+	cudaMemcpyToSymbol(delta, &d_delta, sizeof(void*));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : delta copy to Symbol failed");
+
+
+	cudaMemcpyToSymbol(childd, &childl, sizeof(void*));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : childl copy to device failed");
+
+
+
+
+	cudaMemcpyToSymbol(grv_potd, &gpotl, sizeof(void*));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : accxl copy to device failed");
+
+
+
+
+
+	cudaMemcpyToSymbol(domdiag, domain_diagonal, sizeof(double));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : domdiag copy to device failed");
+	bytes += 1 * sizeof(double);
+
+
+
+ //memcpy
+
+	cudaMemcpy(massl, m_data, sizeof(double) * cells, cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : mass copy to device failed");
+	bytes += cells * sizeof(double);
+
+	cudaMemcpy(posxl, x_data, sizeof(double) * cells, cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : posx copy to device failed");
+	bytes += cells * sizeof(double);
+
+	cudaMemcpy(posyl, y_data, sizeof(double) * cells, cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : posy copy to device failed");
+	bytes +=cells * sizeof(double);
+
+	cudaMemcpy(poszl, z_data, sizeof(double) * cells, cudaMemcpyHostToDevice); 
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : posz copy to device failed");
+	bytes += cells * sizeof(double);
+
+
+
+
+	cudaMemcpy(root_maxyl, maxy, sizeof(double) , cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : posy copy to device failed");
+	bytes += 1 * sizeof(double);
+
+	cudaMemcpy(root_maxxl, maxx, sizeof(double), cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : posx copy to device failed");
+	bytes += 1 * sizeof(double);
+
+	cudaMemcpy(root_maxzl, maxz, sizeof(double) , cudaMemcpyHostToDevice); 
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : posz copy to device failed");
+	bytes += 1 * sizeof(double);
+
+	cudaMemcpy(root_minxl, minx, sizeof(double), cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : posx copy to device failed");
+	bytes += 1 * sizeof(double);
+             
+	cudaMemcpy(root_minyl, miny, sizeof(double) , cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : posy copy to device failed");
+	bytes += 1 * sizeof(double);
+                    
+	cudaMemcpy(root_minzl, minz, sizeof(double) , cudaMemcpyHostToDevice);
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : posz copy to device failed");
+	bytes += 1 * sizeof(double);
+
+       cudaEventRecord(stop, 0);
+       cudaEventSynchronize(start); 
+       cudaEventSynchronize(stop);
+       cudaEventElapsedTime(&time, start, stop);
+
+       memory_time[1] += time; 
+       memory_time[2] += bytes; 
+       bytes = 0;
+      
+
+
+
+            
+
+ if (periodicity != 0){
+     esize = *ewald_size;
+ if (cudaSuccess != cudaMalloc((void **)&gridl, sizeof(double) * esize)) 
+       fprintf(stderr, "could not allocate gridl with %d doubles\n",esize); 
+                         CudaTest( (char*)"couldn't allocate gridl");
+
+
+ if (cudaSuccess != cudaMemcpyToSymbol(d_grid, &gridl, sizeof(void*))) 
+                     fprintf(stderr, "gr_bh_gpu_calcforces : copying of childl to device failed\n"); 
+                     CudaTest( (char*)"gr_bh_gpu_calcforces : childl copy to device failed");
+
+
+
+ if (cudaSuccess != cudaMemcpy(gridl, e_grid, sizeof(double) * esize, cudaMemcpyHostToDevice))
+                     fprintf(stderr, "gr_bh_gpu_calcforces : copying of posy to device failed\n"); 
+                     CudaTest( (char*)"gr_bh_gpu_calcforces : posy copy to device failed");
+
+}
+
+
+
+
+
+
+
+
+ cudaEventRecord(start, 0);
+
+//memset        
+	cudaMemset((void*) d_delta, 0, sizeof(double) * (nnodes+1));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : couldn't initialize dist_com");
+
+	cudaMemset((void*) gpotl, 0, sizeof(double) * (cells));
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : couldn't initialize  gpot");
+
+
+
+        InitializationKernel<<<1, 1>>>();
+        CudaTest( (char*)"gr_bh_gpu_Potential_Simple :  InitializationKernel launch failed");
+
+
+
+
+	MakeRootKernel<<<1,1>>>(root_minxl,root_maxxl,root_minyl,root_maxyl,root_minzl,root_maxzl);
+	CudaTest( (char*)"gr_bh_gpu_Potential_Simple : MakeRootKernel launch failed");
+
+   
+
+    cudaEventRecord(stop, 0);
+    cudaEventSynchronize(start); 
+    cudaEventSynchronize(stop);
+    cudaEventElapsedTime(&time, start, stop);
+
+    kernel_time[0] += time; 
+
+    cudaEventRecord(start, 0);
+                   //printf("values: %d %d %d\n", blocks, FACTOR2, THREADS2);
+                   TreeBuildingKernel<<<blocks * FACTOR2, THREADS2>>>();
+                   CudaTest( (char*)"gr_bh_gpu_Potential_Simple : TreeBuildingKernel launch failed");
+         
+    cudaEventRecord(stop, 0);
+    cudaEventSynchronize(start); 
+    cudaEventSynchronize(stop);
+    cudaEventElapsedTime(&time, start, stop);
+
+    kernel_time[1] += time; 
+
+    cudaEventRecord(start, 0);
+
+                   SummarizationKernel<<<blocks * FACTOR3, THREADS3>>>();
+                   CudaTest( (char*)"gr_bh_gpu_Potential_Simple :SummarizationKernel launch failed");
+
+    cudaEventRecord(stop, 0);
+    cudaEventSynchronize(start); 
+    cudaEventSynchronize(stop);
+    cudaEventElapsedTime(&time, start, stop);
+
+    kernel_time[2] += time; 
+
+    cudaEventRecord(start, 0);
+                   
+
+ int n_blocks = cells/256;
+
+if(cells % 256) 
+   n_blocks++; 
+  
+  
+
+  
+ 
+   ForceCalculationKernel_periodic_self_deep<<<blocks * FACTOR8, 256>>>(cells);
+   CudaTest( (char*)"gr_bh_gpu_Potential_Simple : ForceCalculationKernel_periodic_self_deep launch failed");
+ 
+
+  
+
+    cudaEventRecord(stop, 0);
+    cudaEventSynchronize(start); 
+    cudaEventSynchronize(stop);
+    cudaEventElapsedTime(&time, start, stop);
+
+    kernel_time[3] += time; 
+
+    cudaEventRecord(start, 0);
+
+
+cudaMemcpy(pot_data, gpotl, sizeof(double) * cells, cudaMemcpyDeviceToHost);
+CudaTest( (char*)"gr_bh_gpu_Potential_Simple :gpot copy to host failed");
+     
+    cudaEventRecord(stop, 0);
+    cudaEventSynchronize(start); 
+    cudaEventSynchronize(stop);
+    cudaEventElapsedTime(&time, start, stop);
+
+    memory_time[3] += time; 
+    memory_time[4] += cells *  sizeof(double); 
+       
+
+
+
+cudaEventRecord(start, 0);
+
+
+  cudaFree(childl);
+   cudaFree(all);
+
+ if (periodicity != 0){
+ cudaFree(gridl);
+} 
+         
+    cudaEventRecord(stop, 0);
+    cudaEventSynchronize(start); 
+    cudaEventSynchronize(stop);
+    cudaEventElapsedTime(&time, start, stop);
+
+    memory_time[0] += time; 
+
+   
+  cudaEventDestroy(start);
+  cudaEventDestroy(stop);       
+
+
+ }
+}
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Calculate_Essentials.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Calculate_Essentials.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Calculate_Essentials.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,94 @@
+!!****if* BHTree/GPU/gr_bh_gpu_Essentials_Calculate_Essentials
+!!
+!! NAME
+!!
+!!  gr_bh_gpu_Essentials_Calculate_Essentials
+!!
+!! SYNOPSIS
+!!
+!!  call gr_bh_gpu_Essentials_Calculate_Essentials(integer(inout) :: offset,
+!!                                                 integer(inout) :: essentials_count,
+!!                                                 integer(inout) :: tree_depth,
+!!                                                 real(inout) :: roots,
+!!                                                 type(c_ptr)(inout) :: all_doubles,
+!!                                                 type(c_ptr)(inout) :: all_ints,
+!!                                                 type(c_ptr)(inout) :: essential_nodes,
+!!                                                 type(c_ptr)(inout) :: essential_blocks)
+!!
+!! DESCRIPTION
+!!
+!! Calculates the essential nodes for the PIDs domain on a GPU device.  
+!!
+!! ARGUMENTS
+!!
+!!   offset : data offset for c arrays
+!!
+!!   essentials_count : number of essentials nodes found for the PIDs domain
+!!
+!!   tree_depth : the depth of the GPU-BH-Tree
+!!
+!!   roots : array of position and mass data for domains on all PIDs.
+!!
+!!   all_doubles : C-Pointer to device memory for allocated doubles
+!!
+!!   all_ints : C-Pointer to device memory for allocated integers
+!!
+!!   essential_nodes : C-Pointer to device memory for essential nodes
+!!
+!!   essential_blocks : C-Pointer to device memory for essential blocks
+!!
+!!
+!!
+!!
+!!***
+
+
+subroutine gr_bh_gpu_Essentials_Calculate_Essentials(offset,essentials_count,tree_depth,roots,all_doubles,all_ints,&
+						    essential_nodes,essential_blocks)
+ use Driver_interface, ONLY : Driver_abortFlash
+ use Grid_data, ONLY : gr_meshMe,gr_meshNumProcs, gr_meshComm
+ use gr_bh_gpu_data, ONLY :  gpu_blocks, gr_bh_gpu_global_domdiagsq,&
+                            gr_bh_gpu_Mac_Angle,essentials_kernel_timings,essentials_memory_timings,&
+			    gr_bh_gpu_node_count,gr_bh_gpu_local_cell_count,device_ID
+
+
+use gr_bh_gpu_Interface, ONLY: gr_bh_gpu_Essentials_Calc_Essentials_Wrap
+
+use, intrinsic :: iso_c_binding
+
+use Timers_interface, ONLY : Timers_start, Timers_stop
+implicit none
+
+#include "Flash_mpi.h"
+
+ 
+
+  integer, intent (inout)         :: offset,essentials_count,tree_depth
+  real, intent (inout)            :: roots(0:4*gr_meshNumProcs)
+  type(c_ptr), intent (inout)     :: all_doubles,all_ints
+  type(c_ptr), intent (inout)     :: essential_nodes,essential_blocks
+
+
+
+
+             
+        call gr_bh_gpu_Essentials_Calc_Essentials_Wrap(device_ID,gpu_blocks,gr_bh_gpu_local_cell_count, &
+                                                    gr_bh_gpu_node_count, &
+                                                    tree_depth,&
+                                                    roots,&
+                                                    gr_bh_gpu_global_domdiagsq,&
+                                                    gr_bh_gpu_Mac_Angle,&
+                                                    essentials_count,&
+                                                    all_doubles,&
+                                                    all_ints,&
+                                                    offset,&
+                                                    essentials_memory_timings,& 
+					            essentials_kernel_timings,&
+						    essential_nodes,essential_blocks)
+  !write(*,*)gr_meshMe, "found", essentials_count
+               
+               
+
+
+return
+end subroutine gr_bh_gpu_Essentials_Calculate_Essentials
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Wait_And_Copy.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Wait_And_Copy.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Wait_And_Copy.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,113 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+#include "mpi.h"
+
+
+extern "C" 
+{
+void gr_bh_gpu_Essentials_Wait_And_Copy_(double ***receivebuff,
+				         MPI_Request **request,
+				         double **hP_X, 
+				         double **hP_Y, 
+				         double **hP_Z, 
+				         double **hP_M,
+					 int *local_cell_count,
+                                         int *ME_RANK,
+                                         int *COMM_SIZE, bool *pinned_mpi)//
+{
+	
+
+
+	double *h_X = *hP_X; 
+	double *h_Y = *hP_Y;  
+	double *h_Z = *hP_Z; 
+	double *h_M = *hP_M; 
+	int ess_count=0; // number of essentials
+	int senderID= 0;//ID of sender of essentials
+	int msg_count = 0;//Count of messages
+	//int error= 0;//well....
+	MPI_Status mstat;//temporary MPI_Status to retrieve message size and sender ID
+	int dstIdx = 0;//memcpy indices for destination
+	//int srcIdx = 0;//memcpy indices for src
+ 
+
+	dstIdx = *local_cell_count;	
+
+
+	while (msg_count <= *COMM_SIZE)
+	{
+		/*Wait for any message to finish*/		
+		MPI_Waitany(*COMM_SIZE,*request,&senderID,&mstat);
+		/*Get the size of the message: unsend messaged have size 0*/
+		MPI_Get_count(&mstat,MPI_DOUBLE_PRECISION,&ess_count); 
+		/*Get the sender ID:  -1 if unsend message*/                            
+		senderID = mstat.MPI_SOURCE; 
+
+ 
+		/*If the message is valid: i.e. Sender is not -1 and message size is not 0*/
+               
+		if(senderID >= 0 && ess_count > 1 )
+                {	/*one data field is 1/4 of the total message with:*/
+			/* [X_0..X_N,Y_0..Y_N,Z_0..Z_N,M_0..M_N]          */
+			ess_count *= 0.25;
+#ifdef GPU_BH_DEBUG
+                        printf("%d will copy buffer from sender %d with size %d\n",*ME_RANK,senderID, ess_count);
+#endif
+                        memcpy(&h_X[dstIdx],&(*receivebuff)[senderID][0],sizeof(double) * ess_count);
+			memcpy(&h_Y[dstIdx],&(*receivebuff)[senderID][ess_count],sizeof(double) * ess_count);
+			memcpy(&h_Z[dstIdx],&(*receivebuff)[senderID][(ess_count*2)],sizeof(double) * ess_count);
+			memcpy(&h_M[dstIdx],&(*receivebuff)[senderID][(ess_count*3)],sizeof(double) * ess_count);
+
+			dstIdx += ess_count;
+			/*Free sendbuffer*/
+                        if(*pinned_mpi){
+                           cudaFreeHost( (*receivebuff)[senderID] );
+                           CudaTest( (char*)"gr_bh_gpu_Essentials_Wait_And_Copy: couldn't free buffer");
+				}
+			if(! *pinned_mpi)
+			   free( (*receivebuff)[senderID] );
+		        
+			(*receivebuff)[senderID]=NULL;
+
+                       
+		}
+	/*update message counter*/
+	msg_count++;      
+	}//done with all mesages
+
+
+}
+}
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_SetGridPot.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_SetGridPot.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_SetGridPot.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,106 @@
+!!****if* source/Grid/GridSolvers/BHTree/GPU/gr_gpu_setGridPot
+!!
+!! NAME
+!!  gr_bh_gpu_SetGridPot
+!!
+!! SYNOPSIS
+!!
+!!  gr_bh_gpu_SetGridPot(Integer (IN)::offset, 
+!!                       Integer (IN)::blockid, 
+!!                       Integer (IN)::counter, 
+!!                       Integer (IN)::ipoisvar, 
+!!                       Integer (IN)::num_cells)
+!!
+!! DESCRIPTION
+!!
+!!  Writes calculated force for all cells in the block(blockid) to the solution vector.
+!!  
+!!  
+!! ARGUMENTS
+!!  
+!!  offset    - offset for the C array
+!!
+!!  blockid   - Paramesh blockid of the Leaf block
+!!
+!!  counter   - Number of current included Leafblocks in the local tree
+!!
+!!  ipoisvar  - the index for the force variable
+!!
+!!  num_cells  - the number of cells 
+!!   
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!!  
+!!
+!!***
+subroutine gr_bh_gpu_SetGridPot(offset,blockid,counter,ipoisvar,num_cells)
+
+use Grid_interface, ONLY :  Grid_getBlkIndexLimits,Grid_getBlkPtr
+use gr_bh_gpu_data, ONLY: gr_bh_gpu_GravFac, gr_bh_gpu_p_data
+use gr_bh_gpu_Interface, ONLY: gr_bh_gpu_Save_Grid_Pot
+ use Grid_data, ONLY : gr_meshMe,gr_meshNumProcs, gr_meshComm
+ implicit none
+
+#include "Flash.h"
+#include "Flash_mpi.h"
+#include "constants.h"
+
+integer, intent(in) :: blockid
+integer, intent(in) :: ipoisvar
+integer, intent(in) :: counter
+integer, intent(in) :: num_cells
+integer, intent(in) :: offset
+
+
+
+
+integer, dimension(LOW:HIGH,MDIM) :: blkLimits,blkLimitsGC
+integer               :: i,j,k,cellid
+
+real, DIMENSION(:,:,:,:), POINTER :: solnData
+real :: gpot
+
+ call Grid_getBlkPtr(blockid,solnData,CENTER)
+ call Grid_getBlkIndexLimits(blockid,blkLimits,blkLimitsGC)
+
+               do k = 1,NXB
+                 do j = 1,NYB
+                    do i = 1,NZB
+
+                  if(NXB==8)then
+                     if (j <= 4 .and. i <= 4) then
+                        cellid = (i-1) + ((j-1)*4) + ((k-1)*4*4) + ((counter)*(NXB*NYB*NZB))
+                  
+                      else  if (j > 4 .and. i > 4) then
+                       cellid = (i-5) + ((j-5)*4) + ((k-1)*4*4) + 128 + ((counter)*(NXB*NYB*NZB))
+                    
+                      else if ( j > 4 .and. i <= 4) then
+                        cellid = (i-1) + ((j-5)*4) + ((k-1)*4*4) + 256 + ((counter)*(NXB*NYB*NZB))
+                 
+                      else if ( j <= 4 .and. i > 4) then
+                        cellid = (i-5) + ((j-1)*4) + ((k-1)*4*4) + 384 + ((counter)*(NXB*NYB*NZB))
+                      end if
+                   else
+                   cellid = (i-1) + ((j-1)*NYB) + ((k-1)*NYB*NXB) + ((counter)*(NXB*NYB*NZB))
+                   endif
+         
+
+            call  gr_bh_gpu_Save_Grid_Pot(offset + cellid,gr_bh_gpu_p_data,gpot)
+
+            !gpot = accx(cellid)
+            solnData(ipoisvar, i+blkLimits(LOW,IAXIS)-1, j+blkLimits(LOW,JAXIS)-1, k+blkLimits(LOW,KAXIS)-1) =  -gr_bh_gpu_GravFac * gpot
+
+
+               ! accx(cellid)
+               ! if(gr_meshMe .eq. 12)write(*,*)"setting",accx(cellid),"from",gr_meshMe
+                    !if(accx(cellid).eq. 0.0)write(*,*)"ERR"
+                    end do !k
+                 end do !j
+               end do !i
+ call Grid_releaseBlkPtr(blockid,solnData,CENTER)
+return
+end subroutine  gr_bh_gpu_SetGridPot
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Init_Kernel.cuh
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Init_Kernel.cuh	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Init_Kernel.cuh	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,6 @@
+#ifndef __InitializationKernel_CUH__
+#define __InitializationKernel_CUH__
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+
+__global__ void InitializationKernel();
+#endif
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Rec_Periodic_Kernel.cuh
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Rec_Periodic_Kernel.cuh	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Rec_Periodic_Kernel.cuh	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,12 @@
+#ifndef __EWALD_REC_PERIODIC_CUH__
+#define __EWALD_REC_PERIODIC_CUH__
+
+
+#include "gr_bh_gpu_Kernel_Constants.cuh"       
+
+
+
+
+__global__ void ewald_rec_periodic_all(double *t_grid_x,double *t_grid_y, double *t_grid_z,double *grid);
+
+#endif
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Real_Periodic_Kernel.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Real_Periodic_Kernel.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Real_Periodic_Kernel.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,57 @@
+#include "gr_bh_gpu_Ewald_Real_Periodic_Kernel.cuh"
+
+
+
+__global__ void ewald_real_periodic_all(double *t_grid_x,double *t_grid_y,double *t_grid_z,double *grid)
+{
+
+
+
+
+int xx = threadIdx.x + blockIdx.x * blockDim.x;
+int inc = blockDim.x * gridDim.x;
+
+
+
+
+double x, y,z,rni,t_real,xni,yni,zni;
+int id,ni,nj,nk;
+
+
+
+ while(xx< (d_NX * d_NY *d_NZ))       
+    {
+ 
+     id =  (xx);
+    x = t_grid_x[(xx)];
+    y = t_grid_y[(xx)];
+    z = t_grid_z[(xx)];
+    t_real = 0.0;
+    
+
+  for(  ni = -d_es_nrx; ni < d_es_nrx; ni++){
+            for(  nj = -d_es_nry; nj < d_es_nry; nj++){
+              for(  nk = -d_es_nrz; nk < d_es_nrz;  nk++){
+              //! terms with non-negligible contributions must lie inside ellipse
+                
+                     xni =x + ni *  d_Lx_per;
+                     yni =y + nj *  d_Ly_per;
+                     zni =z + nk *  d_Lz_per;
+                     rni = sqrt(xni*xni + yni*yni + zni*zni);
+
+                  
+                    t_real += erfc(d_ewald_alpha* rni) / (rni + 1.0E-99);
+                 
+                
+                
+              }
+            }
+          }
+
+
+grid[id]= t_real;
+//printf("ww %e\n",t_real);
+xx += inc;
+     }
+  
+}
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Receive_Message.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Receive_Message.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Receive_Message.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,71 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+#include "mpi.h"
+
+
+extern "C" {
+void gr_bh_gpu_Essentials_Receive_Message_(double ***recv_buff,MPI_Request **request,int *count, 
+					   int *sender,int *tag, int *comm, bool *pinned_mpi)//
+    {
+     // int error  = 0;
+     // error = error;
+      if(*sender >= 0){
+     
+
+
+      if(*pinned_mpi)
+        {
+        cudaMallocHost((void**) &((*recv_buff)[*sender]), sizeof(double) * *count);
+        CudaTest((char*)"gr_bh_gpu_Essentials_Receive_Message.cu: Malloc receive buffer FAILED");
+        }
+ 
+      if(! *pinned_mpi)
+ 	((*recv_buff)[*sender]) = (double*) malloc(sizeof(double)* *count);
+
+
+#ifdef GPU_BH_DEBUG
+printf("Rank %d will receive %d doubles from rank %d\n",*tag,*count,*sender);
+#endif
+      
+      MPI_Comm gr_meshComm = MPI_Comm_f2c(*comm);
+      MPI_Irecv( (*recv_buff)[*sender],*count, MPI_DOUBLE_PRECISION,*sender, *tag, gr_meshComm, &((*request)[*sender])/*&req*/);
+
+
+
+
+      }
+      
+
+    }
+}
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Temp_Field_Kernel.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Temp_Field_Kernel.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Temp_Field_Kernel.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,34 @@
+#include "gr_bh_gpu_Ewald_Temp_Field_Kernel.cuh"
+
+__global__ void gen_temp_field(double *t_gridx,double *t_gridy,double *t_gridz) 
+{
+ int zz,xx,yy,id;
+
+
+      for(xx = threadIdx.x + blockDim.x * blockIdx.x;xx < d_NX; xx += blockDim.x * gridDim.x) 
+        { //printf("XX %d,YY %d\n",xx,yy);
+         
+        for (yy = threadIdx.y + blockDim.y * blockIdx.y;yy < d_NY; yy += blockDim.y * gridDim.y) 
+        {
+           
+  for (zz =0; zz < d_NZ; zz++) 
+             {
+              
+              
+             
+
+             id = (xx + yy * (d_NX) + zz * (d_NX) * (d_NY));
+            
+             t_gridx[id]=(xx) * ewald_xmax / (d_NX-1);
+             t_gridy[id]=(yy) * ewald_ymax / (d_NY-1);
+             t_gridz[id]=(zz) * ewald_zmax / (d_NZ-1);
+               // if(xx == 0)
+               // printf("XX %e,YY %e, ZZ %e\n",t_gridx[id],t_gridy[id], t_gridz[id]);
+             }
+        }
+      
+      
+ }
+}
+
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Save_Grid_Pot.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Save_Grid_Pot.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Save_Grid_Pot.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,41 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+
+extern "C" {
+void gr_bh_gpu_Save_Grid_Pot_(int *idx,double **c_arr,double *target_val)//
+    {   
+          *target_val = (*c_arr)[*idx];
+   
+    }
+}
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Calculate_Local_Kernel.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Calculate_Local_Kernel.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Calculate_Local_Kernel.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,234 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+#include "gr_bh_gpu_Potential_Calculate_Local_Kernel.cuh"
+
+
+
+
+
+__device__ __forceinline__ double interp_ewald_noref(double x, double y, double z)
+{
+
+
+  double dx, dy, dz, p, q, r;
+  int i, j, k;
+  int  ip1,  jp1, kp1;
+
+
+  dx = ewald_xmax / (d_NX-1);
+  i = (int)floor(x/dx);
+/*
+  if ((i .LT. 0) .or. (i .GE. gr_bhEwaldFieldNx)) then
+    print *, i, x, dx, gr_bhEwaldXMax, gr_bhEwaldFieldNx
+    call Driver_abortFlash("FATAL: Ewald field i out of limits")
+  endif
+*/
+  dy = ewald_ymax / (d_NY-1);
+  j = (int)floor(y/dy);
+/*
+  if ((j .LT. 0) .or. (j .GE. gr_bhEwaldFieldNy)) then
+    call Driver_abortFlash("FATAL: Ewald field j out of limits")
+  endif
+*/
+  dz = ewald_zmax / (d_NZ-1);
+  k = (int)floor(z/dz);
+/*
+  if ((k .LT. 0) .or. (k .GE. gr_bhEwaldFieldNz)) then
+    call Driver_abortFlash("FATAL: Ewald field k out of limits")
+  endif
+  */
+
+
+ 
+  if (i == d_NX-1){
+    ip1 = i;
+    
+  }else{ 
+    ip1 = i+1;
+  }
+  if (j == d_NY-1) {
+    jp1 = j;
+  }else {
+    jp1 = j+1;
+ }
+  if (k == d_NZ-1) {
+    kp1 = k;
+  }else {
+    kp1 = k+1;
+}
+
+  p = (x - i*dx)/dx;
+  q = (y - j*dy)/dy;
+  r = (z - k*dz)/dz;
+
+
+
+ 
+  
+// linear interpolation
+ 
+
+   return ( 
+       (1.0 - p)*(1.0 - q)*(1.0 - r) * d_grid[((i)   + (j)   * (d_NX) + (k)  * (d_NX) * (d_NY))] +
+       (      p)*(1.0 - q)*(1.0 - r) * d_grid[((ip1) + (j)   * (d_NX) + (k)  * (d_NX) * (d_NY))] +
+       (1.0 - p)*(      q)*(1.0 - r) * d_grid[((i)   + (jp1) * (d_NX) + (k)  * (d_NX) * (d_NY))] +
+       (      p)*(      q)*(1.0 - r) * d_grid[((ip1) + (jp1) * (d_NX) + (k)  * (d_NX) * (d_NY))] +
+       (1.0 - p)*(1.0 - q)*(      r) * d_grid[((i)   + (j)   * (d_NX) + (kp1)* (d_NX) * (d_NY))] +
+       (      p)*(1.0 - q)*(      r) * d_grid[((ip1) + (j)   * (d_NX) + (kp1)* (d_NX) * (d_NY))] +
+       (1.0 - p)*(      q)*(      r) * d_grid[((i)   + (jp1) * (d_NX) + (kp1)* (d_NX) * (d_NY))] +
+       (      p)*(      q)*(      r) * d_grid[((ip1) + (jp1) * (d_NX) + (kp1)* (d_NX) * (d_NY))]  
+       );
+
+
+
+
+
+
+}
+
+
+
+
+__global__ void
+#if __CUDA_ARCH__ >= 300
+__launch_bounds__(512,4)
+#else 
+__launch_bounds__(256,6)
+#endif
+ForceCalculationKernel_periodic_self_deep(int n_old)
+{
+  register int  j, k,n, depth, warpID, laneID ,diff,ch;
+  register double px, py, pz, ax, dx, dy, dz, tmp, size;
+#if __CUDA_ARCH__ >= 300
+//  int stack[(MAXDEPTH * 8)];
+__shared__ volatile int stack[(MAXDEPTH * 8) * (512 /WARPSIZE)]; //shared memory of 8 times MAXDEPTH for each warp  
+#else 
+
+ __shared__ volatile int stack[(MAXDEPTH * 8) * (256 /WARPSIZE)]; //shared memory of 8 times MAXDEPTH for each warp
+
+#endif  
+ 
+    laneID = threadIdx.x % WARPSIZE;
+
+//#if __CUDA_ARCH__ >= 300
+//    warpID = 0;
+//#else //no shfl intrinsic
+    warpID = threadIdx.x / WARPSIZE; // WARP ID
+//#endif    
+
+    diff = warpID * (MAXDEPTH * 8);
+
+
+
+    // iterate over all bodies assigned to thread
+    for (k = threadIdx.x + blockIdx.x * blockDim.x; k < n_old; k += blockDim.x * gridDim.x) {
+      //i = sortd[k];  // get permuted/sorted index
+      // cache position info
+      px = posxd[k];
+      py = posyd[k];
+      pz = poszd[k];
+
+      ax = 0;
+    
+      // initialize iteration stack, i.e., push root node onto stack
+      
+      if (laneID==0) {
+          stack[diff+1]=nnodesd;
+          
+          
+      }
+      depth = 1;
+     
+        
+       while (depth > 0) 
+           {
+         // #if __CUDA_ARCH__ >= 300
+           //   n = __shfl(stack[depth+diff], 0);
+         // #else
+              n = stack[depth+diff];
+         // #endif    
+                 dx = posxd[n] - px;
+                 dy = posyd[n] - py;
+                 dz = poszd[n] - pz;
+                 size = posrd[n] + delta[n]; 
+  
+             if (periodic == 7 ||  periodic == 1)
+                dx = min (min(abs(dx - (d_grv_bhLx)), abs(dx +(d_grv_bhLx))), abs(dx));
+   
+             if ( periodic == 7 ||  periodic == 3 || periodic == 2 )
+                dy = min (min(abs(dy - (d_grv_bhLy)), abs(dy +(d_grv_bhLy))), abs(dy));
+
+             if ( periodic == 7 ||  periodic == 4 ||  periodic == 6  ||  periodic == 5  ) 
+                  dz = min (min(abs(dz - (d_grv_bhLz)), abs(dz +(d_grv_bhLz))), abs(dz));
+        
+                 
+                  tmp = dx*dx + (dy*dy + (dz*dz));
+                  
+              if(n < nbodiesd || __all(tmp >size) )
+              {
+                if(n != k)
+                 {                       
+                  if ( periodic !=0)
+                   {  
+                  ax += massd[n] * interp_ewald_noref(fabs(dx),fabs(dy),fabs(dz));
+                  }
+                  else{                    
+                  tmp = rsqrt(tmp); 
+                  ax += massd[n]*tmp;
+                    }
+                 }
+                 depth--;
+               
+               }else{  
+                       depth--;  
+                        n *= 8;
+                        for (j = 0; j < 8 ; j++)
+                             {
+                             ch = childd[n + j];
+                              if(ch >= 0 )
+                                 {
+                                  depth++;
+                                  if (laneID==0) 
+                                      stack[depth+diff]=ch;
+                                 }
+                              }
+                    
+                     }
+                  
+             
+           }//end  while (depth > 0) 
+           grv_potd[k] += ax;
+ 
+           
+        }
+ }
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_CleanUp_Device_Wrap.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_CleanUp_Device_Wrap.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_CleanUp_Device_Wrap.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,65 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+
+extern "C" {
+void gr_bh_gpu_Essentials_CleanUp_Device_Wrap_( int *device_ID,double **all_doubles,int **all_ints,
+						int **essential_nodes,
+						int **essential_blocks, int *memory_timer)//
+{
+
+
+ cudaSetDevice(*device_ID);
+ float time = 0;
+ cudaEvent_t start, stop;  
+ cudaEventCreate(&start);
+ cudaEventCreate(&stop);
+ cudaEventRecord(start, 0);
+
+	cudaFree(*all_doubles);
+
+        cudaFree(*all_ints);
+
+	cudaFreeHost(*essential_nodes);
+	cudaFreeHost(*essential_blocks);
+
+    cudaEventRecord(stop, 0);
+    cudaEventSynchronize(start); 
+    cudaEventSynchronize(stop);
+    cudaEventElapsedTime(&time, start, stop);            
+    memory_timer[0] += time;
+
+}
+}
+
+
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Reallocate_CellData.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Reallocate_CellData.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Reallocate_CellData.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,111 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+
+
+extern "C" 
+{
+void gr_bh_gpu_Essentials_Reallocate_CellData_(double **x_data, 
+				double **y_data, 
+				double **z_data, 
+				double **m_data,
+				int *local_cell_count,
+				int *total_cell_count,
+                                bool *pinnedCELL,
+                                int *me_rank
+                                )//
+{
+
+
+int total_cells_bytes  = *total_cell_count * sizeof(double);//bytesize for all data cells
+int local_cells_bytes  = *local_cell_count * sizeof(double);//bytesize for local data cells 
+
+        if(*pinnedCELL){
+#ifdef GPU_BH_DEBUG
+printf("Rank %d will reallcoate pinned memory with 4 * %d doubles.\n",*me_rank,*total_cell_count );
+#endif
+        /*Allocate pinned host memory work space for X,Y,Z,Mass*/ 
+        double *x_dat;
+	double *y_dat;
+	double *z_dat;
+	double *m_dat;
+
+          
+	cudaMallocHost((void**)&x_dat, total_cells_bytes );
+        CudaTest((char*)"gr_bh_gpu_Essentials_Reallocate_CellData.cu: malloc pinned x_data failed");
+        memcpy(x_dat,*x_data, local_cells_bytes);
+        cudaFreeHost(*x_data);
+        *x_data = x_dat;
+   
+        cudaMallocHost((void**)&y_dat, total_cells_bytes );
+        CudaTest((char*)"gr_bh_gpu_Essentials_Reallocate_CellData.cu: malloc pinned y_data failed");
+        memcpy(y_dat,*y_data, local_cells_bytes);
+	cudaFreeHost(*y_data);
+ 	*y_data = y_dat;
+
+        cudaMallocHost((void**)&z_dat, total_cells_bytes );
+        CudaTest((char*)"gr_bh_gpu_Essentials_Reallocate_CellData.cu: malloc pinned z_data failed");
+        memcpy(z_dat,*z_data, local_cells_bytes);
+	cudaFreeHost(*z_data);
+ 	*z_data = z_dat;
+       
+        cudaMallocHost((void**)&m_dat, total_cells_bytes );
+        CudaTest((char*)"gr_bh_gpu_Essentials_Reallocate_CellData.cu: malloc pinned m_data failed");
+        memcpy(m_dat,*m_data, local_cells_bytes);
+	cudaFreeHost(*m_data);
+	 *m_data = m_dat;
+
+	}
+        if(! *pinnedCELL)
+          {
+
+#ifdef GPU_BH_DEBUG
+printf("Rank %d will reallcoate pageable memory with 4 * %d doubles.\n",*me_rank,*total_cell_count );
+#endif
+                *x_data = (double *) realloc(*x_data, sizeof(double) * *total_cell_count );
+ 		*y_data = (double *) realloc(*y_data, sizeof(double) * *total_cell_count );
+ 		*z_data = (double *) realloc(*z_data, sizeof(double) * *total_cell_count );
+ 		*m_data = (double *) realloc(*m_data, sizeof(double) * *total_cell_count );
+ 		
+
+
+          }
+
+
+
+}
+}
+
+
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Calculate_Local_Kernel.cuh
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Calculate_Local_Kernel.cuh	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Calculate_Local_Kernel.cuh	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,44 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+
+#ifndef __POT_LOC_TREE_CUH__
+#define __POT_LOC_TREE_CUH__
+
+
+#include "gr_bh_gpu_Kernel_Constants.cuh"  
+//#include "gr_bh_gpu_Periodic_InterpolateInField.cuh"
+
+__global__
+void ForceCalculationKernel_periodic_self_deep(int n_old);
+
+#endif
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Data.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Data.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Data.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,152 @@
+!!****if* source/Grid/GridSolvers/BHTree/gr_bh_gpu_Data
+!!
+!! NAME
+!!  gr_gpu_data
+!!
+!! SYNOPSIS
+!!
+!!  use gr_gpu_data
+!!
+!! DESCRIPTION
+!!
+!!  Variable and routine declarations/definitions for the gpu tree solver.
+!!  
+!! 
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!!  
+!!
+!!***
+
+
+
+
+
+module gr_bh_gpu_Data
+!use Grid_data, ONLY : gr_meshMe,gr_meshNumProcs, gr_meshComm
+use, intrinsic :: iso_c_binding
+implicit none
+
+#include "constants.h"
+
+save
+!====================================================
+! DATAFIELDS
+!====================================================
+
+
+real :: gr_bh_gpu_roots_own(0:11)
+real, allocatable :: gr_bh_gpu_roots_received(:)
+
+integer                 :: gpu_rank,gpu_comm,gpu_commSize !MPI FOR GPU : One comm grp for each GPU 
+integer 		:: DeviceCount,device_ID
+
+
+
+!========================
+!GPU timer fields  
+!========================
+real                    :: essentials_memory_timings(0:4)
+                                 !0 = allocate/deallocate
+                                 !1 = HtoD
+				 !2 = Bytes
+				 !3 = DtoH
+ 				 !4 = Bytes
+real                    :: potential_memory_timings(0:4)
+                                 !0 = allocate/deallocate
+                                 !1 = HtoD
+				 !2 = Bytes
+				 !3 = DtoH
+ 				 !4 = Bytes
+
+real                    :: essentials_kernel_timings(0:3)
+                                 !0 = init_kernels
+                                 !1 = tree_build
+				 !2 = tree_summ
+				 !3 = calc_essentials
+
+real                    :: potential_kernel_timings(0:3)
+                                 !0 = init_kernels
+                                 !1 = tree_build
+				 !2 = tree_summ
+				 !3 = calc_force
+
+                                 
+
+logical :: first_call
+
+
+integer, parameter :: gr_bh_gpu_MAX_GPU_CELLS = 30000000
+integer       :: gr_bh_gpu_EwaldSize
+real          :: gr_bh_gpu_GravFac   !Gravitational constant
+integer       :: gr_bh_gpu_idensvar    !mass index for solution vector
+integer       :: gr_bh_gpu_ipoisvar    !force index for solution vector
+real          :: gr_bh_gpu_Mac_Angle
+real          :: gr_bh_gpu_local_domdiagsq
+
+!GLOBAL DOMAIN
+real         :: gr_bh_gpu_xmin_domain
+real         :: gr_bh_gpu_xmax_domain
+real         :: gr_bh_gpu_ymin_domain
+real         :: gr_bh_gpu_ymax_domain
+real         :: gr_bh_gpu_zmin_domain
+real         :: gr_bh_gpu_zmax_domain
+
+!LOCAL DOMAIN
+real        :: gr_bh_gpu_local_xmax
+real         :: gr_bh_gpu_local_xmin
+real        :: gr_bh_gpu_local_ymax
+real        :: gr_bh_gpu_local_ymin
+real         :: gr_bh_gpu_local_zmax
+real         :: gr_bh_gpu_local_zmin
+
+
+real, dimension(MDIM) :: gr_bh_gpu_mcs
+integer :: gpu_blocks ! multiprocessor count on GPU
+integer :: gr_bh_gpu_periodic
+
+!============================
+!Global Fields for EwaldCorrection
+!allocated here or in init
+!========================
+
+integer              :: gr_bh_gpu_EwaldFieldNx,gr_bh_gpu_EwaldFieldNy
+integer               :: gr_bh_gpu_EwaldFieldNz,gr_bh_gpu_EwaldSeriesN,gr_bh_gpu_EwaldSIsoFac
+real                  :: gr_bh_gpu_ewald_xmax,gr_bh_gpu_ewald_ymax ,gr_bh_gpu_ewald_zmax
+real                :: gr_bh_gpu_Lx, gr_bh_gpu_Ly, gr_bh_gpu_Lz
+real,     allocatable :: gr_bh_gpu_EwaldField(:)
+
+!============================
+!Temporary Fields for Tree
+!allocated during each step and deallocated each step
+!========================
+
+ 
+
+
+  real :: gr_bh_gpu_global_domdiagsq
+
+   
+ ! integer ,save:: gr_bh_gpu_LeafblockCount  !tmp variable stores the count of blocks when parsing the paramesh grid
+
+  integer ::gr_bh_gpu_message_count
+  integer ::gr_bh_gpu_essentials_count
+  integer ::gr_bh_gpu_local_cell_count
+  integer ::gr_bh_gpu_total_cell_count
+  integer ::gr_bh_gpu_node_count
+  
+  type(c_ptr)  :: send_buffer, recv_buffer
+  type(c_ptr)  :: send_request, recv_request
+  type(c_ptr)  :: gr_bh_gpu_x_data ! cell data x 
+  type(c_ptr)  :: gr_bh_gpu_y_data ! cell data y
+  type(c_ptr)  :: gr_bh_gpu_z_data ! cell data z
+  type(c_ptr)  :: gr_bh_gpu_m_data ! cell data mass
+  type(c_ptr)  :: gr_bh_gpu_p_data ! cell data pot
+
+  logical ::gr_bh_gpu_pinned_mpi,gr_bh_gpu_pinned_cells
+ 
+end module gr_bh_gpu_Data
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Clean.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Clean.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Clean.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,119 @@
+
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+
+extern "C" {
+void gr_bh_gpu_Potential_Clean_(int *device_ID,int *local_cell_count, 
+                                     int *other,
+                                     int *periodicity,
+                                     double *memory_time,
+                                     double **gpot_sol,
+                                     double **root_maxxd,
+                                     double **root_maxyd,
+                                     double **root_maxzd,
+                                     double **root_minxd,
+                                     double **root_minyd,
+                                     double **root_minzd, 
+                                     double **xd, 
+                                     double **yd,
+                                     double **zd,
+                                     double **gpot_sold,
+                                     double **e_gridd
+                                     )
+    {
+  float time;
+  
+  cudaSetDevice(*device_ID);
+  cudaEvent_t start, stop;
+  cudaEventCreate(&start);
+  cudaEventCreate(&stop);
+  cudaEventRecord(start, 0);
+
+cudaMemcpy(*gpot_sol, *gpot_sold, sizeof(double) * (*local_cell_count), cudaMemcpyDeviceToHost); 
+CudaTest( (char*)"gpot copy to host failed");
+     
+    cudaEventRecord(stop, 0);
+    cudaEventSynchronize(start); 
+    cudaEventSynchronize(stop);
+    cudaEventElapsedTime(&time, start, stop);
+
+    memory_time[3] += time; 
+    memory_time[4] += sizeof(double) * (*local_cell_count); 
+       
+    cudaEventRecord(start, 0);
+
+
+
+
+
+
+
+    cudaFree(*root_maxxd);
+    cudaFree(*root_maxyd);
+    cudaFree(*root_maxzd);
+    cudaFree(*root_minxd);
+    cudaFree(*root_minyd);
+    cudaFree(*root_minzd);
+    cudaFree(*gpot_sold);
+   
+if(*other == 1)
+  {
+     cudaFree(*xd);
+     cudaFree(*yd);
+     cudaFree(*zd);           
+  }
+
+                
+
+ if (*periodicity != 0){
+ cudaFree(*e_gridd);
+} 
+         
+    cudaEventRecord(stop, 0);
+    cudaEventSynchronize(start); 
+    cudaEventSynchronize(stop);
+    cudaEventElapsedTime(&time, start, stop);
+
+    memory_time[0] += time; 
+
+   
+         
+       
+  cudaEventDestroy(start);
+  cudaEventDestroy(stop); 
+
+ }
+}
+
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Summarization_Kernel.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Summarization_Kernel.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Summarization_Kernel.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,96 @@
+#include "gr_bh_gpu_Summarization_Kernel.cuh"
+
+
+
+
+__global__
+__launch_bounds__(THREADS3, FACTOR3)
+void SummarizationKernel()
+{
+  register int i, j, k, ch, inc, missing, cnt, bottom;
+  register double m, cm, px, py, pz,pcx,pcy,pcz;
+  __shared__ volatile int child[THREADS3 * 8];
+
+  bottom = bottomd;
+  inc = blockDim.x * gridDim.x;
+  k = (bottom & (-WARPSIZE)) + threadIdx.x + blockIdx.x * blockDim.x;  // align to warp size
+
+  if (k < bottom) k += inc;
+// printf("root is: %e, %e, %e \n",posxd[nnodesd],posyd[nnodesd],poszd[nnodesd] );
+  missing = 0;
+  // iterate over all cells assigned to thread
+  while (k <= nnodesd) {
+    if (missing == 0) {
+      // new cell, so initialize
+      cm = 0.0;
+      px = 0.0;
+      py = 0.0;
+      pz = 0.0;
+      cnt = 0;
+      j = 0;
+      for (i = 0; i < 8; i++) {
+        ch = childd[k*8+i];
+        if (ch >= 0) {
+          
+          child[missing*THREADS3+threadIdx.x] = ch;  // cache missing children
+          m = massd[ch];
+          missing++;
+          if (m >= 0.0) {
+            // child is ready
+            missing--;
+            
+            // add child's contribution
+            cm += m;
+            px += posxd[ch] * m;
+            py += posyd[ch] * m;
+            pz += poszd[ch] * m;
+          }
+          j++;
+        }
+      }
+      cnt += j;
+    }
+
+    if (missing != 0) {
+      do {
+        // poll missing child
+        ch = child[(missing-1)*THREADS3+threadIdx.x];
+        m = massd[ch];
+        if (m >= 0.0) {
+          // child is now ready
+          missing--;
+          
+          // add child's contribution
+          cm += m;
+          px += posxd[ch] * m;
+          py += posyd[ch] * m;
+          pz += poszd[ch] * m;
+ 
+        }
+        // repeat until we are done or child is not ready
+      } while ((m >= 0.0) && (missing != 0));
+    }
+
+    if (missing == 0) {
+      // all children are ready, so store computed information
+     
+      m = 1.0 / cm;
+      pcx = posxd[k];
+      pcy = posyd[k];
+      pcz = poszd[k];
+      posxd[k] = px * m;
+      posyd[k] = py * m;
+      poszd[k] = pz * m;
+      delta[k]= ((pcx-posxd[k]) *(pcx-posxd[k]) +
+                 (pcy-posyd[k]) *(pcy-posyd[k]) +
+                 (pcz-poszd[k]) *(pcz-poszd[k]));
+      __threadfence();  // make sure data are visible before setting mass
+      massd[k] = cm;
+     // if(k == nnodesd)printf("SUM FORCE is: %e, %e, %e ,%e\n",posxd[nnodesd],posyd[nnodesd],poszd[nnodesd],massd[nnodesd] );
+      
+      k += inc;  // move on to next cell
+    }
+  }
+}
+
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Clear_Buffers.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Clear_Buffers.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Clear_Buffers.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,92 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+#include "mpi.h"
+
+
+extern "C" {
+void gr_bh_gpu_Clear_Buffers_(  double ***recvbuff,
+				double ***sendbuff,
+				MPI_Request **R_request,
+				MPI_Request **S_request,
+				int *COMM_SIZE,bool *pinned_mpi
+				)//
+    {
+
+
+     
+
+         if(*COMM_SIZE > 1){
+
+	MPI_Waitall(*COMM_SIZE, *R_request,MPI_STATUSES_IGNORE);
+	MPI_Waitall(*COMM_SIZE, *S_request,MPI_STATUSES_IGNORE);
+  /**RESET MPI REQUEST AND FREE BUFFER MEMORY**/
+	for(int j = 0; j < *COMM_SIZE; j++)
+	{
+		if((*S_request)[j] != MPI_REQUEST_NULL)
+                   (*S_request)[j] = MPI_REQUEST_NULL;
+
+		if((*R_request)[j] != MPI_REQUEST_NULL)
+                   (*R_request)[j] = MPI_REQUEST_NULL;
+
+		if((*sendbuff)[j]!=NULL)
+		{       
+                        if(*pinned_mpi)
+                           cudaFreeHost((*sendbuff)[j] );
+
+                        if(!*pinned_mpi)                     
+	       		   free( (*sendbuff)[j] );
+
+			(*sendbuff)[j]=NULL;
+		}
+                   
+
+		if((*recvbuff)[j]!=NULL)
+		{
+		    if(*pinned_mpi)
+                        cudaFreeHost((*recvbuff)[j] );
+
+	            if(!*pinned_mpi) 
+		        free( (*recvbuff)[j] );
+
+		      (*recvbuff)[j]=NULL;
+		}
+	}
+
+ /**RESET MPI REQUEST AND FREE BUFFER MEMORY**/
+    }
+
+
+    }
+}
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_SetAndCommExportTree.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_SetAndCommExportTree.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_SetAndCommExportTree.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,284 @@
+!!****if* BHTree/GPU/gr_bh_gpu_SetAndCommExportTree
+!!
+!! NAME
+!!
+!!  gr_bh_gpu_SetAndCommExportTree
+!!
+!! SYNOPSIS
+!!
+!!  call gr_bh_gpu_SetAndCommExportTree()
+!!
+!! DESCRIPTION
+!!
+!!
+!! ARGUMENTS
+!!
+!!   No arguments
+!!
+!! AUTOGENROBODOC
+!!
+!!
+!!***
+
+
+
+!====================================================
+!Private :: gr_gpu_find_and_comm_neighbors
+!====================================================
+!! NAME
+!!
+!! gr_gpu_find_and_comm_neighbors
+!!
+!! SYNOPSIS
+!!
+!!  gr_gpu_find_and_comm_neighbors()
+!!                         
+!!
+!! DESCRIPTION
+!!
+!!  This routine finds and communicates the neighbours for every cell such that a Local essential tree can be build.
+!!
+!!
+!! ARGUMENTS
+!!  
+!!
+!!
+!! NOTES
+!! 
+!! A major part of the time is used up during communication. Communication and calculation time is overlapped using non-blocking MPI calls.
+!! 
+!! BUGFIX:: setting MPI FLAG --mca mpi_leave_pinned = 0.
+!!            
+!!            Althoug buffers are deallocated after calling MPI_WAIT, 
+!!            this roputine crashes when using registered memory.  
+!!
+!! SEE ALSO 
+!!
+!! fill_buffers()
+!!***
+!====================================================
+subroutine gr_bh_gpu_SetAndCommExportTree()
+ use Grid_data, ONLY : gr_meshMe,gr_meshNumProcs, gr_meshComm
+ use Timers_interface, ONLY : Timers_start, Timers_stop
+ use Driver_interface, ONLY : Driver_abortFlash
+
+ use gr_bh_gpu_data, ONLY : gpu_comm,gpu_rank,gpu_commSize,gr_bh_gpu_recieve_buffers, gr_bh_gpu_recieve_request,&
+                            gr_bh_gpu_localblocks, gr_bh_gpu_send_buffers,gr_bh_gpu_recieve_buffers,&
+                            gr_bh_gpu_recieve_request,gr_bh_gpu_send_request,&
+                            gr_bh_gpu_localblocks,&
+                             gr_bh_gpu_xmin_domain,gr_bh_gpu_ymin_domain,gr_bh_gpu_zmin_domain,&
+                             gr_bh_gpu_xmax_domain,gr_bh_gpu_ymax_domain,gr_bh_gpu_zmax_domain,&
+                             gr_bh_gpu_roots_recieved,gr_bh_gpu_cells_count,&
+                             gr_bh_gpu_recieve_request_sizes, gr_bh_gpu_send_request_sizes,recieved_sizes,sended_sizes
+
+use gr_bh_gpu_interface, ONLY :  gr_bh_gpu_CalcExportTree
+implicit none
+#include "Flash_mpi.h"
+#include "Flash.h"
+ 
+  integer :: l,m,n,i,i2
+  integer :: sender, reciever
+  integer :: recieve_blockid,size1,size2,start
+  integer :: ierr
+  integer :: sendsize,recievesize
+
+
+ 
+ 
+
+  real,allocatable :: tmp_arr(:)
+
+  
+
+  integer :: mstat(MPI_STATUS_SIZE)
+  integer :: istat,nblocks
+  integer :: bsize
+  integer :: buffer_send_size
+  integer :: buffer_index_size
+ 
+
+
+
+   ! VT_ON()
+
+ 
+
+
+ 
+
+
+  !--------------------------------------------!
+  !------prepare array for gpu ----------------!
+  !--------------------------------------------!
+
+call Timers_start("NB_PROBE")
+!if(gr_bh_gpu_localblocks%num_cells .gt. 1) then
+size2 = 0
+
+
+recieved_sizes=0
+
+do sender = 0, gr_meshNumProcs-1
+  gr_bh_gpu_recieve_request(sender)=MPI_REQUEST_NULL 
+   gr_bh_gpu_recieve_request_sizes(sender)=MPI_REQUEST_NULL   
+  
+  if(sender .ne. gr_meshMe) then
+    allocate(gr_bh_gpu_recieve_buffers(sender)%sendbuff(0:(gr_bh_gpu_cells_count(sender)*4)-1),stat = istat)
+    
+
+    call MPI_IRECV(recieved_sizes(sender),1 ,MPI_INT, sender, gr_meshMe, gr_meshComm,gr_bh_gpu_recieve_request_sizes(sender), ierr)
+               
+    call MPI_IRECV(gr_bh_gpu_recieve_buffers(sender)%sendbuff,(gr_bh_gpu_cells_count(sender)*4) ,FLASH_REAL, sender,&
+               & gr_meshMe, gr_meshComm,gr_bh_gpu_recieve_request(sender), ierr)
+  !  recieved_sizes(sender)=bsize
+  ! size2 = size2+gr_bh_gpu_roots_recieved(sender)
+  
+ end if
+end do
+
+call Timers_stop("NB_PROBE")
+!------------------------------------!
+!------calculate neighbors on gpu----!
+!------------------------------------!
+call Timers_start("NB_SEARCH")
+ do i = 0,gpu_commSize-1
+     
+   if(gpu_rank .eq. i) then
+
+      call gr_bh_gpu_CalcExportTree()
+      
+   else
+     call Mpi_Barrier(gpu_comm,ierr)
+   end if
+ end do
+call Timers_stop("NB_SEARCH")
+
+
+  
+
+
+call Timers_start("NB_WAIT")
+ m = 1
+ sender =0
+
+
+ do while (m .le. gr_meshNumProcs-1 )
+    call MPI_WAITANY(gr_meshNumProcs,gr_bh_gpu_recieve_request_sizes,sender,mstat,ierr)!!find first recieved message pair
+      sender = sender-1
+      m = m+1
+    ! write(*,*),gr_meshMe,"recieved",recieved_sizes(sender)/4,"from",sender
+end do !(m .le. gr_meshNumProcs-1 )
+
+call Timers_stop("NB_WAIT")
+
+
+
+
+call Timers_start("NB_ALLOC")
+if(gr_bh_gpu_localblocks%num_cells .gt. 0)then
+! !----------------------------------------------!
+! !------Fill LET with recieved data ------------!
+! !----------------------------------------------!  
+!!write(*,*),gr_meshMe,"recieved nr items", size2/4,"and i have", gr_bh_gpu_localblocks%num_cells, "items"
+              size2 = sum(recieved_sizes)/4
+              size1 =gr_bh_gpu_localblocks%num_cells
+
+              allocate (tmp_arr(0:(size1)-1),stat = istat)
+!!X
+              tmp_arr(0:size1-1) = gr_bh_gpu_localblocks%x(0:size1-1) 
+              deallocate(gr_bh_gpu_localblocks%x)
+              allocate(gr_bh_gpu_localblocks%x(0:(size2)-1 + size1),stat = istat) 
+              gr_bh_gpu_localblocks%x(0:size1-1) = tmp_arr(0:size1-1) 
+!!Y              
+!            
+              tmp_arr(0:size1-1) = gr_bh_gpu_localblocks%y(0:size1-1) 
+              deallocate(gr_bh_gpu_localblocks%y) 
+              allocate(gr_bh_gpu_localblocks%y(0:(size2)-1 + size1),stat = istat)
+              gr_bh_gpu_localblocks%y(0:size1-1) = tmp_arr(0:size1-1)
+!!z             
+              tmp_arr(0:size1-1) = gr_bh_gpu_localblocks%z(0:size1-1) 
+              deallocate(gr_bh_gpu_localblocks%z) 
+              allocate(gr_bh_gpu_localblocks%z(0:(size2)-1 + size1),stat = istat)
+              gr_bh_gpu_localblocks%z(0:size1-1) = tmp_arr(0:size1-1)
+!!mass
+              tmp_arr(0:size1-1) = gr_bh_gpu_localblocks%mass(0:size1-1) 
+              deallocate(gr_bh_gpu_localblocks%mass) 
+              allocate(gr_bh_gpu_localblocks%mass(0:(size2)-1 + size1),stat = istat)
+              gr_bh_gpu_localblocks%mass(0:size1-1) = tmp_arr(0:size1-1)
+! 
+!               
+              deallocate(tmp_arr) 
+
+!      
+          gr_bh_gpu_localblocks%num_cells = gr_bh_gpu_localblocks%num_cells +size2
+endif
+call Timers_stop("NB_ALLOC")
+
+
+
+ 
+
+call Timers_start("NB_WAIT2")
+ m = 1
+ sender =0
+nblocks=size1
+ do while (m .le. gr_meshNumProcs-1 )
+    call MPI_WAITANY(gr_meshNumProcs,gr_bh_gpu_recieve_request,sender,mstat,ierr)!!find first recieved message pair
+      sender = sender-1
+      m = m+1
+      if(sender .ne. gr_meshMe)then
+         
+         size2 =  recieved_sizes(sender)/4
+
+         if(size2 .gt. 0)then
+          ! write(*,*)gr_meshMe,"Will start to recieve ",size2,"from ",sender
+           gr_bh_gpu_localblocks%x(nblocks:nblocks+size2-1) = gr_bh_gpu_recieve_buffers(sender)%sendbuff(0:size2 -1)
+           gr_bh_gpu_localblocks%y(nblocks:nblocks+size2-1) = gr_bh_gpu_recieve_buffers(sender)%sendbuff(size2:(size2*2) -1)
+           gr_bh_gpu_localblocks%z(nblocks:nblocks+size2-1) = gr_bh_gpu_recieve_buffers(sender)%sendbuff(size2*2:(size2*3) -1)
+           gr_bh_gpu_localblocks%mass(nblocks:nblocks+size2-1) = gr_bh_gpu_recieve_buffers(sender)%sendbuff(size2*3:(size2*4) -1)
+           nblocks = nblocks+size2
+         endif
+         
+!   !   write(*,*)gr_meshMe,"recieved ",size2,"from ",sender
+!           gr_bh_gpu_localblocks%x(nblocks:nblocks+size2-1) = gr_bh_gpu_recieve_buffers(sender)%sendbuff(0:size2 -1)
+!           gr_bh_gpu_localblocks%y(nblocks:nblocks+size2-1) = gr_bh_gpu_recieve_buffers(sender)%sendbuff(size2:(size2*2) -1)
+!           gr_bh_gpu_localblocks%z(nblocks:nblocks+size2-1) = gr_bh_gpu_recieve_buffers(sender)%sendbuff(size2*2:(size2*3) -1)
+!           gr_bh_gpu_localblocks%mass(nblocks:nblocks+size2-1) = gr_bh_gpu_recieve_buffers(sender)%sendbuff(size2*3:(size2*4) -1)
+!           nblocks = nblocks+size2
+           if(allocated(gr_bh_gpu_recieve_buffers(sender)%sendbuff))then
+              deallocate(gr_bh_gpu_recieve_buffers(sender)%sendbuff)
+           endif
+
+ 
+  
+ 
+  end if!(sender .ne. gr_meshMe)
+
+  end do !(m .le. gr_meshNumProcs-1 )
+
+
+ 
+
+         
+         
+
+
+
+call Timers_stop("NB_WAIT2")      
+
+
+
+
+
+
+
+
+
+
+
+
+
+return
+
+end subroutine  gr_bh_gpu_SetAndCommExportTree
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_WaitForEssentialData.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_WaitForEssentialData.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_WaitForEssentialData.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,59 @@
+!!****if* BHTree/GPU/gr_bh_gpu_WaitForEssentialData
+!!
+!! NAME
+!!
+!!  gr_bh_gpu_WaitForEssentialData
+!!
+!! SYNOPSIS
+!!
+!!  call gr_bh_gpu_WaitForEssentialData()
+!!
+!! DESCRIPTION
+!! 
+!! This routine calls the C function gr_bh_gpu_Essentials_Wait_And_Copy(). 
+!! The foreign subtrees are recieved with mpi and copied from the MPI buffer to pinned or paged memory.
+!! 
+!! ARGUMENTS
+!!
+!!   No arguments
+!!
+!! AUTOGENROBODOC
+!!
+!!
+!!***
+
+
+subroutine gr_bh_gpu_WaitForEssentialData()
+ use Grid_data, ONLY : gr_meshMe,gr_meshNumProcs, gr_meshComm
+
+
+
+ use gr_bh_gpu_data, ONLY : recv_buffer,recv_request,gr_bh_gpu_local_cell_count,&
+				gr_bh_gpu_pinned_mpi,gr_bh_gpu_pinned_cells, gr_bh_gpu_x_data,gr_bh_gpu_y_data,&
+				gr_bh_gpu_z_data,gr_bh_gpu_m_data
+
+ use gr_bh_gpu_interface, ONLY: gr_bh_gpu_Essentials_Wait_And_Copy
+ use, intrinsic :: iso_c_binding
+implicit none
+#include "Flash_mpi.h"
+#include "Flash.h"
+ 
+
+
+
+
+  call gr_bh_gpu_Essentials_Wait_And_Copy(recv_buffer,&
+				         recv_request,&
+ 					 gr_bh_gpu_x_data,&
+					 gr_bh_gpu_y_data,&
+					 gr_bh_gpu_z_data,&
+					 gr_bh_gpu_m_data,&
+					 gr_bh_gpu_local_cell_count, &
+                                         gr_meshMe,&
+                                         gr_meshNumProcs,gr_bh_gpu_pinned_mpi)
+ 
+
+return
+
+end subroutine  gr_bh_gpu_WaitForEssentialData
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Send_Essentials_Wrap.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Send_Essentials_Wrap.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Send_Essentials_Wrap.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,141 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+#include "mpi.h"
+
+
+
+extern "C" 
+{
+void gr_bh_gpu_Essentials_Send_Essentials_Wrap_(double ***sendbuff,
+				               MPI_Request **request,
+				               double *tx, 
+				               double *ty, 
+				               double *tz, 
+				               double *tm,
+				               int *msg_size,
+				               int *top_IDX,
+				               int *receiver,
+                                               int *ME_RANK,
+                                               int *COMM_SIZE,
+				               int *comm,
+				               int **essential_nodes_pin, bool *pinned_mpi)//
+{
+	int idx;
+        int T_IDX;
+        int T_FLAG;
+        int *essential_nodes = *essential_nodes_pin;
+        
+
+        MPI_Comm gr_meshComm = MPI_Comm_f2c(*comm);
+
+   //=======FIND AN INDEX FOR AN ALREADY FINISHED COMM=====!
+           T_FLAG = 0;
+	   T_IDX = MPI_UNDEFINED;
+
+	   MPI_Testany(*COMM_SIZE, *request, &T_IDX, &T_FLAG, MPI_STATUS_IGNORE);
+/*
+            !============= ONE INDEX IS FREE =====================!
+            !
+            ! CLEAR THE OLD BUFFER AND ALLOCATE THE NEW BUFFER
+            !
+            !=====================================================!
+*/
+
+	if( ! (T_FLAG && T_IDX != MPI_UNDEFINED) )
+	{
+
+
+		*top_IDX=*top_IDX + 1;
+		T_IDX = *top_IDX;
+
+	}
+
+	if((*sendbuff)[T_IDX]!= NULL) // assure buffer is empty      
+                 {             
+                   if(! *pinned_mpi)
+			free((*sendbuff)[T_IDX]);
+
+		   if(*pinned_mpi)
+			{
+                         cudaFreeHost((*sendbuff)[T_IDX]);
+			 CudaTest((char*)"gr_bh_gpu_Essentials_Send_Essentials_Wrap.cu: Free cuda pinned memory failed");
+			}
+		}
+
+
+
+                  if(*pinned_mpi)
+		     {
+                     cudaMallocHost((void**) &((*sendbuff)[T_IDX]), sizeof(double) * (*msg_size * 4));
+ 		     CudaTest((char*)"gr_bh_gpu_Essentials_Send_Essentials_Wrap.cu: allocate cuda pinned memory failed");
+		     }
+		  if(! *pinned_mpi)
+                    ((*sendbuff)[T_IDX]) = (double *) malloc(sizeof(double) * (*msg_size * 4));  
+
+    
+
+      
+           //!==================== FILL THE COMMUNICATION BUFFER ==========================!
+             for(int i = 0; i < (*msg_size); i++)
+		{
+                idx = essential_nodes[i];
+
+	       
+                (*sendbuff)[T_IDX][                  i]                = tx[idx];
+		(*sendbuff)[T_IDX][ *msg_size      + i]                = ty[idx];
+		(*sendbuff)[T_IDX][(*msg_size * 2) + i]                = tz[idx];
+		(*sendbuff)[T_IDX][(*msg_size * 3) + i]                = tm[idx];
+
+		}
+		
+             // !==============================================================================!
+
+
+		(*request)[T_IDX]=MPI_REQUEST_NULL; 
+		MPI_Isend((*sendbuff)[T_IDX],
+					*msg_size*4,
+					MPI_DOUBLE_PRECISION,
+					*receiver,
+					*receiver,
+					gr_meshComm,
+					&((*request)[T_IDX]));
+
+
+
+           
+
+}
+}
+
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Build_Local_Tree.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Build_Local_Tree.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Build_Local_Tree.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,116 @@
+!!****if* BHTree/GPU/gr_bh_gpu_Essentials_Build_Local_Tree
+!!
+!! NAME
+!!
+!!  gr_bh_gpu_Essentials_Build_Local_Tree
+!!
+!! SYNOPSIS
+!!
+!!  call gr_bh_gpu_Essentials_Build_Local_Tree(real , allocatable(inout) :: tx,
+!!                                             real , allocatable(inout) :: ty,
+!!                                             real , allocatable(inout) :: tz,
+!!                                             real , allocatable(inout) :: tm,
+!!                                             real (inout) :: roots,
+!!                                             type(c_ptr)(inout) :: essential_nodes,
+!!                                             type(c_ptr)(inout) :: essential_blocks,
+!!                                             integer (inout) :: tree_depth,
+!!                                             type( c_ptr )(inout) :: all_ints,
+!!                                             type( c_ptr )(inout) :: all_doubles)
+!!
+!! DESCRIPTION
+!!
+!! Builds a local BH-Tree on the GPU device.
+!!
+!! ARGUMENTS
+!!
+!!   tx,ty,tz,tm : position and mass data for the tree
+!!
+!!   roots : array of position and mass data for domains on all PIDs.
+!!
+!!   essential_nodes : C-Pointer to device memory for essential nodes
+!!
+!!   essential_blocks : C-Pointer to device memory for essential blocks
+!!
+!!   tree_depth : depth of the local BH tree
+!!
+!!   all_doubles : C-Pointer to device memory for allocated doubles
+!!
+!!   all_ints : C-Pointer to device memory for allocated integers 
+!!
+!! 
+!!
+!!
+!!***
+
+
+subroutine gr_bh_gpu_Essentials_Build_Local_Tree(tx,ty,tz,tm,&
+						roots,&
+						essential_nodes,essential_blocks,&
+						tree_depth,all_ints,all_doubles)
+ use Driver_interface, ONLY : Driver_abortFlash
+ use Grid_data, ONLY : gr_meshMe,gr_meshNumProcs, gr_meshComm
+ use gr_bh_gpu_data, ONLY : gpu_blocks,&
+                            gr_bh_gpu_Mac_Angle,gr_bh_gpu_periodic,&
+                            gr_bh_gpu_global_domdiagsq,&
+                            gr_bh_gpu_xmax_domain, gr_bh_gpu_xmin_domain, &
+                            gr_bh_gpu_ymax_domain, gr_bh_gpu_ymin_domain,&
+                            gr_bh_gpu_zmax_domain, gr_bh_gpu_zmin_domain,&
+                            essentials_kernel_timings,essentials_memory_timings,&
+			    gr_bh_gpu_local_cell_count,gr_bh_gpu_node_count,&
+			    gr_bh_gpu_x_data,gr_bh_gpu_y_data,gr_bh_gpu_z_data,gr_bh_gpu_m_data,&
+                            device_ID
+
+use gr_bh_gpu_Interface, ONLY: gr_bh_gpu_Essentials_Build_Init_Wrap
+use, intrinsic :: iso_c_binding
+
+use Timers_interface, ONLY : Timers_start, Timers_stop
+implicit none
+
+#include "Flash_mpi.h"
+
+ 
+  integer , intent (inout):: tree_depth
+  real , allocatable, intent (inout)::  tx(:)
+  real , allocatable, intent (inout)::  ty(:)
+  real , allocatable, intent (inout)::  tz(:)
+  real , allocatable, intent (inout)::  tm(:)
+  real , intent (inout):: roots(0:4*gr_meshNumProcs)
+  type( c_ptr ), intent (inout) :: all_ints
+  type( c_ptr ), intent (inout) :: all_doubles
+  !integer,  intent (inout):: essential_nodes(0:(gr_bh_gpu_local_cell_count +1))
+  type(c_ptr), intent (inout)     :: essential_nodes,essential_blocks
+
+
+
+
+allocate(tx(0:gr_bh_gpu_node_count+1))
+allocate(ty(0:gr_bh_gpu_node_count+1))
+allocate(tz(0:gr_bh_gpu_node_count+1))
+allocate(tm(0:gr_bh_gpu_node_count+1))
+
+      call gr_bh_gpu_Essentials_Build_Init_Wrap(device_ID,gpu_blocks,&
+                                            gr_bh_gpu_local_cell_count, &
+                                             gr_bh_gpu_node_count, &
+                                             gr_bh_gpu_Mac_Angle,&
+                                             gr_bh_gpu_global_domdiagsq,&
+                                            gr_bh_gpu_x_data,&
+                                            gr_bh_gpu_y_data,&
+                                            gr_bh_gpu_z_data,&
+                                            gr_bh_gpu_m_data,&
+                                             gr_bh_gpu_xmax_domain, gr_bh_gpu_xmin_domain, &
+                                             gr_bh_gpu_ymax_domain, gr_bh_gpu_ymin_domain,&
+                                             gr_bh_gpu_zmax_domain, gr_bh_gpu_zmin_domain,&      
+                                             tree_depth,&
+                                             gr_bh_gpu_periodic,&
+                                             roots,&
+                                             gr_meshNumProcs,&
+                                             essentials_memory_timings,& 
+                                            essentials_kernel_timings,&
+                                            tx,ty,tz,tm,&
+                                            essential_nodes,essential_blocks,&
+                                            all_ints, all_doubles)
+
+
+return
+end subroutine gr_bh_gpu_Essentials_Build_Local_Tree
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Summarization_Kernel.cuh
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Summarization_Kernel.cuh	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Summarization_Kernel.cuh	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,6 @@
+#ifndef __SUMM_KERNEL_CUH__
+#define __SUMM_KERNEL_CUH__
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+__global__
+void SummarizationKernel();
+#endif
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Extrapolate_Kernel.cuh
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Extrapolate_Kernel.cuh	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Extrapolate_Kernel.cuh	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,15 @@
+#ifndef __EWALD_EXTRAPOLATE_CUH__
+#define __EWALD_EXTRAPOLATE_CUH__
+
+
+#include "gr_bh_gpu_Kernel_Constants.cuh"       
+
+__device__ double atomicAdd(double* address, double val);
+
+__global__ void ewald_extrapolate_real(double x,double y, double z,double *cell);
+
+
+__global__ void ewald_extrapolate_rec(double x,double y, double z,double *cell);
+
+#endif
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Kernel_Constants.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Kernel_Constants.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Kernel_Constants.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,29 @@
+#include "gr_bh_gpu_Kernel_Constants.cuh" 
+
+__device__ volatile int bottomd;
+__device__ volatile int maxdepthd;
+__device__ volatile int blkcntd;
+__constant__ int nnodesd,nbodiesd;
+__constant__ volatile double *posxd, *posyd, *poszd;
+__constant__ volatile double *posrd;
+__constant__ volatile double *massd;
+__constant__ volatile double *grv_potd;
+__constant__ volatile double *delta;
+__constant__ volatile int   *childd;
+__constant__ double  itolsqd;
+__constant__ double  G;
+__device__ volatile double domdiag;
+__device__ volatile double radiusd, radiusd2;
+__constant__ volatile double *maxxd, *maxyd, *maxzd, *minxd, *minyd, *minzd;
+/********EWALD CONSTANTS************/
+__constant__ int periodic;
+__constant__ double d_ewald_alpha;
+__constant__ double d_Ln;
+__constant__ volatile double d_grv_bhLx, d_grv_bhLy,d_grv_bhLz;
+__constant__ double *d_grid;
+__constant__ double ewald_xmax,ewald_ymax,ewald_zmax;
+__constant__ int d_NX,d_NY,d_NZ;
+__constant__ int d_es_nrx, d_es_nry, d_es_nrz;
+__constant__ double d_Lx_per,d_Ly_per,d_Lz_per;
+
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Calculate_ForeignTree_Kernel.cuh
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Calculate_ForeignTree_Kernel.cuh	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Calculate_ForeignTree_Kernel.cuh	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,44 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+
+#ifndef __POT_FOREIGN_TREE_CUH__
+#define __POT_FOREIGN_TREE_CUH__
+
+
+#include "gr_bh_gpu_Kernel_Constants.cuh"  
+//#include "gr_bh_gpu_Periodic_InterpolateInField.cuh"
+
+__global__
+void ForceCalculationKernel_periodic_step_deep(int n_old,double *selfx, double *selfy, double *selfz);
+
+#endif
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Probe_Messages.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Probe_Messages.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Probe_Messages.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,85 @@
+!!****if* BHTree/GPU/gr_bh_gpu_Essentials_Probe_Messages
+!!
+!! NAME
+!!
+!!  gr_bh_gpu_Essentials_Probe_Messages
+!!
+!! SYNOPSIS
+!!
+!!  call gr_bh_gpu_Essentials_Probe_Messages(logical(inout) :: flag,
+!!                                           integer(inout) :: sender_id,
+!!                                           integer(inout) :: message_size)
+!!
+!! DESCRIPTION
+!!
+!! Test if there is any pending MPI message. Probe the message and return the message size and the MPI rank of the sender. 
+!!
+!! ARGUMENTS
+!!
+!!   flag : True if there is a message.
+!!
+!!   sender_id : The rank of the sender
+!!
+!!   message_size : Size of the pending message 
+!!
+!! 
+!!
+!!
+!!***
+
+
+subroutine gr_bh_gpu_Essentials_Probe_Messages(flag,sender_ID,message_size)
+use Grid_data,ONLY : gr_meshNumProcs,gr_meshMe,gr_meshComm
+
+ 
+use, intrinsic :: iso_c_binding
+
+implicit none 
+
+#include "Flash_mpi.h"
+#include "Flash.h"
+
+logical, intent(inout) :: flag
+integer, intent(inout) :: sender_ID
+integer, intent(inout) :: message_size
+integer :: mstat(MPI_STATUS_SIZE),ierr
+	
+	!RESET VALUES
+	flag = .false.
+	sender_ID=-1
+	message_size=0
+
+	!Is there any message in the queue ??!
+ 	!write(*,*)gr_meshMe, "will probe "
+        call MPI_Iprobe(MPI_ANY_SOURCE,&
+		gr_meshMe,&
+		gr_meshComm,&
+		flag,&
+		mstat,&
+		ierr)
+	!There is one message ready to receive
+	if(flag)then
+		!Get the message size
+		call MPI_Get_count(mstat,&
+				FLASH_REAL,&
+				message_size,&
+				ierr)
+		!Get the ID of the sender                            
+		sender_ID = mstat(MPI_SOURCE)
+		!Update the total count of essentials
+  		!gr_bh_gpu_essentials_count = gr_bh_gpu_essentials_count + (message_size * 0.25)
+		!Receive the message in C_CODE and allocate pinned memory as a buffer
+
+
+
+
+
+
+	endif
+
+
+
+
+return
+
+end subroutine  gr_bh_gpu_Essentials_Probe_Messages
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Calc_Essentials_Wrap.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Calc_Essentials_Wrap.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Calc_Essentials_Wrap.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,175 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+
+
+//ESSENTIAL NODES
+#include "gr_bh_gpu_Essentials_Calculate_Nodes_Kernel.cuh" 
+
+
+extern "C" {
+void gr_bh_gpu_Essentials_Calc_Essentials_Wrap_( int *device_ID,   int *gpu_blocks,
+						    int *cell_count,
+                                                    int *node_count,
+                                                    int *depth,
+                                                    double *roots,
+                                                    double *domain_diagonal,double *angle,
+                                                    int *size,
+                                                    double **all_doubles,
+						    int **all_ints,
+                                                    int *offset,
+                                                    double *memory_timings, 
+						    double *kernel_timings,
+						    int **essential_nodes_pinned, int **essential_counter)
+    {
+
+                           
+
+ 
+ int n_blocks = *gpu_blocks * FACTOR8;
+// int nbodies = *cell_count;
+ int nnodes = *node_count;
+ int *essential_nodes = *essential_nodes_pinned;
+ int *essentials = (*all_ints) + ((nnodes+1) * 8) + n_blocks; // *essential_nodes_dev;
+ int *cnt_dev    = (*all_ints) + ((nnodes+1) * 8);            //*essential_counter_dev;
+ int *res        = (*all_ints) + ((nnodes+1) * 8) + n_blocks + (nnodes+1); //*res_dev;
+
+ double mac_angle = 1.0 /((*angle) * (*angle));       
+ double  nodediag = *domain_diagonal * mac_angle;
+ int maxdepth = *depth;               
+ 
+ int loopdepth = 1;
+ int breaking = 0;
+ double bytes = n_blocks * sizeof(int);
+ float time = 0;
+ 
+ cudaSetDevice(*device_ID);
+ CudaTest((char*)"gr_bh_gpu_Essentials_Calc_Essentials_Wrap: setting device_ID failed ");
+
+  
+ cudaEvent_t start, stop;  
+ cudaEventCreate(&start);
+ cudaEventCreate(&stop);
+ 
+
+ double *roots_device = (*all_doubles) + 0;
+
+
+	cudaEventRecord(start, 0);
+
+	NeighborsKernel_Force_assure<<<n_blocks,768>>>(essentials);
+	CudaTest( (char*)"gr_bh_gpu_Essentials_Calc_Essentials_Wrap: kernel assure launch failed");
+
+	cudaEventRecord(stop, 0);
+        cudaEventSynchronize(start); 
+        cudaEventSynchronize(stop);
+        cudaEventElapsedTime(&time, start, stop);            
+        kernel_timings[3] += time;
+
+	while(loopdepth <= maxdepth)
+	{
+		breaking = 0;
+		cudaEventRecord(start, 0);
+
+		NeighborsKernel_Force_nodes<<<n_blocks,512>>>(loopdepth,nodediag,roots_device,*offset,essentials,cnt_dev);
+		CudaTest((char*)"gr_bh_gpu_Essentials_Calc_Essentials_Wrap: Find_EssentialNodes_BF kernel failed");
+
+		cudaEventRecord(stop, 0);
+    	cudaEventSynchronize(start); 
+    	cudaEventSynchronize(stop);
+    	cudaEventElapsedTime(&time, start, stop);            
+    	kernel_timings[3] += time;
+
+        cudaEventRecord(start, 0);
+
+                      cudaMemcpy(*essential_counter,cnt_dev, sizeof(int) *  n_blocks, cudaMemcpyDeviceToHost);
+                      CudaTest((char*)"gr_bh_gpu_Essentials_Calc_Essentials_Wrap:  dev copy to host failed");
+
+        cudaEventRecord(stop, 0);
+    	cudaEventSynchronize(start); 
+    	cudaEventSynchronize(stop);
+    	cudaEventElapsedTime(&time, start, stop);
+		memory_timings[3] += time;            
+		memory_timings[4] += bytes;
+                      
+
+        	      for(int i= 0; i  <= n_blocks; i++)
+           		  breaking += (*essential_counter)[i];
+              
+                      if(breaking == 0)
+                          break;
+                        
+				
+
+                      nodediag = nodediag * 0.25;
+                      loopdepth +=1;
+
+                    }
+                    
+	cudaEventRecord(start, 0);
+	thrust::device_ptr<int> dev_ptr = thrust::device_pointer_cast(essentials);
+	thrust::device_ptr<int> re_ptr = thrust::device_pointer_cast(res);
+
+	thrust::device_ptr<int> iter = thrust::copy_if(thrust::device, dev_ptr, dev_ptr + (nnodes+1), re_ptr, cp(nnodes+1));
+	cudaEventRecord(stop, 0);
+	cudaEventSynchronize(start); 
+	cudaEventSynchronize(stop);
+	cudaEventElapsedTime(&time, start, stop);            
+	kernel_timings[3] += time;
+
+	const unsigned int index = iter - re_ptr;
+
+
+
+   
+    cudaEventRecord(start, 0);
+
+    thrust::copy(re_ptr , re_ptr + index  ,essential_nodes );
+
+    cudaEventRecord(stop, 0);
+    cudaEventSynchronize(start); 
+    cudaEventSynchronize(stop);
+    cudaEventElapsedTime(&time, start, stop);
+	memory_timings[4] += (index * sizeof(int)) ;
+	memory_timings[3] += time ;
+
+    res = thrust::raw_pointer_cast(re_ptr);
+    essentials = thrust::raw_pointer_cast(dev_ptr);
+    *size = index;
+
+	cudaEventDestroy(start);
+	cudaEventDestroy(stop);
+    
+               
+}
+}
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CommLocalRoots.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CommLocalRoots.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CommLocalRoots.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,39 @@
+!!****if* BHTree/GPU/gr_bh_gpu_CommLocalRoots
+!!
+!! NAME
+!!
+!!  gr_bh_gpu_CommLocalRoots
+!!
+!! SYNOPSIS
+!!
+!!  call gr_bh_gpu_CommLocalRoots()
+!!
+!! DESCRIPTION
+!!
+!! distribute local domain data to all PIDs.
+!!
+!! ARGUMENTS
+!!
+!!   No arguments
+!!
+!! AUTOGENROBODOC
+!!
+!!
+!!***
+
+subroutine gr_bh_gpu_CommLocalRoots()
+   use Grid_data, ONLY : gr_meshMe,gr_meshNumProcs, gr_meshComm
+   use Driver_interface, ONLY : Driver_abortFlash
+   use gr_bh_gpu_data, ONLY : gr_bh_gpu_roots_own,gr_bh_gpu_roots_received
+   implicit none
+#include "Flash_mpi.h"
+
+   integer :: ierr
+
+     call MPI_Allgather(gr_bh_gpu_roots_own, 12, FLASH_REAL, gr_bh_gpu_roots_received, 12, FLASH_REAL, gr_meshComm,ierr)
+    
+
+
+return
+end subroutine gr_bh_gpu_CommLocalRoots
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_TreeBuilding_Kernel.cuh
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_TreeBuilding_Kernel.cuh	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_TreeBuilding_Kernel.cuh	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,6 @@
+#ifndef __TREE_BUILD_KERNEL_CUH__
+#define __TREE_BUILD_KERNEL_CUH__
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+__global__
+void TreeBuildingKernel();
+#endif
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Interface.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Interface.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Interface.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,731 @@
+!!****if* source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Interface.F90
+!!
+!! NAME
+!!  gr_bh_gpu_Interface
+!!
+!! SYNOPSIS
+!!
+!!  
+!!
+!! DESCRIPTION
+!!
+!!  
+!! This is the header file for the Barnes-Hut tree solver that defines its
+!! interfaces private to the GPU BH tree implementation.
+!!  
+!!  
+!! ARGUMENTS
+!!
+!!
+!!***
+
+
+Module gr_bh_gpu_Interface
+!use, intrinsic :: iso_c_binding
+ implicit none
+
+!====================================
+!INITIALIZE
+!====================================
+ interface gr_bh_gpu_MakeGpuCommGrp
+    subroutine gr_bh_gpu_MakeGpuCommGrp()
+    end subroutine gr_bh_gpu_MakeGpuCommGrp
+ end interface
+
+ interface gr_bh_gpu_Init_Allocate_MPI_Buffer
+  subroutine gr_bh_gpu_Init_Allocate_MPI_Buffer(recvbuff,sendbuff,R_request,S_request,comm_size)&
+						bind(C, name="gr_bh_gpu_Init_Allocate_MPI_Buffer_")
+
+       use, intrinsic :: iso_c_binding
+       implicit none
+
+       type( c_ptr ), intent (inout) ::  recvbuff,sendbuff
+       type( c_ptr ), intent (inout) ::  R_request,S_request
+       integer, intent(in) :: comm_size
+
+  end subroutine gr_bh_gpu_Init_Allocate_MPI_Buffer
+ end interface
+
+  interface gr_bh_gpu_CalcEwaldField
+    subroutine gr_bh_gpu_CalcEwaldField()
+    end subroutine gr_bh_gpu_CalcEwaldField
+  end interface
+
+!====================================
+!GATHER
+!====================================
+
+  interface gr_bh_gpu_GatherCells
+    subroutine gr_bh_gpu_GatherCells()
+    end subroutine gr_bh_gpu_GatherCells
+  end interface
+
+
+  interface gr_bh_gpu_set_nodes
+    subroutine gr_bh_gpu_set_nodes(blocks,cell_count,nodes)bind(C, name ="gr_bh_gpu_set_nodes_")
+      use, intrinsic :: iso_c_binding
+       implicit none
+       integer,     intent(inout)    :: blocks,cell_count,nodes
+    end subroutine gr_bh_gpu_set_nodes
+  end interface
+
+  interface gr_bh_gpu_Gather_Allocate_CellData
+   subroutine gr_bh_gpu_Gather_Allocate_CellData(x_data,y_data,z_data,m_data,pot_data,cell_count,pinned_cells,rank)&
+						bind(C, name="gr_bh_gpu_Gather_Allocate_CellData_")
+       use, intrinsic :: iso_c_binding
+       implicit none
+       type(c_ptr), intent(inout) :: x_data,y_data,z_data,m_data,pot_data
+       integer,     intent(in)    :: cell_count
+       logical,     intent(in)    :: pinned_cells
+	integer, intent(in) :: rank
+
+    end subroutine gr_bh_gpu_Gather_Allocate_CellData
+  end interface
+
+
+  interface gr_bh_gpu_GetGridData
+    subroutine  gr_bh_gpu_GetGridData(blockid,num_bodies,counter,idensvar,minx,maxx,miny,maxy,minz,maxz,cx,cy,cz,cm)
+     
+      integer, intent(in) :: num_bodies
+      integer, intent(in) :: blockid
+      integer, intent(in) :: idensvar
+      integer, intent(in) :: counter
+
+     real, intent(inout)               :: minx
+     real, intent(inout)               :: maxx
+     real, intent(inout)               :: miny
+     real, intent(inout)               :: maxy
+     real, intent(inout)               :: minz
+     real, intent(inout)               :: maxz
+
+     real, intent(out)               :: cx
+     real, intent(out)               :: cy
+     real, intent(out)               :: cz
+     real, intent(out)               :: cm
+
+
+    end subroutine gr_bh_gpu_GetGridData
+  end interface
+
+
+ interface gr_bh_gpu_GetGridData_SetValue
+    subroutine gr_bh_gpu_GetGridData_SetValue(x_data,y_data,z_data,m_data,x,y,z,m,idx)&
+					bind(C,name="gr_bh_gpu_GetGridData_SetValue_")
+       use, intrinsic :: iso_c_binding
+       implicit none
+
+       type(c_ptr), intent(inout) :: x_data,y_data,z_data,m_data
+       real, intent(in)           :: x,y,z,m
+       integer, intent(in)        :: idx
+
+    end subroutine gr_bh_gpu_GetGridData_SetValue
+ end interface 
+
+!====================================
+!CALCULATE ESSENTIALS
+!====================================
+
+
+  interface gr_bh_gpu_CommLocalRoots
+    subroutine gr_bh_gpu_CommLocalRoots()
+    end subroutine gr_bh_gpu_CommLocalRoots
+  end interface
+
+ interface gr_bh_gpu_CalcExportTree
+    subroutine gr_bh_gpu_CalcExportTree()
+    end subroutine gr_bh_gpu_CalcExportTree
+  end interface
+
+  interface gr_bh_gpu_Essentials_Build_Local_Tree
+      subroutine gr_bh_gpu_Essentials_Build_Local_Tree(tx,ty,tz,tm,&
+						roots,&
+						essential_nodes,essential_blocks,&
+						tree_depth,all_ints,all_doubles)
+       use Grid_data, ONLY : gr_meshNumProcs
+
+
+       use, intrinsic :: iso_c_binding
+       implicit none
+  	integer, intent (inout)       ::  tree_depth
+ 	real, allocatable, intent (inout) ::  tx(:)
+  	real, allocatable, intent (inout) ::  ty(:)
+  	real, allocatable, intent (inout) ::  tz(:)
+  	real, allocatable, intent (inout) ::  tm(:)
+  	real, intent (inout)          ::  roots(0:4*gr_meshNumProcs)
+  	type(c_ptr ), intent (inout) ::  all_ints
+ 	type(c_ptr ), intent (inout) ::  all_doubles
+        type(c_ptr ), intent (inout)     :: essential_nodes,essential_blocks
+        !type(c_ptr), intent (inout)     :: essential_nodes_dev,essential_blocks_dev,res
+
+     end subroutine gr_bh_gpu_Essentials_Build_Local_Tree
+  end interface
+    
+
+  interface gr_bh_gpu_Essentials_Build_Init_Wrap
+    subroutine gr_bh_gpu_Essentials_Build_Init_Wrap(device_id,mp_count,&
+                                             num_cells, &
+                                             num_nodes, &
+                                             angle,&
+                                             domdiagsq,&
+                                             x, &
+                                             y, & 
+                                             z, & 
+                                             mass, &
+                                             xmax ,xmin,&
+                                             ymax, ymin,&
+                                             zmax, zmin,&      
+                                             found_depth,&
+                                             periodic,&
+                                             roots,&
+                                             gr_meshNumProcs,&
+                                             memory_time,kernel_time,tx,ty,tz,tm,&
+				      	     essential_nodes,essential_blocks,&
+					     all_ints,all_doubles)&
+							bind(C, name="gr_bh_gpu_Essentials_Build_Init_Wrap_")
+
+	use, intrinsic :: iso_c_binding
+	implicit none
+
+      integer, intent(in)        ::    device_id !multiprocessor count
+      integer, intent(in)  	 ::    mp_count !multiprocessor count
+      integer, intent(in)  	 ::    num_cells !data cell count
+      integer, intent(in)  	 ::    num_nodes !node_count
+      real, intent(in)     	 ::    angle !bh angle
+      real, intent(in)     	 ::    domdiagsq !domain size
+!      real, intent(in)    	 ::    x(0:num_cells-1),y(0:num_cells-1),&   !cell data 
+!				       z(0:num_cells-1),mass(0:num_cells-1)
+      type(c_ptr), intent(inout) ::    x,y,z,mass
+      real, intent (in)          ::    xmax, xmin, ymax, ymin, zmax, zmin !domain dim      
+      type(c_ptr), intent(inout) ::    all_ints,all_doubles !pointer for cell data,tree, boundaries
+      integer, intent(in)  	 ::    periodic !periodic boundaries?
+      integer, intent(in)  	 ::    gr_meshNumProcs
+      real, intent(in)  	 ::    roots(0:4*gr_meshNumProcs)      
+      integer, intent(inout)  	 ::    found_depth    
+      real, intent(inout)  	 ::    memory_time(0:4),kernel_time(0:3)
+      real, intent(inout)        ::    tx(0:num_nodes+1),ty(0:num_nodes+1),tz(0:num_nodes+1),tm(0:num_nodes+1)
+      type(c_ptr), intent (inout)     :: essential_nodes,essential_blocks
+      
+
+    end subroutine gr_bh_gpu_Essentials_Build_Init_Wrap
+  end interface
+
+ interface gr_bh_gpu_Essentials_Calculate_Essentials
+   subroutine gr_bh_gpu_Essentials_Calculate_Essentials(offset,essentials_count,tree_depth,roots,all_doubles,all_ints,&
+							essential_nodes,essential_blocks)
+
+
+        use Grid_data, ONLY : gr_meshNumProcs
+	use, intrinsic :: iso_c_binding
+	implicit none
+
+		integer, intent (inout)         :: offset,essentials_count,tree_depth
+		real, intent (inout)            :: roots(0:4*gr_meshNumProcs)
+		type( c_ptr ), intent (inout)   :: all_doubles,all_ints
+		type(c_ptr), intent (inout)     :: essential_nodes,essential_blocks
+              !  type(c_ptr), intent (inout)     :: essential_nodes_dev,essential_blocks_dev,res
+
+   end subroutine gr_bh_gpu_Essentials_Calculate_Essentials
+ end interface
+
+
+
+
+
+ interface gr_bh_gpu_Essentials_Calc_Essentials_Wrap
+   subroutine gr_bh_gpu_Essentials_Calc_Essentials_Wrap(device_ID,gpu_blocks,cell_count,node_count,tree_depth,&
+						        roots,domdiagsq,angle,essentials_count,&
+						        all_doubles,all_ints,offset,memory_time,&
+							kernel_time,&
+							essential_nodes,essential_blocks)&
+							bind(C,name="gr_bh_gpu_Essentials_Calc_Essentials_Wrap_")
+	use Grid_data, ONLY : gr_meshNumProcs
+	use, intrinsic :: iso_c_binding
+	implicit none
+
+	integer, intent(inout)::device_ID,gpu_blocks,cell_count,&
+				node_count,tree_depth,essentials_count,offset
+	real, intent(inout)::domdiagsq,angle
+	real, intent(inout)::memory_time(0:4), kernel_time(0:3)
+	type(c_ptr), intent (inout)     :: essential_nodes,essential_blocks
+       ! type(c_ptr), intent (inout)     :: essential_nodes_dev,essential_blocks_dev,res
+	real, intent (inout)  ::roots(0:4*gr_meshNumProcs)
+	type( c_ptr ), intent (inout)   :: all_doubles,all_ints
+
+   end subroutine gr_bh_gpu_Essentials_Calc_Essentials_Wrap
+ end interface
+
+
+
+
+ interface gr_bh_gpu_Essentials_CleanUp_Device_Wrap
+  subroutine gr_bh_gpu_Essentials_CleanUp_Device_Wrap(device_id,all_doubles,&
+						all_ints,&
+						essential_nodes,essential_block, memory_timer)&
+						bind(C,name="gr_bh_gpu_Essentials_CleanUp_Device_Wrap_")
+  	use, intrinsic :: iso_c_binding
+	implicit none
+
+        integer, intent(in)             :: device_id
+	type( c_ptr ), intent (inout)   :: all_doubles
+	type(c_ptr), intent (inout)     :: essential_nodes,essential_block
+        type(c_ptr), intent (inout)     :: all_ints
+	real, intent (inout)  ::memory_timer(0:4)
+
+  end subroutine gr_bh_gpu_Essentials_CleanUp_Device_Wrap
+ end interface
+
+
+
+ interface gr_bh_gpu_Essentials_Send_Essentials_Wrap
+   subroutine gr_bh_gpu_Essentials_Send_Essentials_Wrap(sendbuff,send_request,&
+							tx,ty,tz,tm,&
+							msg_size,top_IDX,receiver,me_rank,comm_size,comm_grp,&
+							essential_nodes,pinned_mpi)&
+							bind(C,name="gr_bh_gpu_Essentials_Send_Essentials_Wrap_")
+
+
+  	use, intrinsic :: iso_c_binding
+	implicit none
+
+	type(c_ptr), intent(inout) :: sendbuff,send_request
+	real, intent(inout):: tx(0:10)
+	real, intent(inout):: ty(0:10)
+	real, intent(inout):: tz(0:10)
+	real, intent(inout):: tm(0:10)
+	integer, intent(in)::msg_size,top_IDX,receiver,me_rank,comm_size,comm_grp
+	type(c_ptr), intent (inout)     :: essential_nodes
+	logical, intent(in)::pinned_mpi
+
+    end subroutine gr_bh_gpu_Essentials_Send_Essentials_Wrap
+ end interface
+
+
+ interface gr_bh_gpu_Essentials_Probe_Messages
+  subroutine gr_bh_gpu_Essentials_Probe_Messages(flag,sender_ID,message_size)
+   logical, intent(inout) :: flag
+   integer, intent(inout) :: sender_ID
+   integer, intent(inout) :: message_size
+  end subroutine gr_bh_gpu_Essentials_Probe_Messages
+ end interface
+
+
+ interface gr_bh_gpu_Essentials_Receive_Message
+   subroutine gr_bh_gpu_Essentials_Receive_Message(recv_buff,recv_request,&
+						element_count, sender,tag, comm_grp, pinned_mpi)&
+						bind(c, name= "gr_bh_gpu_Essentials_Receive_Message_")
+
+  	use, intrinsic :: iso_c_binding
+	implicit none
+
+	type(c_ptr), intent(inout) :: recv_buff,recv_request
+	integer, intent(inout)  :: element_count, sender,tag, comm_grp
+	logical, intent(inout)  :: pinned_mpi
+
+   end subroutine gr_bh_gpu_Essentials_Receive_Message
+ end interface
+
+
+
+
+ interface gr_bh_gpu_Essentials_Wait_And_Copy
+   subroutine gr_bh_gpu_Essentials_Wait_And_Copy(recv_buff,recv_request,&
+						  h_X, h_Y, h_Z, h_M,&
+						  local_cell_count,me_rank,comm_size,pinned_mpi)&
+						bind(c, name= "gr_bh_gpu_Essentials_Wait_And_Copy_")
+
+
+
+  	use, intrinsic :: iso_c_binding
+	implicit none
+
+	type(c_ptr), intent(inout) :: recv_buff,recv_request
+        type(c_ptr), intent(inout)        :: h_X, h_Y,h_Z, h_M
+	integer, intent(inout)     :: me_rank, comm_size,local_cell_count
+	logical, intent(in)        :: pinned_mpi
+
+    end subroutine gr_bh_gpu_Essentials_Wait_And_Copy
+  end interface
+
+
+
+
+
+  interface gr_bh_gpu_Essentials_Reallocate_CellData
+   subroutine gr_bh_gpu_Essentials_Reallocate_CellData(x_data,y_data,z_data,m_data,&
+                                                       local_cell_count, total_cellcount,&
+                                                       pinned_cells, rank)&
+						bind(C, name="gr_bh_gpu_Essentials_Reallocate_CellData_")
+
+  	use, intrinsic :: iso_c_binding
+	implicit none
+        type(c_ptr), intent(inout) :: x_data,y_data,z_data,m_data
+	integer, intent(in) :: local_cell_count, total_cellcount
+        logical, intent(in) :: pinned_cells
+	integer, intent(in) :: rank
+   end subroutine gr_bh_gpu_Essentials_Reallocate_CellData
+  end interface
+
+!====================================
+!CALCULATE POTENTIAL
+!====================================
+
+
+
+
+ interface gr_bh_gpu_CalcPotSelf
+    subroutine gr_bh_gpu_CalcPotSelf()
+    end subroutine gr_bh_gpu_CalcPotSelf
+  end interface
+
+
+ interface gr_bh_gpu_Potential_Simple
+  subroutine gr_bh_gpu_Potential_Simple(device_ID,gpu_blocks,angle,&
+                           bodies_body_count,  &
+                           x,y, z, mass, gpot_sol,& 
+                           domain_diagonal,&
+                           minx,  miny, minz, maxx, maxy, maxz,& 
+                           e_grid, ewald_size, &
+			   periodics,&
+                           memory_time,kernel_time)bind(C, name="gr_bh_gpu_Potential_Simple_")
+
+  	use, intrinsic :: iso_c_binding
+	implicit none
+
+        integer, intent(inout)     :: gpu_blocks, bodies_body_count,device_ID
+	integer, intent(inout)     :: ewald_size,periodics
+	real, intent(inout)        :: domain_diagonal,angle
+	real, intent(inout)        :: minx,  miny, minz, maxx, maxy, maxz
+        real, intent(inout)        :: e_grid(0:ewald_size)
+        type(c_ptr), intent(inout)        :: gpot_sol	
+        type(c_ptr), intent(inout)        :: x, y, z,mass
+	real, intent(inout) ::memory_time(0:4), kernel_time(0:3)
+
+   end subroutine gr_bh_gpu_Potential_Simple
+ end interface 
+
+
+
+ interface gr_bh_gpu_CalcPot
+    subroutine gr_bh_gpu_CalcPot()
+    end subroutine gr_bh_gpu_CalcPot
+  end interface
+
+ interface gr_bh_gpu_Potential_Init
+    subroutine  gr_bh_gpu_Potential_Init(device_ID,itolsq,&
+                                         local_cell_count,&
+                                         domdiagsq,&
+                                         xmin_domain, ymin_domain, zmin_domain,&
+                                         xmax_domain, ymax_domain, zmax_domain,&      
+                                         EwaldField,EwaldSize,&
+                                         periodic,&
+                                         memory_time,&
+                                         gpotd,&
+                                         e_gridd,&
+                                         maxxd, maxyd,maxzd, minxd, minyd, minzd)&
+                                         bind(C, name="gr_bh_gpu_Potential_Init_")         
+                              
+      use, intrinsic :: ISO_C_BINDING
+        implicit none
+
+      integer, intent(in)        :: local_cell_count,device_ID
+      real, intent(in)           :: itolsq,domdiagsq
+      real, intent(in)           :: xmin_domain,ymin_domain,zmin_domain,&
+                                    xmax_domain, ymax_domain,zmax_domain
+
+      integer, intent(in)        :: EwaldSize,periodic
+      real ,intent(in)           :: EwaldField(0:EwaldSize-1)
+      real, intent(inout)        :: memory_time(0:4)
+      !device Pointer     
+      type(c_ptr),intent(inout)  :: gpotd,maxxd, maxyd, maxzd, minxd, minyd, minzd, e_gridd
+
+
+    end subroutine gr_bh_gpu_Potential_Init
+  end interface
+
+
+ interface gr_bh_gpu_Potential_Local
+    subroutine gr_bh_gpu_Potential_Local(device_ID,gpu_blocks,&
+                                         local_cell_count, & 
+                                         x_data,y_data,z_data,m_data,&                                  
+                                          memory_time,kernel_time,&
+                                 maxxd,maxyd,maxzd,&
+                                 minxd,minyd,minzd)&
+                                         bind(C, name="gr_bh_gpu_Potential_Local_")         
+                              
+      use, intrinsic :: ISO_C_BINDING
+        implicit none
+
+      integer, intent(in) ::gpu_blocks,local_cell_count,device_ID
+      type(c_ptr),intent(inout) ::x_data,y_data,z_data,m_data
+      real, intent(inout) :: memory_time(0:4),kernel_time(0:3)
+      !device Pointer     
+      type(c_ptr),intent(inout)  :: maxxd, maxyd, maxzd, minxd, minyd, minzd
+
+
+    end subroutine gr_bh_gpu_Potential_Local
+  end interface
+
+
+
+ interface gr_bh_gpu_Potential_Subset
+    subroutine gr_bh_gpu_Potential_Subset(device_ID,gpu_blocks,local_cell_count,other_cell_count,idx,& 
+                                 x_data,y_data, z_data, m_data,&
+                                 maxxd,maxyd, maxzd,minxd,minyd,minzd, &
+                                 x_device,y_device,z_device, &
+                                 memory_time,kernel_time)&
+                                         bind(C, name="gr_bh_gpu_Potential_Subset_")         
+                              
+      use, intrinsic :: ISO_C_BINDING
+        implicit none
+
+        integer, intent(in) ::gpu_blocks,local_cell_count,&
+			   other_cell_count,idx,device_ID
+       type(c_ptr),intent(inout) ::x_data,y_data, z_data, m_data
+      real, intent(inout) :: memory_time(0:4), kernel_time(0:3)
+      !device Pointer     
+      type(c_ptr),intent(inout)  :: maxxd, maxyd, maxzd, minxd, minyd, minzd
+      type(c_ptr),intent(inout)  :: x_device,y_device,z_device
+
+    end subroutine gr_bh_gpu_Potential_Subset
+  end interface
+
+
+
+
+ interface gr_bh_gpu_Potential_Copy
+    subroutine  gr_bh_gpu_Potential_Copy(device_ID,local_cell_count,&
+                                       x_coords, y_coords, z_coords,& 
+                                       x_device,y_device,z_device, &
+                                       memory_time)&
+                                       bind(C, name="gr_bh_gpu_Potential_Copy_")         
+                              
+      use, intrinsic :: ISO_C_BINDING
+        implicit none
+
+
+      integer, intent(in) ::local_cell_count,device_ID
+
+      type(c_ptr),intent(inout) ::x_coords,y_coords,z_coords
+
+      real, intent(inout)        :: memory_time(0:4)
+      !device Pointer     
+      type(c_ptr),intent(inout)  :: x_device,y_device,z_device
+
+
+    end subroutine gr_bh_gpu_Potential_Copy
+  end interface
+
+
+
+ interface gr_bh_gpu_Potential_All
+    subroutine gr_bh_gpu_Potential_All(device_ID,gpu_blocks,local_cell_count,total_cell_count,& 
+                                 x_data,y_data, z_data, m_data,&
+                                 memory_time, kernel_time,&
+ 				 maxxd,maxyd, maxzd,minxd,minyd,minzd)&
+                                         bind(C, name="gr_bh_gpu_Potential_All_")         
+                              
+      use, intrinsic :: ISO_C_BINDING
+        implicit none
+
+      integer, intent(in):: gpu_blocks,local_cell_count,&
+			   total_cell_count,device_ID
+      type(c_ptr),intent(inout)  ::  x_data,y_data, z_data, m_data
+      real, intent(inout) :: memory_time(0:4), kernel_time(0:3)
+      !device Pointer     
+      type(c_ptr),intent(inout)  :: maxxd, maxyd, maxzd, minxd, minyd, minzd
+   
+
+    end subroutine gr_bh_gpu_Potential_All
+  end interface
+
+
+ interface gr_bh_gpu_Potential_Clean
+
+    subroutine gr_bh_gpu_Potential_Clean(device_ID,local_cell_count,&
+						other,&
+						periodicity,&
+                                     		memory_time,&
+                                     		gpot_sol,&						
+                                    		maxxd,maxyd,maxzd,&
+                                     		minxd,minyd,minzd, &
+                                    		x_device,y_device,z_device,& 
+                                     		gpot_sold,e_gridd)&
+                                         	bind(C, name="gr_bh_gpu_Potential_Clean_")         
+                              
+      use, intrinsic :: ISO_C_BINDING
+        implicit none
+
+      integer, intent(in) ::local_cell_count,other,periodicity,device_ID
+      type(c_ptr),intent(inout) :: gpot_sol
+      real, intent(inout) :: memory_time(0:4)
+      !device Pointer     
+      type(c_ptr),intent(inout)  :: maxxd, maxyd, maxzd, minxd, minyd, minzd
+      type(c_ptr),intent(inout)  :: x_device,y_device,z_device
+      type(c_ptr),intent(inout)  :: gpot_sold,e_gridd
+  
+    end subroutine gr_bh_gpu_Potential_Clean
+  end interface
+
+
+
+!====================================
+!SAVE
+!====================================
+
+ interface gr_bh_gpu_SavePot
+    subroutine gr_bh_gpu_SavePot()
+    end subroutine gr_bh_gpu_SavePot
+  end interface
+
+
+  interface gr_bh_gpu_SetGridPot
+    subroutine gr_bh_gpu_SetGridPot(offset,blockid,counter,ipoisvar,num_bodies)
+      integer, intent(in) :: blockid,offset
+      integer, intent(in) :: ipoisvar
+      integer, intent(in) :: counter
+      integer, intent(in) :: num_bodies
+
+    end subroutine gr_bh_gpu_SetGridPot
+  end interface
+
+
+
+ interface gr_bh_gpu_Save_Grid_Pot
+  subroutine gr_bh_gpu_Save_Grid_Pot(idx,a_data,val)bind(C, name="gr_bh_gpu_Save_Grid_Pot_")
+	use, intrinsic :: iso_c_binding
+	implicit none
+
+        type(c_ptr), intent(inout) :: a_data
+	integer, intent(in) :: idx
+        real, intent(inout) :: val
+
+  end subroutine gr_bh_gpu_Save_Grid_Pot
+ end interface
+!====================================
+!CLEAN UP
+!====================================
+
+ interface gr_bh_gpu_CleanUp
+    subroutine  gr_bh_gpu_CleanUp()
+    end subroutine  gr_bh_gpu_CleanUp
+ end interface
+
+
+  interface gr_bh_gpu_Clear_Buffers
+     subroutine gr_bh_gpu_Clear_Buffers(recvbuff,sendbuff,&
+				        R_request,S_request,&
+				        comm_size,pinned_mpi)&
+                                        bind(C,name="gr_bh_gpu_Clear_Buffers_")
+
+  	use, intrinsic :: iso_c_binding
+	implicit none
+
+	type(c_ptr), intent(inout) :: recvbuff,sendbuff
+        type(c_ptr), intent(inout) :: R_request,S_request
+        integer, intent(inout)     :: comm_size
+	logical, intent(in) :: pinned_mpi
+
+    end subroutine gr_bh_gpu_Clear_Buffers
+  end interface
+
+
+  interface gr_bh_gpu_CleanUp_CellData
+   subroutine gr_bh_gpu_CleanUp_CellData(x_data,y_data,z_data,m_data,pot_data,pinned)&
+					bind(C, name="gr_bh_gpu_CleanUp_CellData_")
+  	use, intrinsic :: iso_c_binding
+	implicit none
+
+        type(c_ptr), intent(inout) :: x_data,y_data,z_data,m_data,pot_data
+        logical, intent(in) :: pinned
+
+   end subroutine gr_bh_gpu_CleanUp_CellData
+ end interface 
+!====================================
+!FINALIZE 
+!====================================
+
+
+ 
+  interface gr_bh_gpu_Finalize_Buffer
+     subroutine gr_bh_gpu_Finalize_Buffer(recvbuff,sendbuff,&
+				        R_request,S_request)&
+                                        bind(C,name="gr_bh_gpu_Finalize_Buffer_")
+
+  	use, intrinsic :: iso_c_binding
+	implicit none
+
+	type(c_ptr), intent(inout) :: recvbuff,sendbuff
+        type(c_ptr), intent(inout) :: R_request,S_request
+        
+    end subroutine gr_bh_gpu_Finalize_Buffer
+  end interface
+
+
+
+
+ 
+
+ interface gr_bh_gpu_IrecvEssentials
+    subroutine gr_bh_gpu_IrecvEssentials()
+    end subroutine gr_bh_gpu_IrecvEssentials
+  end interface
+
+ interface gr_bh_gpu_WaitForEssentialSizes
+    subroutine gr_bh_gpu_WaitForEssentialSizes()
+    end subroutine gr_bh_gpu_WaitForEssentialSizes
+  end interface
+
+ interface gr_bh_gpu_WaitForEssentialData
+    subroutine gr_bh_gpu_WaitForEssentialData()
+    end subroutine gr_bh_gpu_WaitForEssentialData
+  end interface
+  
+   interface gr_bh_gpu_WaitAndCalcPot
+    subroutine gr_bh_gpu_WaitAndCalcPot()
+    end subroutine gr_bh_gpu_WaitAndCalcPot
+  end interface
+
+
+ interface gr_bh_gpu_ReAllocBlocks
+    subroutine gr_bh_gpu_ReAllocBlocks()
+    end subroutine gr_bh_gpu_ReAllocBlocks
+  end interface
+
+!====================================
+!EWALD
+!====================================
+ interface gr_bh_gpu_Ewald_MakeField
+   subroutine gr_bh_gpu_Ewald_MakeField(NX, NY, NZ, grid,x_cell,y_cell,z_cell,&
+                                        e_xmax,e_ymax,e_zmax, periodics)bind(C,name="gr_bh_gpu_Ewald_MakeField_")
+ 	use, intrinsic :: iso_c_binding
+	implicit none
+
+        integer, intent(in) :: periodics,NX,NY,NZ
+        real, intent(in)    :: x_cell,y_cell,z_cell
+	real, intent(in)    :: e_xmax,e_ymax,e_zmax
+        real, intent(inout) :: grid(0:NX*NY*NZ)
+
+   end subroutine gr_bh_gpu_Ewald_MakeField
+ end interface
+
+ interface gr_bh_gpu_Ewald_Init
+   subroutine gr_bh_gpu_Ewald_Init(NX,NY,NZ,grv_bhEwaldSeriesN,gr_bhEwaldIsoFac,&
+                               grv_bhLx,grv_bhLy,grv_bhLz,&
+                                e_xmax,e_ymax,e_zmax, periodics)bind(C,name="gr_bh_gpu_Ewald_Init_")
+ 	use, intrinsic :: iso_c_binding
+	implicit none
+        integer, intent(in) :: periodics,NX,NY,NZ,grv_bhEwaldSeriesN,gr_bhEwaldIsoFac
+        real, intent(in)    :: grv_bhLx,grv_bhLy,grv_bhLz
+	real, intent(in)    ::  e_xmax,e_ymax,e_zmax
+  
+
+   end subroutine gr_bh_gpu_Ewald_Init
+ end interface
+
+
+
+end Module  gr_bh_gpu_Interface
+
+
+
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_All.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_All.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_All.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,303 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+
+// GENERAL TREE BUILDING
+#include "gr_bh_gpu_Init_Kernel.cuh"
+#include "gr_bh_gpu_MakeRootKernel.cuh"
+#include "gr_bh_gpu_Summarization_Kernel.cuh"
+#include "gr_bh_gpu_TreeBuilding_Kernel.cuh"
+
+
+// FORCE CALCULATION
+#include "gr_bh_gpu_Potential_Calculate_Direkt_Kernel.cuh"
+#include "gr_bh_gpu_Potential_Calculate_ForeignTree_Kernel.cuh"
+#include "gr_bh_gpu_Potential_Calculate_Local_Kernel.cuh"
+
+
+
+extern "C" {
+void gr_bh_gpu_Potential_All_(int *device_ID,int *gpu_blocks,
+                                 int *local_cell_count,
+                                 int *total_cell_count,  
+                                 double **x_Pdata,
+                                 double **y_Pdata, 
+                                 double **z_Pdata, 
+                                 double **m_Pdata,
+                                 double *memory_time,
+                                 double *kernel_time,
+                                  double **root_maxxd,
+                                   double **root_maxyd,
+                                   double **root_maxzd,
+                                   double **root_minxd,
+                                   double **root_minyd,
+                                   double **root_minzd)
+    {
+
+  double *x = *x_Pdata;
+  double *y = *y_Pdata;
+  double *z = *z_Pdata;
+  double *m = *m_Pdata;
+
+  int  blocks = *gpu_blocks;
+  int nnodes = (*total_cell_count) * 2;
+
+        if (nnodes < 1024*blocks)
+          nnodes = 1024*blocks;
+
+      while ((nnodes & (WARPSIZE-1)) != 0)
+            nnodes++;
+
+      nnodes--;
+
+  cudaEvent_t start, stop;
+  
+  float time;
+
+
+
+
+  double *d_delta;
+
+  int  *childl;
+  double *all;
+  double *posxl;
+  double *posyl;
+  double *poszl;
+  double *massl;
+  double *radii; 
+
+
+  double  bytes = 0;
+
+
+
+ cudaSetDevice(*device_ID);
+ cudaEventCreate(&start);
+ cudaEventCreate(&stop);
+ cudaEventRecord(start, 0);
+
+//malloc
+
+
+
+cudaMalloc((void **)&childl, sizeof(int) * (nnodes+1) * 8);
+CudaTest( (char*)"gr_bh_gpu_Potential_All: couldn't allocate childd");
+
+
+cudaMalloc((void **)&all, sizeof(double) * ((nnodes+1)*6)); 
+CudaTest( (char*)"gr_bh_gpu_Potential_All: gr_bh_gpu_calcforces : couldn't allocate all");
+
+
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+memory_time[0] += time; 
+
+
+posxl  =all+0;
+posyl  =all+(nnodes+1);
+poszl  =all+((nnodes+1)*2);
+massl  =all+((nnodes+1)*3);
+radii  =all+((nnodes+1)*4); 
+d_delta=all+((nnodes+1)*5);
+
+
+cudaEventRecord(start, 0);
+
+//cpytoSymb
+cudaMemcpyToSymbol(nbodiesd, total_cell_count, sizeof(int));
+CudaTest( (char*)"gr_bh_gpu_Potential_All: nbody copy to Symbol failed");
+bytes += 1 * sizeof(int);
+
+cudaMemcpyToSymbol(nnodesd, &nnodes, sizeof(int));
+CudaTest( (char*)"gr_bh_gpu_Potential_All: nnode copy to Symbol failed");
+bytes += 1 * sizeof(int);
+
+cudaMemcpyToSymbol(delta, &d_delta, sizeof(void*));
+CudaTest( (char*)"gr_bh_gpu_Potential_All: delta copy to Symbol failed");
+
+
+cudaMemcpyToSymbol(poszd, &poszl, sizeof(void*));
+CudaTest( (char*)"gr_bh_gpu_Potential_All: posyl copy to Symbol failed");
+
+
+cudaMemcpyToSymbol(childd, &childl, sizeof(void*));
+CudaTest( (char*)"gr_bh_gpu_Potential_All: childl copy to Symbol failed");
+
+
+cudaMemcpyToSymbol(massd, &massl, sizeof(void*));
+CudaTest( (char*)"gr_bh_gpu_Potential_All: massl copy to Symbol failed");
+
+
+cudaMemcpyToSymbol(posxd, &posxl, sizeof(void*)); 
+CudaTest( (char*)"gr_bh_gpu_Potential_All: posxl copy to Symbol failed");
+
+
+
+cudaMemcpyToSymbol(posyd, &posyl, sizeof(void*));
+CudaTest( (char*)"gr_bh_gpu_Potential_All: posyl copy to Symbol failed");
+
+
+cudaMemcpyToSymbol(posrd, &radii, sizeof(void*));
+CudaTest( (char*)"gr_bh_gpu_Potential_All: radii copy to Symbol failed");
+
+
+//memcpy
+cudaMemcpy(massl, m, sizeof(double) * (*total_cell_count), cudaMemcpyHostToDevice);
+CudaTest( (char*)"gr_bh_gpu_Potential_All: mass copy to device failed");
+bytes += (*total_cell_count) * sizeof(double);
+
+cudaMemcpy(posxl, x, sizeof(double) * (*total_cell_count), cudaMemcpyHostToDevice);
+CudaTest( (char*)"gr_bh_gpu_Potential_All: posx copy to device failed");
+bytes += (*total_cell_count) * sizeof(double);
+
+cudaMemcpy(posyl, y, sizeof(double) * (*total_cell_count), cudaMemcpyHostToDevice);
+CudaTest( (char*)"gr_bh_gpu_Potential_All: posy copy to device failed");
+bytes += (*total_cell_count) * sizeof(double);
+
+cudaMemcpy(poszl, z, sizeof(double) * (*total_cell_count), cudaMemcpyHostToDevice);
+CudaTest( (char*)"gr_bh_gpu_Potential_All: posz copy to device failed");
+bytes += (*total_cell_count) * sizeof(double);
+
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+memory_time[1] += time; 
+memory_time[2] += bytes;
+
+//bytes = 134217718 * 8;
+//bytes = bytes/1073741824.0;
+//time = time / 1000;
+//printf("  Device to Host bandwidth (GB/s): %f, %f\n", bytes/time, bytes);
+
+bytes = 0; 
+
+
+cudaEventRecord(start, 0);
+
+
+
+cudaMemset((void*) d_delta, 0, sizeof(double) * (nnodes+1));
+CudaTest( (char*)"gr_bh_gpu_Potential_All: couldn't initialize dist_com");
+
+
+InitializationKernel<<<1, 1>>>();
+CudaTest( (char*)"gr_bh_gpu_Potential_All: InitializationKernel launch failed");
+
+
+MakeRootKernel<<<1,1>>>(*root_minxd,*root_maxxd,*root_minyd,*root_maxyd,*root_minzd,*root_maxzd);
+CudaTest( (char*)"gr_bh_gpu_Potential_All: MakeRootKernel launch failed");
+
+ 
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+
+kernel_time[0] += time; 
+
+cudaEventRecord(start, 0);
+
+TreeBuildingKernel<<<blocks * FACTOR2, THREADS2>>>();
+CudaTest( (char*)"gr_bh_gpu_Potential_All: TreeBuildingKernel launch failed");
+         
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+
+kernel_time[1] += time; 
+
+cudaEventRecord(start, 0);
+
+SummarizationKernel<<<blocks * FACTOR3, THREADS3>>>();
+CudaTest( (char*)"gr_bh_gpu_Potential_All: SummarizationKernel launch failed");
+
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+
+kernel_time[2] += time; 
+
+int n_blocks = ((*local_cell_count)+ THREADS1-1)/THREADS1;
+if (n_blocks > 65536)
+    n_blocks = 65536;
+
+cudaEventRecord(start, 0);
+                   
+//ForceCalculationKernel_periodic_self_deep<<<n_blocks, THREADS1>>>((*local_cell_count));
+
+#if __CUDA_ARCH__ >= 300
+ForceCalculationKernel_periodic_self_deep<<<n_blocks, 512>>>((*local_cell_count));
+#else
+ForceCalculationKernel_periodic_self_deep<<<n_blocks, THREADS1>>>((*local_cell_count));
+#endif
+CudaTest( (char*)"gr_bh_gpu_Potential_All: ForceCalculationKernel_periodic_self_deep launch failed");
+ 
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+
+kernel_time[3] += time;  
+
+   
+cudaEventRecord(start, 0);       
+
+
+
+
+
+
+    cudaFree(childl);
+    cudaFree(all);
+
+
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+memory_time[0] += time;  
+
+     cudaEventDestroy(start);
+     cudaEventDestroy(stop);
+         
+
+
+ }
+}
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Periodic_InterpolateInField.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Periodic_InterpolateInField.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Periodic_InterpolateInField.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,82 @@
+#include "gr_bh_gpu_Periodic_InterpolateInField.cuh"
+
+
+__device__ __forceinline__ double interp_ewald_noref(double x, double y, double z)
+{
+
+
+  double dx, dy, dz, p, q, r;
+  int i, j, k;
+  int  ip1,  jp1, kp1;
+
+
+  dx = ewald_xmax / (d_NX-1);
+  i = (int)floor(x/dx);
+/*
+  if ((i .LT. 0) .or. (i .GE. gr_bhEwaldFieldNx)) then
+    print *, i, x, dx, gr_bhEwaldXMax, gr_bhEwaldFieldNx
+    call Driver_abortFlash("FATAL: Ewald field i out of limits")
+  endif
+*/
+  dy = ewald_ymax / (d_NY-1);
+  j = (int)floor(y/dy);
+/*
+  if ((j .LT. 0) .or. (j .GE. gr_bhEwaldFieldNy)) then
+    call Driver_abortFlash("FATAL: Ewald field j out of limits")
+  endif
+*/
+  dz = ewald_zmax / (d_NZ-1);
+  k = (int)floor(z/dz);
+/*
+  if ((k .LT. 0) .or. (k .GE. gr_bhEwaldFieldNz)) then
+    call Driver_abortFlash("FATAL: Ewald field k out of limits")
+  endif
+  */
+
+
+ 
+  if (i == d_NX-1){
+    ip1 = i;
+    
+  }else{ 
+    ip1 = i+1;
+  }
+  if (j == d_NY-1) {
+    jp1 = j;
+  }else {
+    jp1 = j+1;
+ }
+  if (k == d_NZ-1) {
+    kp1 = k;
+  }else {
+    kp1 = k+1;
+}
+
+  p = (x - i*dx)/dx;
+  q = (y - j*dy)/dy;
+  r = (z - k*dz)/dz;
+
+
+
+ 
+  
+// linear interpolation
+ 
+
+   return ( 
+       (1.0 - p)*(1.0 - q)*(1.0 - r) * d_grid[((i)   + (j)   * (d_NX) + (k)  * (d_NX) * (d_NY))] +
+       (      p)*(1.0 - q)*(1.0 - r) * d_grid[((ip1) + (j)   * (d_NX) + (k)  * (d_NX) * (d_NY))] +
+       (1.0 - p)*(      q)*(1.0 - r) * d_grid[((i)   + (jp1) * (d_NX) + (k)  * (d_NX) * (d_NY))] +
+       (      p)*(      q)*(1.0 - r) * d_grid[((ip1) + (jp1) * (d_NX) + (k)  * (d_NX) * (d_NY))] +
+       (1.0 - p)*(1.0 - q)*(      r) * d_grid[((i)   + (j)   * (d_NX) + (kp1)* (d_NX) * (d_NY))] +
+       (      p)*(1.0 - q)*(      r) * d_grid[((ip1) + (j)   * (d_NX) + (kp1)* (d_NX) * (d_NY))] +
+       (1.0 - p)*(      q)*(      r) * d_grid[((i)   + (jp1) * (d_NX) + (kp1)* (d_NX) * (d_NY))] +
+       (      p)*(      q)*(      r) * d_grid[((ip1) + (jp1) * (d_NX) + (kp1)* (d_NX) * (d_NY))]  
+       );
+
+
+
+
+
+
+}
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Init_Allocate_MPI_Buffer.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Init_Allocate_MPI_Buffer.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Init_Allocate_MPI_Buffer.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,55 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+#include "mpi.h"
+
+
+extern "C" {
+void gr_bh_gpu_Init_Allocate_MPI_Buffer_(double ***recvbuff,double ***sendbuff,MPI_Request **R_request,MPI_Request **S_request,int *comm_size)//
+    {
+     //  cudaMallocHost((void**)*sendbuff, sizeof(double *) * *comm_size);
+
+	*sendbuff = (double **) malloc(sizeof(double *) * *comm_size);
+	*recvbuff = (double **) malloc(sizeof(double *) * *comm_size);
+	*S_request  = (MPI_Request *) malloc(sizeof(MPI_Request) * *comm_size);
+	*R_request  = (MPI_Request *) malloc(sizeof(MPI_Request) * *comm_size);
+	for(int j = 0; j < *comm_size; j++){
+		(*S_request)[j]=MPI_REQUEST_NULL;
+		(*R_request)[j]=MPI_REQUEST_NULL;
+		(*sendbuff)[j]=NULL;
+		(*recvbuff)[j]=NULL;
+	}
+
+    }
+}
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Calculate_ForeignTree_Kernel.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Calculate_ForeignTree_Kernel.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Calculate_ForeignTree_Kernel.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,259 @@
+/*
+!!****if* source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_CalculateForeignTree_Kernel.cu
+!!
+!! NAME
+!!  gr_bh_gpu_Finalize
+!!
+!! SYNOPSIS
+!!
+!!  gr_bh_gpu_Potential_CalculateForeignTree_Kernel(int n_old,double *selfx, double *selfy, double *selfz)
+!!
+!! DESCRIPTION
+!!
+!!  This cuda kernel walks down the BH tree on the GPU. This version calculates the potetnial for given set of data cells
+!!  distinct from the cells stored in the tree.
+!!
+!! AUTHOR
+!!
+!!  © 2016 Gunther Lukat 
+!!
+!!  
+!!  
+!! ARGUMENTS
+!!
+!!  n_old - number of cells in th the set
+!!  selfx -x position data of set
+!!  selfy -y position data of set
+!!  selfz -z position data of set
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!!
+!! Gunther Lukat, Robi Banerjee, A GPU accelerated Barnes Hut tree code for FLASH4, New Astronomy, Volume 45, May 2016, 
+!! Pages 14-28, ISSN 1384-1076, http://dx.doi.org/10.1016/j.newast.2015.10.007.
+!!  
+!!
+!!***
+*/
+#include "gr_bh_gpu_Potential_Calculate_ForeignTree_Kernel.cuh"
+
+
+
+
+
+
+__device__ __forceinline__ double interp_ewald_noref(double x, double y, double z)
+{
+
+
+  double dx, dy, dz, p, q, r;
+  int i, j, k;
+  int  ip1,  jp1, kp1;
+
+
+  dx = ewald_xmax / (d_NX-1);
+  i = (int)floor(x/dx);
+/*
+  if ((i .LT. 0) .or. (i .GE. gr_bhEwaldFieldNx)) then
+    print *, i, x, dx, gr_bhEwaldXMax, gr_bhEwaldFieldNx
+    call Driver_abortFlash("FATAL: Ewald field i out of limits")
+  endif
+*/
+  dy = ewald_ymax / (d_NY-1);
+  j = (int)floor(y/dy);
+/*
+  if ((j .LT. 0) .or. (j .GE. gr_bhEwaldFieldNy)) then
+    call Driver_abortFlash("FATAL: Ewald field j out of limits")
+  endif
+*/
+  dz = ewald_zmax / (d_NZ-1);
+  k = (int)floor(z/dz);
+/*
+  if ((k .LT. 0) .or. (k .GE. gr_bhEwaldFieldNz)) then
+    call Driver_abortFlash("FATAL: Ewald field k out of limits")
+  endif
+  */
+
+
+ 
+  if (i == d_NX-1){
+    ip1 = i;
+    
+  }else{ 
+    ip1 = i+1;
+  }
+  if (j == d_NY-1) {
+    jp1 = j;
+  }else {
+    jp1 = j+1;
+ }
+  if (k == d_NZ-1) {
+    kp1 = k;
+  }else {
+    kp1 = k+1;
+}
+
+  p = (x - i*dx)/dx;
+  q = (y - j*dy)/dy;
+  r = (z - k*dz)/dz;
+
+
+
+ 
+  
+// linear interpolation
+ 
+
+   return ( 
+       (1.0 - p)*(1.0 - q)*(1.0 - r) * d_grid[((i)   + (j)   * (d_NX) + (k)  * (d_NX) * (d_NY))] +
+       (      p)*(1.0 - q)*(1.0 - r) * d_grid[((ip1) + (j)   * (d_NX) + (k)  * (d_NX) * (d_NY))] +
+       (1.0 - p)*(      q)*(1.0 - r) * d_grid[((i)   + (jp1) * (d_NX) + (k)  * (d_NX) * (d_NY))] +
+       (      p)*(      q)*(1.0 - r) * d_grid[((ip1) + (jp1) * (d_NX) + (k)  * (d_NX) * (d_NY))] +
+       (1.0 - p)*(1.0 - q)*(      r) * d_grid[((i)   + (j)   * (d_NX) + (kp1)* (d_NX) * (d_NY))] +
+       (      p)*(1.0 - q)*(      r) * d_grid[((ip1) + (j)   * (d_NX) + (kp1)* (d_NX) * (d_NY))] +
+       (1.0 - p)*(      q)*(      r) * d_grid[((i)   + (jp1) * (d_NX) + (kp1)* (d_NX) * (d_NY))] +
+       (      p)*(      q)*(      r) * d_grid[((ip1) + (jp1) * (d_NX) + (kp1)* (d_NX) * (d_NY))]  
+       );
+
+
+
+
+
+
+}
+
+
+
+
+
+
+
+
+
+#if __CUDA_ARCH__ >= 300
+__launch_bounds__(512,4)
+#else 
+__launch_bounds__(256,6)
+#endif
+__global__
+void ForceCalculationKernel_periodic_step_deep(int n_old,double *selfx, double *selfy, double *selfz)
+{
+  register int  j, k,n, depth, warpID, laneID, diff,ch;
+  register double px, py, pz, ax, dx, dy, dz, tmp, size;
+
+  
+#if __CUDA_ARCH__ >= 300
+__shared__ volatile int stack[(MAXDEPTH * 8) * (512 /WARPSIZE)];
+#else 
+ __shared__ volatile int stack[(MAXDEPTH * 8) * (THREADS1 /WARPSIZE)];
+#endif
+
+
+
+// figure out first thread in each warp (lane 0)
+   
+  //  warpID = threadIdx.x / WARPSIZE; // number the warps
+    laneID = threadIdx.x % WARPSIZE;
+    //sbase = base * WARPSIZE; //all threads in a warp now have same sbase. (lane 0 IF (threadidx.x == sbase))
+    
+
+
+    warpID = threadIdx.x / WARPSIZE; // number the warps
+
+    diff = warpID * (MAXDEPTH * 8);
+
+ 
+    // iterate over all bodies assigned to thread
+    for (k = threadIdx.x + blockIdx.x * blockDim.x; k < n_old; k += blockDim.x * gridDim.x) {
+      //i = sortd[k];  // get permuted/sorted index
+      // cache position info
+      px = selfx[k];
+      py = selfy[k];
+      pz = selfz[k];
+
+      ax = 0;
+    
+      // initialize iteration stack, i.e., push root node onto stack
+      
+      if (laneID==0) {
+          stack[diff+1]=nnodesd;
+         
+          
+      }
+      depth = 1;
+     
+        
+       while (depth > 0) 
+           {
+   
+          
+              n = stack[depth+diff];
+         
+
+      
+              
+                 dx = posxd[n] - px;
+                 dy = posyd[n] - py;
+                 dz = poszd[n] - pz;
+                 size = posrd[n];
+               
+             if (periodic == 7 ||  periodic == 1)
+                dx = min (min(abs(dx - (d_grv_bhLx)), abs(dx +(d_grv_bhLx))), abs(dx));
+   
+             if ( periodic == 7 ||  periodic == 3 || periodic == 2 )
+                dy = min (min(abs(dy - (d_grv_bhLy)), abs(dy +(d_grv_bhLy))), abs(dy));
+
+             if ( periodic == 7 ||  periodic == 4 ||  periodic == 6  ||  periodic == 5  ) 
+                  dz = min (min(abs(dz - (d_grv_bhLz)), abs(dz +(d_grv_bhLz))), abs(dz));
+        
+                              
+                  tmp = dx*dx + (dy*dy + (dz*dz));
+              if(n < nbodiesd ||__all(tmp > size))
+              {
+                                      
+                 if (periodic != 0){ 
+                  ax += massd[n] * interp_ewald_noref(fabs(dx),fabs(dy),fabs(dz));
+                  }
+                   else{                    
+                  tmp = rsqrt(tmp); 
+                  ax += massd[n]*tmp;
+                  }
+                
+                 depth--;
+                // if(n > nbodiesd)
+                  //  printf("refined %d with size %e\n",n,size);
+               
+               }else{  
+                       depth--;  
+                        n *= 8;
+                        for (j = 0; j < 8 ; j++)
+                             {
+                             ch = childd[n + j];
+                              if(ch >= 0 )
+                                 {
+                                  depth++;
+                                  if (laneID == 0) 
+                                      stack[depth+diff]=ch;
+                                 }
+                              }
+                    
+                     }
+                  
+             
+           }//end  while (depth > 0) 
+           grv_potd[k] += ax;
+ 
+           
+        }
+ }
+    
+
+
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_SavePot.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_SavePot.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_SavePot.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,52 @@
+!!****if* source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_SavePot
+!!
+!! NAME
+!!
+!!  gr_bh_gpu_SavePot
+!!
+!! SYNOPSIS
+!!
+!!  call gr_bh_gpu_SavePot()
+!!
+!! DESCRIPTION
+!! Loop over all LEAFBLOCKS and store the calculated potential in the solution vector.
+!!
+!! ARGUMENTS
+!!
+!!   No arguments
+!!
+!! 
+!!
+!!
+!!***
+
+subroutine gr_bh_gpu_SavePot()
+
+ 
+
+ use gr_bh_gpu_data, ONLY : gr_bh_gpu_ipoisvar,&
+                            gpu_rank,gr_bh_gpu_local_cell_count
+
+ use Grid_Interface, ONLY: Grid_getListOfBlocks,Grid_getCellCoords
+ use gr_bh_gpu_interface, ONLY: gr_bh_gpu_SetGridPot
+ implicit none
+
+#include "Flash.h"
+#include "constants.h"
+ integer :: k,m
+ integer :: size1 =NXB*NYB*NZB
+ integer :: gr_bh_gpu_LeafblockCount 
+ integer :: gr_bh_gpu_LeafblockList(MAXBLOCKS)
+ 
+ call Grid_getListOfBlocks(LEAF,gr_bh_gpu_LeafblockList,gr_bh_gpu_LeafblockCount)
+if(gr_bh_gpu_local_cell_count .ge. 1)then
+      do m =0, gr_bh_gpu_LeafblockCount-1
+
+         call gr_bh_gpu_SetGridPot(m*size1,&
+                                   gr_bh_gpu_LeafblockList(m+1),0,gr_bh_gpu_ipoisvar,NXB*NYB*NZB)
+
+       end do
+
+endif
+return
+end subroutine gr_bh_gpu_SavePot
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Local.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Local.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Local.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,276 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  .
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+
+
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+
+// GENERAL TREE BUILDING
+#include "gr_bh_gpu_Init_Kernel.cuh"
+#include "gr_bh_gpu_MakeRootKernel.cuh"
+#include "gr_bh_gpu_Summarization_Kernel.cuh"
+#include "gr_bh_gpu_TreeBuilding_Kernel.cuh"
+
+
+// FORCE CALCULATION
+#include "gr_bh_gpu_Potential_Calculate_Direkt_Kernel.cuh"
+#include "gr_bh_gpu_Potential_Calculate_ForeignTree_Kernel.cuh"
+#include "gr_bh_gpu_Potential_Calculate_Local_Kernel.cuh"
+
+
+
+extern "C" {
+void gr_bh_gpu_Potential_Local_(int *device_ID,int *gpu_blocks,
+                                 int *local_cell_count,  
+                                 double **x_data,
+                                 double **y_data, 
+                                 double **z_data, 
+                                 double **m_data,
+                                 double *memory_time, 
+				 double *kernel_time,
+                                 double **root_maxxd,
+                                 double **root_maxyd,
+                                 double **root_maxzd,
+                                 double **root_minxd,
+                                 double **root_minyd,
+                                 double **root_minzd)
+    {
+
+  double *x_cells = *x_data;
+  double *y_cells =*y_data; 
+  double *z_cells =*z_data; 
+  double *m_cells =*m_data;
+
+  int  blocks = *gpu_blocks;
+  int nnodes  = (*local_cell_count) * 2;
+
+  if (nnodes < 1024*blocks)
+        nnodes = 1024*blocks;
+
+  while ((nnodes & (WARPSIZE-1)) != 0)nnodes++;
+  nnodes--;
+
+
+  cudaSetDevice(*device_ID);
+  cudaEvent_t start, stop;
+  double bytes = 0;
+  float time;    
+  int  *childl;
+  double *posxl;
+  double *posyl;
+  double *poszl;
+  double *massl;
+  double *radii; 
+  double *d_delta;
+  double *all;
+ cudaEventCreate(&start);
+ cudaEventCreate(&stop);
+ cudaEventRecord(start, 0);
+
+
+//malloc
+
+
+cudaMalloc((void **)&childl, sizeof(int) * (nnodes+1) * 8);
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: couldn't allocate childd");
+
+cudaMalloc((void **)&all, sizeof(double) * ((nnodes+1)*6));
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: couldn't allocate all");
+
+
+
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+memory_time[0] += time; 
+
+
+posxl  =all+0;
+posyl  =all+(nnodes+1);
+poszl  =all+((nnodes+1)*2);
+massl  =all+((nnodes+1)*3);
+radii  =all+((nnodes+1)*4); 
+d_delta=all+((nnodes+1)*5);
+
+
+cudaEventRecord(start, 0);
+//cpytosy
+cudaMemcpyToSymbol(nbodiesd, local_cell_count, sizeof(int));
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: nbody copy to Symbol failed");
+bytes += 1 * sizeof(int);
+
+cudaMemcpyToSymbol(nnodesd, &nnodes, sizeof(int));
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: nnode copy to Symbol failed");
+bytes += 1 * sizeof(int);
+
+cudaMemcpyToSymbol(delta, &d_delta, sizeof(void*));
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: delta copy to Symbol failed");
+
+cudaMemcpyToSymbol(massd, &massl, sizeof(void*));
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: massl copy to Symbol failed");
+
+cudaMemcpyToSymbol(posxd, &posxl, sizeof(void*)); 
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: posxl copy to Symbolfailed");
+
+
+cudaMemcpyToSymbol(childd, &childl, sizeof(void*)); 
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: childl copy to Symbol failed");
+
+
+cudaMemcpyToSymbol(posyd, &posyl, sizeof(void*));
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: posyl copy to Symbol failed");
+
+
+cudaMemcpyToSymbol(poszd, &poszl, sizeof(void*));
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: poszl copy to Symbol failed");
+
+
+cudaMemcpyToSymbol(posrd, &radii, sizeof(void*));
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: radii copy to Symbol failed");
+//cpyt
+
+   
+
+cudaMemcpy(massl, m_cells, sizeof(double) * (*local_cell_count), cudaMemcpyHostToDevice);
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: mass copy to device failed");
+bytes += (*local_cell_count) * sizeof(double);
+
+cudaMemcpy(posxl, x_cells, sizeof(double) * (*local_cell_count), cudaMemcpyHostToDevice);
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: posx copy to device failed");
+bytes += (*local_cell_count) * sizeof(double);
+
+cudaMemcpy(posyl, y_cells, sizeof(double) * (*local_cell_count), cudaMemcpyHostToDevice);
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: posy copy to device failed");
+bytes += (*local_cell_count) * sizeof(double);
+
+cudaMemcpy(poszl, z_cells, sizeof(double) * (*local_cell_count), cudaMemcpyHostToDevice); 
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: posz copy to device failed");
+bytes += (*local_cell_count) * sizeof(double);
+
+
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+
+memory_time[1] += time; 
+memory_time[2] += bytes;
+ 
+cudaEventRecord(start, 0);
+
+
+
+
+
+
+cudaMemset((void*) d_delta, 0, sizeof(double) * (nnodes+1));
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: couldn't initialize dist_com");
+
+InitializationKernel<<<1, 1>>>();
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: InitializationKernel launch failed");
+
+MakeRootKernel<<<1,1>>>(*root_minxd,*root_maxxd,*root_minyd,*root_maxyd,*root_minzd,*root_maxzd);
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: MakeRootKernel launch failed");
+
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+kernel_time[0] += time; 
+
+cudaEventRecord(start, 0);
+
+TreeBuildingKernel<<<blocks * FACTOR2, THREADS2>>>();
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: TreeBuildingKernel launch failed");
+         
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+kernel_time[1] += time;
+
+cudaEventRecord(start, 0);
+
+SummarizationKernel<<<blocks * FACTOR3, THREADS3>>>();
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: SummarizationKernel launch failed");
+
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+
+kernel_time[2] += time; 
+
+cudaEventRecord(start, 0);
+                   
+#if __CUDA_ARCH__ >= 300
+ForceCalculationKernel_periodic_self_deep<<<blocks * FACTOR8, 512>>>((*local_cell_count));
+#else
+ForceCalculationKernel_periodic_self_deep<<<blocks * FACTOR8, THREADS1>>>((*local_cell_count));
+#endif
+
+CudaTest( (char*)"gr_bh_gpu_Potential_Local: ForceCalculationKernel_periodic_self_deep launch failed");
+ 
+
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+
+
+kernel_time[3] += time;  
+
+cudaEventRecord(start, 0);
+   
+cudaFree(childl);
+cudaFree(all);
+
+
+
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+
+memory_time[0] += time; 
+
+   
+       
+cudaEventDestroy(start);
+cudaEventDestroy(stop);     
+
+
+ }
+}
+
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CalcExportTree.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CalcExportTree.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CalcExportTree.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,199 @@
+!!****if* BHTree/GPU/gr_bh_gpu_CalcExportTree
+!!
+!! NAME
+!!
+!!  gr_bh_gpu_CalcExportTree
+!!
+!! SYNOPSIS
+!!
+!!  call gr_bh_gpu_CalcExportTree()
+!!
+!! DESCRIPTION
+!!
+!! Calculate the essential data for a foreign domain. Send the essential data to the domain holder and recieve essential data sent by other PIDs.
+!!
+!! ARGUMENTS
+!!
+!!   
+!!
+!! 
+!!
+!!
+!!***
+
+
+subroutine gr_bh_gpu_CalcExportTree()
+ use Driver_interface, ONLY : Driver_abortFlash
+ use Grid_data, ONLY : gr_meshMe,gr_meshNumProcs, gr_meshComm
+ use gr_bh_gpu_data, ONLY :gr_bh_gpu_roots_received,&
+                           gr_bh_gpu_Mac_Angle,gr_bh_gpu_global_domdiagsq,gr_bh_gpu_local_domdiagsq,&
+                           gr_bh_gpu_xmax_domain,  gr_bh_gpu_xmin_domain, &
+                           gr_bh_gpu_ymax_domain,  gr_bh_gpu_ymin_domain,&
+                           gr_bh_gpu_zmax_domain,  gr_bh_gpu_zmin_domain ,&
+                           gr_bh_gpu_roots_own,    gpu_blocks,&
+                           gr_bh_gpu_periodic,essentials_memory_timings,&
+			   gr_bh_gpu_message_count,&
+				send_buffer, recv_buffer, send_request,&
+				recv_request,gr_bh_gpu_essentials_count,&
+				gr_bh_gpu_node_count,gr_bh_gpu_local_cell_count,gr_bh_gpu_pinned_mpi,&
+				device_ID
+
+
+use gr_bh_gpu_Interface, ONLY: gr_bh_gpu_Essentials_Build_Init_Wrap,gr_bh_gpu_Essentials_Build_Local_Tree,&
+			       gr_bh_gpu_Essentials_Calculate_Essentials,gr_bh_gpu_Essentials_CleanUp_Device_Wrap,&
+				gr_bh_gpu_Essentials_Send_Essentials_Wrap,gr_bh_gpu_Essentials_Probe_Messages,&
+				gr_bh_gpu_Essentials_Receive_Message
+
+use, intrinsic :: iso_c_binding
+
+use Timers_interface, ONLY : Timers_start, Timers_stop
+implicit none
+
+#include "Flash_mpi.h"
+
+ 
+
+  logical :: essentials_flag
+  integer :: sender,size1,istat
+  real,allocatable  ::  tx(:)
+  real,allocatable  ::  ty(:)
+  real,allocatable  ::  tz(:)
+  real,allocatable  ::  tm(:)
+  real ::  roots(0:4*gr_meshNumProcs)
+  type( c_ptr ) :: all_ints
+  type( c_ptr ) :: all_doubles
+ !integer  :: essential_nodes(0:(gr_bh_gpu_local_cell_count))
+  type(c_ptr)  :: essential_nodes,essential_blocks
+ !type(c_ptr) :: essential_nodes_dev,essential_blocks_dev,res
+  integer :: flag,top_index
+
+  real ::rootx,rooty,rootz,rootm
+
+  integer :: e_count
+ 
+
+  integer :: free_index,mstat(MPI_STATUS_SIZE),e_nodes,elements
+  integer :: found_depth
+  integer :: req, mpistat(MPI_STATUS_SIZE)
+  integer :: receiver,offset,recv_msg_size,sender_ID
+
+ 
+  free_index = -1
+  flag = 0
+  top_index = -1
+  receiver = 0
+  offset = 0
+!!===================GPU==GPU==GPU=====================================
+  if(gr_bh_gpu_local_cell_count .gt. 1) then
+  do receiver=0, gr_meshNumProcs - 1
+  
+                roots(offset + 0)= gr_bh_gpu_roots_received(0 + receiver * 12)
+                roots(offset + 1)= gr_bh_gpu_roots_received(1 + receiver * 12)
+                roots(offset + 2)= gr_bh_gpu_roots_received(2 + receiver * 12)
+                roots(offset + 3)= gr_bh_gpu_roots_received(4 + receiver * 12)
+   offset = offset + 4
+   end do
+
+!====================root size = DOMAIN size ======================!
+
+
+       call gr_bh_gpu_Essentials_Build_Local_Tree(tx,ty,tz,tm,&
+						roots,&
+						essential_nodes,essential_blocks,&
+						found_depth,all_ints,all_doubles)
+
+
+
+
+!====================root size = DOMAIN size END======================!
+
+      offset = 0
+       do receiver=0, gr_meshNumProcs - 1 !WALK OVER ALL RANKS
+         
+          if(receiver .ne. gr_meshMe) then !NOT SELF RANK
+             e_count = 0
+              if( gr_bh_gpu_roots_received(3 + receiver * 12) .gt. 0 ) then !OTHER RANK HAS DATA
+       
+       
+                  call gr_bh_gpu_Essentials_Calculate_Essentials(offset,e_count,found_depth,roots,all_doubles,all_ints,&
+						    essential_nodes,essential_blocks)
+                  
+       
+
+                  call gr_bh_gpu_Essentials_Send_Essentials_Wrap(send_buffer, send_request,&
+				             		      tx, ty, tz, tm,&
+				              		      e_count, top_index, receiver,&
+                                               		      gr_meshMe, gr_meshNumProcs,gr_meshComm,&
+				             		      essential_nodes,gr_bh_gpu_pinned_mpi)
+
+	
+              else !OTHER RANK HAS NO DATA
+                 e_count = 0
+                  call gr_bh_gpu_Essentials_Send_Essentials_Wrap(send_buffer, send_request,&
+				             		      tx, ty, tz, tm,&
+				              		      e_count, top_index, receiver,&
+                                               		      gr_meshMe, gr_meshNumProcs,gr_meshComm,&
+				             		      essential_nodes,gr_bh_gpu_pinned_mpi)
+
+
+              endif !OTHER RANK HAS DATA
+
+           
+
+!=============
+             essentials_flag = .false.
+             call gr_bh_gpu_Essentials_Probe_Messages(essentials_flag,sender_ID,recv_msg_size)
+
+          
+             if(essentials_flag)then
+
+               call gr_bh_gpu_Essentials_Receive_Message(recv_buffer,recv_request,&
+						    recv_msg_size,&
+						    sender_ID,&
+						    gr_meshMe,&
+						    gr_meshComm,gr_bh_gpu_pinned_mpi)
+                  gr_bh_gpu_essentials_count = gr_bh_gpu_essentials_count + recv_msg_size    
+				gr_bh_gpu_message_count = gr_bh_gpu_message_count +1 
+	     endif
+      
+        
+
+
+
+
+
+
+        endif !NOT SELF RANK
+       offset = offset + 4
+       end do
+ !CLEAR GPU DATA!
+
+ call gr_bh_gpu_Essentials_CleanUp_Device_Wrap(device_ID, all_doubles,all_ints,&
+						essential_nodes,essential_blocks,essentials_memory_timings)
+
+
+ else !(gr_bh_gpu_localblocks%num_nodes .gt. 1) then
+!EMPTY DATA
+ do receiver=0, gr_meshNumProcs-1
+   if(receiver .ne. gr_meshMe) then
+   
+   
+                 e_count = 0
+                  call gr_bh_gpu_Essentials_Send_Essentials_Wrap(send_buffer, send_request,&
+				             		      tx, ty, tz, tm,&
+				              		      e_count, top_index, receiver,&
+                                               		      gr_meshMe, gr_meshNumProcs,gr_meshComm,&
+				             		      essential_nodes,gr_bh_gpu_pinned_mpi)
+
+   end if
+ end do
+endif
+
+if(allocated(tx))deallocate(tx)
+if(allocated(ty))deallocate(ty)
+if(allocated(tz))deallocate(tz)
+if(allocated(tm))deallocate(tm)
+
+return
+end subroutine gr_bh_gpu_CalcExportTree
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_TreeBuilding_Kernel.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_TreeBuilding_Kernel.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_TreeBuilding_Kernel.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,144 @@
+#include "gr_bh_gpu_TreeBuilding_Kernel.cuh"    
+
+
+
+
+__global__
+__launch_bounds__(THREADS2, FACTOR2)
+void TreeBuildingKernel()
+{
+  register int i, j, k, depth, localmaxdepth, skip, inc;
+  register double x, y, z, r;
+  register double px, py, pz;
+  register int ch, n, cell, locked, patch;
+  register double radius, rootx, rooty, rootz;
+  
+  // cache root data
+  radius = radiusd;
+  rootx = posxd[nnodesd];
+  rooty = posyd[nnodesd];
+  rootz = poszd[nnodesd];
+  //posrd[nnodesd]=  diagonal;
+   localmaxdepth = 1;
+  skip = 1;
+  inc = blockDim.x * gridDim.x;
+  i =  threadIdx.x + blockIdx.x * blockDim.x;
+
+
+  // iterate over all bodies assigned to thread
+ 
+  while (i < nbodiesd) {
+    if (skip != 0) {
+      // new body, so start traversing at root
+      skip = 0;
+      px = posxd[i];
+      py = posyd[i];
+      pz = poszd[i];
+
+      n = nnodesd;
+     
+      depth = 1;
+      r = radius;
+      j = 0;
+      // determine which child to follow
+      if (rootx < px) j = 1;
+      if (rooty < py) j += 2;
+      if (rootz < pz) j += 4;
+
+    
+    }
+
+    // follow path to leaf cell
+    ch = childd[n*8+j];
+    while (ch >= nbodiesd) {
+      n = ch;
+      depth++;
+      r *= 0.5;
+ 
+      j = 0;
+      // determine which child to follow
+      if (posxd[n] < px) j = 1;
+      if (posyd[n] < py) j += 2;
+      if (poszd[n] < pz) j += 4;
+      ch = childd[n*8+j];
+     
+    }
+
+    if (ch != -2) {  // skip if child pointer is locked and try again later
+      locked = n*8+j;
+      if (ch == atomicCAS((int *)&childd[locked], ch, -2)) {  // try to lock
+        if (ch == -1) {
+          // if null, just insert the new body
+          childd[locked] = i;
+          
+       //   printf("inserted a body %d\n", i);
+        } else {  // there already is a body in this position
+          patch = -1;
+          // create new cell(s) and insert the old and new body
+          do {
+            depth++;
+
+            cell = atomicSub((int *)&bottomd, 1) - 1;
+            if (cell <= nbodiesd) {
+             // printf("ERROR\n");
+              bottomd = nnodesd;
+            }
+            patch = max(patch, cell);
+
+            x = (j & 1) * r;
+            y = ((j >> 1) & 1) * r;
+            z = ((j >> 2) & 1) * r;
+            r *= 0.5;
+            //d *= 0.5;
+
+            massd[cell] = -1.0;
+           
+            posrd[cell] = posrd[n] * 0.25;
+            x = posxd[cell] = posxd[n] - r + x;
+            y = posyd[cell] = posyd[n] - r + y;
+            z = poszd[cell] = poszd[n] - r + z;
+        
+            for (k = 0; k < 8; k++)
+               { 
+               childd[cell*8+k] = -1;
+               //posrd[cell*8+k] = posrd[cell]*0.25;
+               }
+
+            if (patch != cell) {
+              childd[n*8+j] = cell;
+             
+            }
+
+            j = 0;
+            if (x < posxd[ch]) j = 1;
+            if (y < posyd[ch]) j += 2;
+            if (z < poszd[ch]) j += 4;
+            childd[cell*8+j] = ch;
+            
+            n = cell;
+            j = 0;
+            if (x < px) j = 1;
+            if (y < py) j += 2;
+            if (z < pz) j += 4;
+
+            ch = childd[n*8+j];
+            // repeat until the two bodies are different children
+          } while (ch >= 0);
+          childd[n*8+j] = i;
+          __threadfence();  // push out subtree
+          childd[locked] = patch;
+        }
+
+        localmaxdepth = max(depth, localmaxdepth);
+        i += inc;  // move on to next body
+        skip = 1;
+      }
+    }
+    __syncthreads();  // throttle
+  }
+  // record maximum tree depth
+  atomicMax((int *)&maxdepthd, localmaxdepth);
+ 
+
+}
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CleanUp.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CleanUp.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CleanUp.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,85 @@
+!!****if* BHTree/GPU/gr_bh_gpu_CleanUp
+!!
+!! NAME
+!!
+!!  gr_bh_gpu_CleanUp
+!!
+!! SYNOPSIS
+!!
+!!  call gr_bh_gpu_CleanUp()
+!!
+!! DESCRIPTION
+!!
+!! Deletes allocated data, and resets timers.
+!!
+!! ARGUMENTS
+!!
+!!   No arguments
+!!
+!! AUTOGENROBODOC
+!!
+!!
+!!***
+
+subroutine gr_bh_gpu_CleanUp()
+use Grid_data, ONLY : gr_meshNumProcs
+use gr_bh_gpu_data, ONLY : gpu_comm,gr_bh_gpu_local_cell_count, &
+                             essentials_memory_timings,potential_memory_timings,&
+			    essentials_kernel_timings,potential_kernel_timings,&  
+                           first_call, send_buffer, recv_buffer,send_request, recv_request,&
+			   gr_bh_gpu_pinned_mpi,gr_bh_gpu_x_data,gr_bh_gpu_y_data,&
+			   gr_bh_gpu_z_data,gr_bh_gpu_m_data,gr_bh_gpu_p_data,gr_bh_gpu_pinned_cells
+
+
+use gr_bh_gpu_Interface, ONLY: gr_bh_gpu_Clear_Buffers,gr_bh_gpu_CleanUp_CellData
+
+
+
+implicit none
+#include "Flash_mpi.h"
+#include "Flash.h"
+integer :: m,ierr
+integer :: mstat(0:MPI_STATUS_SIZE*gr_meshNumprocs)
+
+
+
+
+
+!====================
+! WAIT for Sender and deallocate send buffers
+!====================
+if(gr_meshNumProcs .gt. 1) then
+ call gr_bh_gpu_Clear_Buffers(send_buffer, recv_buffer,send_request, recv_request,gr_meshNumProcs,gr_bh_gpu_pinned_mpi)
+endif    
+         
+   
+if(gr_bh_gpu_local_cell_count .gt. 1)then 
+	call gr_bh_gpu_CleanUp_CellData(gr_bh_gpu_x_data,gr_bh_gpu_y_data,gr_bh_gpu_z_data,&
+					gr_bh_gpu_m_data,gr_bh_gpu_p_data,gr_bh_gpu_pinned_cells)
+endif
+
+
+
+
+  
+  
+
+
+
+
+
+  
+if(first_call .eqv. .true.)then
+ first_call=.false.
+	essentials_memory_timings= 0
+	potential_memory_timings=0
+	essentials_kernel_timings=0
+	potential_kernel_timings=0
+endif
+  
+ 
+  
+
+ 
+return
+end subroutine gr_bh_gpu_CleanUp
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Calculate_Nodes_Kernel.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Calculate_Nodes_Kernel.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Essentials_Calculate_Nodes_Kernel.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,159 @@
+#include "gr_bh_gpu_Essentials_Calculate_Nodes_Kernel.cuh"  
+
+
+  
+__global__
+void NeighborsKernel_Force_assure(int *essentials)
+{
+ int l= threadIdx.x + blockIdx.x * blockDim.x;
+ 
+
+  for(; l <= nnodesd ; l += blockDim.x * gridDim.x)
+              {  essentials[l]= (MAXDEPTH+1) *-1;
+                 if(l == nnodesd)
+                   essentials[l]=-1;
+              }
+}
+
+
+__global__
+void NeighborsKernel_Force_copy(int *essentials, int  *mediate, int count)
+{
+ int l;
+
+for(l = threadIdx.x + blockIdx.x * blockDim.x; l <= count ; l += blockDim.x * gridDim.x)
+   {
+    mediate[l]=essentials[l];
+
+   }
+  
+}
+
+
+
+      
+__global__
+void NeighborsKernel_Force_nodes(int depth,double nodediag,double *roots,int offset,int *essentials, int *cnt)
+{
+  register int l,j,ch,dp,dp2,bottom;
+  register double px,py,pz,pr,x,y,z,dx,dy,dz,tmp,r;
+   __shared__ int sdata[512];
+  bottom = bottomd;
+  px = roots[offset + 0];
+  py = roots[offset + 1];
+  pz = roots[offset + 2];
+  pr = roots[offset + 3];
+  dp = depth * -1;
+  dp2 = (depth+1)* -1;
+  sdata[threadIdx.x]=0;
+          for(l =  bottom + threadIdx.x + blockIdx.x * blockDim.x; l <= nnodesd /*end*/ ; l += blockDim.x * gridDim.x)
+              {  
+                 
+                  if(essentials[l]==dp)
+                    {                     
+                   //  px = roots[offset + 0];
+                   //  py = roots[offset + 1];
+                   //  pz = roots[offset + 2];
+                   //  pr = roots[offset + 3];
+                    
+                     x = posxd[l]; //CENTER OF MASS
+                     y = posyd[l];
+                     z = poszd[l];
+                   
+                     r = nodediag +  delta[l];
+
+
+
+
+                     dx = max( max( (px - pr) - x , 0.0),
+                       max(  x - (px + pr), 0.0 )  );
+ 
+                     dy = max( max( (py - pr) - y , 0.0),
+                       max(  y - (py + pr), 0.0 )  );
+
+                     dz = max( max( (pz - pr) - z , 0.0),
+                       max(  z - (pz + pr), 0.0 )  );
+
+     
+
+                    if (periodic == 7 ||  periodic == 1)
+                     dx = min ( dx , min( max( max( (px - pr) - (x-d_grv_bhLx) , 0.0),max(  (x-d_grv_bhLx) - (px + pr), 0.0 )  )
+                                     , max( max( (px - pr) - (x+d_grv_bhLx) , 0.0),max(  (x+d_grv_bhLx) - (px + pr), 0.0 )  )));
+
+
+
+                   if( periodic == 7 ||  periodic == 3 ||  periodic == 2 ) 
+                     dy = min ( dy , min( max( max( (py - pr) - (y-d_grv_bhLy) , 0.0),max(  (y-d_grv_bhLy) - (py + pr), 0.0 )  )
+                                     , max( max( (py - pr) - (y+d_grv_bhLy) , 0.0),max(  (y+d_grv_bhLy) - (py + pr), 0.0 )  )));
+
+
+                   if( periodic == 7 ||  periodic == 4 ||  periodic == 6  ||  periodic == 5  ) 
+                     dz = min ( dz , min( max( max( (pz - pr) - (z-d_grv_bhLz) , 0.0),max(  (z-d_grv_bhLz) - (pz + pr), 0.0 )  )
+                                     , max( max( (pz - pr) - (z+d_grv_bhLz) , 0.0),max(  (z+d_grv_bhLz) - (pz + pr), 0.0 )  )));
+
+                 
+  
+                   
+                   
+
+            tmp = (dx*dx + (dy*dy + (dz*dz))) ;
+               if(tmp > r || l < nbodiesd)
+                  {
+                   
+                  essentials[l]=l;
+                 sdata[threadIdx.x] +=1;
+                   }
+               else
+                   {
+                    
+                      
+
+                     for (j = 0; j < 8; j++) 
+                          {
+                           
+                            ch = childd[l*8+j];
+                            if(ch >= 0 )
+                             {
+                              if(ch < nbodiesd)
+                                {
+ 				essentials[ch]=ch;
+				sdata[threadIdx.x] +=1;
+                                }
+                              if(ch >=nbodiesd){
+                              essentials[ch] = dp2;
+			      sdata[threadIdx.x] +=1;
+					}
+
+                             }
+                          
+
+                            
+                          } 
+                   }
+
+
+
+                    }
+                   
+                }
+ __syncthreads();
+
+if (blockDim.x >= 512) { if (threadIdx.x < 256) { sdata[threadIdx.x] += sdata[threadIdx.x + 256]; } __syncthreads(); } 
+if (blockDim.x >= 256) { if (threadIdx.x < 128) { sdata[threadIdx.x] += sdata[threadIdx.x + 128]; } __syncthreads(); } 
+if (blockDim.x >= 128) { if (threadIdx.x < 64)  { sdata[threadIdx.x] += sdata[threadIdx.x +  64]; } __syncthreads(); }
+
+if(threadIdx.x < 32)
+  {
+  if (blockDim.x >= 64)sdata[threadIdx.x] += sdata[threadIdx.x + 32];
+  if (blockDim.x >= 32)sdata[threadIdx.x] += sdata[threadIdx.x + 16];
+  if (blockDim.x >= 16)sdata[threadIdx.x] += sdata[threadIdx.x + 8];
+  if (blockDim.x >= 8) sdata[threadIdx.x] += sdata[threadIdx.x + 4];
+  if (blockDim.x >= 4) sdata[threadIdx.x] += sdata[threadIdx.x + 2];
+  if (blockDim.x >= 2) sdata[threadIdx.x] += sdata[threadIdx.x + 1];
+  }
+ if (threadIdx.x == 0){
+      cnt[blockIdx.x]=sdata[0];                
+    }
+
+                 
+}
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CleanUp_CellData.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CleanUp_CellData.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CleanUp_CellData.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,74 @@
+/*
+!!
+!!
+!! NAME
+!!  
+!!
+!! SYNOPSIS
+!!
+!! DESCRIPTION
+!!
+!!  
+!!  
+!! ARGUMENTS
+!!
+!!
+!!  
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!! SEE ALSO
+!! 
+!! AUTHOR
+!! Gunther Lukat <g.lukat@gmx.net>
+!!
+!!***
+*/
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+
+extern "C" {
+void gr_bh_gpu_CleanUp_CellData_(double **x_data, 
+			       double **y_data, 
+			       double **z_data, 
+			       double **m_data,
+			       double **pot_data,
+			       bool *pinned)//
+    {
+
+
+
+        if(*pinned){
+
+
+        if(*x_data != NULL)cudaFreeHost(*x_data);
+	if(*y_data != NULL)cudaFreeHost(*y_data);
+	if(*z_data != NULL)cudaFreeHost(*z_data);
+	if(*m_data != NULL)cudaFreeHost(*m_data);
+	if(*pot_data != NULL)cudaFreeHost(*pot_data);
+
+        }
+        else{
+
+
+        if(*x_data != NULL)free(*x_data);
+       // *x_dtata = NULL
+	if(*y_data != NULL)free(*y_data);
+      // *y_dtata = NULL
+	if(*z_data != NULL)free(*z_data);
+       // *z_dtata = NULL
+	if(*m_data != NULL)free(*m_data);
+       // *m_dtata = NULL
+        if(*pot_data != NULL)free(*pot_data);
+        //*pot_dtata = NULL
+
+
+            }
+
+    }
+}
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_MakeRootKernel.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_MakeRootKernel.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_MakeRootKernel.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,36 @@
+
+#include "gr_bh_gpu_MakeRootKernel.cuh"
+__launch_bounds__(THREADS0, FACTOR3)
+__global__ void MakeRootKernel(double *roots_minx,double *roots_maxx,double *roots_miny,double *roots_maxy,double *roots_minz,double *roots_maxz)
+{
+  register int i, j, k;
+  register double val;
+ 
+  i = threadIdx.x ;
+
+if(i == 0){
+ val = max(*roots_maxx - *roots_minx, *roots_maxy - *roots_miny);
+ radiusd2 = radiusd = max(val, *roots_maxz - *roots_minz)* 0.5; // inner radius
+
+      // create root node
+      k = nnodesd;
+      bottomd = k;
+
+      massd[k] = -1.0;
+
+      posxd[k] = (*roots_minx + *roots_maxx) * 0.5;
+      posyd[k] = (*roots_miny + *roots_maxy) * 0.5;
+      poszd[k] = (*roots_minz + *roots_maxz) * 0.5;
+      posrd[k] = domdiag *itolsqd;
+     
+
+
+      k *= 8;
+      for (j = 0; j < 8; j++)
+      {
+       childd[k + j] = -1;
+    
+      } 
+  
+}
+}
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_set_nodes.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_set_nodes.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_set_nodes.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,20 @@
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+
+extern "C" {
+void gr_bh_gpu_set_nodes_(int *blocks,int *cell_count,int *_nodes)
+{
+
+
+
+     int nnodes = *cell_count * 2;
+     if (nnodes < 1024*(*blocks)) nnodes = 1024* (*blocks);
+     while ((nnodes & (WARPSIZE-1)) != 0) nnodes++;
+     nnodes--;
+
+
+        *_nodes = nnodes;
+
+}
+}
+
+     
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Subset.cu
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Subset.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Potential_Subset.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,250 @@
+
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+
+// GENERAL TREE BUILDING
+#include "gr_bh_gpu_Init_Kernel.cuh"
+#include "gr_bh_gpu_MakeRootKernel.cuh"
+#include "gr_bh_gpu_Summarization_Kernel.cuh"
+#include "gr_bh_gpu_TreeBuilding_Kernel.cuh"
+
+
+// FORCE CALCULATION
+#include "gr_bh_gpu_Potential_Calculate_Direkt_Kernel.cuh"
+#include "gr_bh_gpu_Potential_Calculate_ForeignTree_Kernel.cuh"
+#include "gr_bh_gpu_Potential_Calculate_Local_Kernel.cuh"
+
+
+extern "C" {
+void gr_bh_gpu_Potential_Subset_(int *device_ID,int *gpu_blocks,
+                                 int *local_cell_count,
+                                 int *other_cell_count,
+                                 int *start_idx, 
+                                 double **x_data,
+                                 double **y_data, 
+                                 double **z_data, 
+                                 double **m_data,
+                                 double **root_maxxd,
+                                 double **root_maxyd,
+                                 double **root_maxzd,
+                                 double **root_minxd,
+                                 double **root_minyd,
+                                 double **root_minzd, 
+                                 double **xd, 
+                                 double **yd,
+                                 double **zd, 
+                                 double *memory_time, 
+				 double *kernel_time)
+    {
+
+
+  int  blocks = *gpu_blocks;
+  int idx = *start_idx;
+  double  bytes = 0;
+  double *all_x_coords = *x_data;
+  double *all_y_coords =*y_data; 
+  double *all_z_coords =*z_data; 
+  double *all_mass =*m_data;
+
+ 
+  float time; 
+  int nnodes; 
+  double *d_delta; 
+  int  *childl;
+  double *massl;
+  double *posxl;
+  double *posyl;
+  double *poszl;
+  double *radii;
+  double *all;
+
+  cudaSetDevice(*device_ID);
+  cudaEvent_t start, stop;
+  cudaEventCreate(&start);
+  cudaEventCreate(&stop);
+  cudaEventRecord(start, 0);
+
+
+      nnodes = (*other_cell_count) * 2;
+      if (nnodes < 1024*blocks) nnodes = 1024*blocks;
+      while ((nnodes & (WARPSIZE-1)) != 0) nnodes++;
+      nnodes--;
+
+
+
+//malloc 
+
+cudaEventRecord(start, 0);
+
+cudaMalloc((void **)&childl, sizeof(int) * (nnodes+1) * 8);
+CudaTest( (char*)"gr_bh_gpu_Potential_Subset: couldn't allocate childd");
+
+cudaMalloc((void **)&all, sizeof(double) * ((nnodes+1)*6)); 
+CudaTest( (char*)"gr_bh_gpu_Potential_Subset:  gr_bh_gpu_calcforces : couldn't allocate all");
+
+
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+
+memory_time[0] += time; 
+
+
+posxl  =all+0;
+posyl  =all+(nnodes+1);
+poszl  =all+((nnodes+1)*2);
+massl  =all+((nnodes+1)*3);
+radii  =all+((nnodes+1)*4); 
+d_delta=all+((nnodes+1)*5);
+
+
+
+cudaEventRecord(start, 0);
+
+//cpy to symb
+
+cudaMemcpyToSymbol(nbodiesd, (other_cell_count), sizeof(int)); 
+CudaTest( (char*)"gr_bh_gpu_Potential_Subset: nbody copy to Symbol failed");
+bytes += 1* sizeof(int);
+
+cudaMemcpyToSymbol(nnodesd, &nnodes, sizeof(int));
+CudaTest( (char*)"gr_bh_gpu_Potential_Subset: nnode copy to Symbole failed");
+bytes += 1* sizeof(int);
+
+cudaMemcpyToSymbol(delta, &d_delta, sizeof(void*));
+CudaTest( (char*)"gr_bh_gpu_Potential_Subset: delta copy to Symbol failed");
+
+
+cudaMemcpyToSymbol(childd, &childl, sizeof(void*));
+CudaTest( (char*)"gr_bh_gpu_Potential_Subset: childl copy to Symbol failed");
+
+
+cudaMemcpyToSymbol(massd, &massl, sizeof(void*));
+CudaTest( (char*)"gr_bh_gpu_Potential_Subset: mass copy to Symbol failed");
+
+
+cudaMemcpyToSymbol(posxd, &posxl, sizeof(void*));
+CudaTest( (char*)"gr_bh_gpu_Potential_Subset: posxl copy to device failed");
+
+
+cudaMemcpyToSymbol(posyd, &posyl, sizeof(void*));
+CudaTest( (char*)"gr_bh_gpu_Potential_Subset: posyl copy to device failed");
+
+
+cudaMemcpyToSymbol(poszd, &poszl, sizeof(void*));
+CudaTest( (char*)"gr_bh_gpu_Potential_Subset: posyl copy to device failed");
+
+cudaMemcpyToSymbol(posrd, &radii, sizeof(void*));
+CudaTest( (char*)"gr_bh_gpu_Potential_Subset: radii copy to device failed");
+
+//cpy
+cudaMemcpy(massl, all_mass + idx, sizeof(double) * (*other_cell_count), cudaMemcpyHostToDevice);
+CudaTest( (char*)"gr_bh_gpu_Potential_Subset: mass copy to device failed");
+bytes += (*other_cell_count) * sizeof(double);
+
+cudaMemcpy(posxl, all_x_coords + idx, sizeof(double) * (*other_cell_count), cudaMemcpyHostToDevice);
+CudaTest( (char*)"gr_bh_gpu_Potential_Subset: posx copy to device failed");
+bytes += (*other_cell_count) * sizeof(double);
+
+cudaMemcpy(posyl, all_y_coords + idx, sizeof(double) * (*other_cell_count), cudaMemcpyHostToDevice);
+CudaTest( (char*)"gr_bh_gpu_Potential_Subset: posy copy to device failed");
+bytes += (*other_cell_count) * sizeof(double);
+
+cudaMemcpy(poszl, all_z_coords + idx, sizeof(double) * (*other_cell_count), cudaMemcpyHostToDevice);
+CudaTest( (char*)"gr_bh_gpu_Potential_Subset: posz copy to device failed");
+bytes += (*other_cell_count) * sizeof(double);
+
+//memset
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+memory_time[1] += time;
+memory_time[2] += bytes;
+
+
+
+
+cudaEventRecord(start, 0);
+
+
+cudaMemset((void*) d_delta, 0, sizeof(double) * (nnodes+1));
+CudaTest( (char*)"gr_bh_gpu_Potential_Subset: couldn't initialize delta");
+
+InitializationKernel<<<1, 1>>>();
+CudaTest( (char*)"gr_bh_gpu_Potential_Subset: InitializationKernel launch failed");
+
+
+MakeRootKernel<<<1,1>>>(*root_minxd,*root_maxxd,*root_minyd,*root_maxyd,*root_minzd,*root_maxzd);
+CudaTest( (char*)"gr_bh_gpu_Potential_Subset: MakeRootKernel launch failed");            
+ 
+
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+
+kernel_time[0] += time; 
+
+cudaEventRecord(start, 0);
+
+TreeBuildingKernel<<<blocks * FACTOR2, THREADS2>>>();
+CudaTest( (char*)"gr_bh_gpu_Potential_Subset: TreeBuildingKernel launch failed");
+         
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+
+kernel_time[1] += time; 
+
+cudaEventRecord(start, 0);
+
+SummarizationKernel<<<blocks * FACTOR3, THREADS3>>>();
+CudaTest( (char*)"SummarizationKernel launch failed CUDA ERROR");
+
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+
+kernel_time[2] += time; 
+
+cudaEventRecord(start, 0);
+
+
+#if __CUDA_ARCH__ >= 300
+ForceCalculationKernel_periodic_step_deep<<<blocks * FACTOR8, 512>>>(*local_cell_count, *xd, *yd, *zd);
+#else
+ForceCalculationKernel_periodic_step_deep<<<blocks * FACTOR8, THREADS1>>>(*local_cell_count, *xd, *yd, *zd);
+#endif               
+CudaTest( (char*)"ForceCalculationKernel_periodic_step_deep launch failed");           
+ 
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+
+kernel_time[3] += time; 
+
+cudaEventRecord(start, 0);
+
+cudaFree(childl);
+cudaFree(all);
+
+      
+cudaEventRecord(stop, 0);
+cudaEventSynchronize(start); 
+cudaEventSynchronize(stop);
+cudaEventElapsedTime(&time, start, stop);
+
+    memory_time[0] += time; 
+
+   
+  cudaEventDestroy(start);
+  cudaEventDestroy(stop);
+
+
+ }
+}
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CalcPotSelf.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CalcPotSelf.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_CalcPotSelf.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,87 @@
+!!****if* BHTree/GPU/gr_bh_gpu_CalcPotSelf
+!!
+!! NAME
+!!
+!!  gr_bh_gpu_CalcPotSelf
+!!
+!! SYNOPSIS
+!!
+!!  call gr_bh_gpu_CalcPotSelf()
+!!
+!! DESCRIPTION
+!!
+!! Calculate the gravitational potential on the GPU-Device. This version is used if foreign data is used.
+!!
+!! ARGUMENTS
+!!
+!!   No arguments
+!!
+!!
+!!
+!!***
+
+
+
+subroutine gr_bh_gpu_CalcPotSelf()
+
+   use Driver_interface, ONLY : Driver_abortFlash
+   use Grid_data, ONLY : gr_meshMe,gr_meshNumProcs, gr_meshComm
+   use gr_bh_gpu_data, ONLY: gr_bh_gpu_Mac_Angle,gr_bh_gpu_global_domdiagsq,&
+                             gr_bh_gpu_GravFac,gr_bh_gpu_EwaldField,gr_bh_gpu_EwaldSize,&
+                             gr_bh_gpu_MAX_GPU_CELLS,&
+                             gr_bh_gpu_roots_own,gpu_blocks,gr_bh_gpu_periodic,gr_bh_gpu_global_domdiagsq, & 
+                             gr_bh_gpu_xmin_domain, gr_bh_gpu_ymin_domain,gr_bh_gpu_zmin_domain,gr_bh_gpu_xmax_domain,&
+                             gr_bh_gpu_ymax_domain,gr_bh_gpu_zmax_domain,&
+                             gr_bh_gpu_total_cell_count,gr_bh_gpu_local_cell_count, & 
+                             potential_memory_timings,potential_kernel_timings,&
+ 			     gr_bh_gpu_x_data,gr_bh_gpu_y_data,gr_bh_gpu_z_data,&
+			     gr_bh_gpu_m_data,gr_bh_gpu_p_data,device_ID
+                             
+   use gr_bh_gpu_Interface , ONLY: gr_bh_gpu_Potential_Simple
+   use, intrinsic :: iso_c_binding
+   implicit none
+   integer :: ncells 
+   
+    integer :: n,k
+    real :: ax,tmp,px,py,pz,dx,dy,dz 
+   ncells = gr_bh_gpu_total_cell_count
+   gr_bh_gpu_total_cell_count = gr_bh_gpu_local_cell_count
+   if( ncells .ge. gr_bh_gpu_MAX_GPU_CELLS)then
+       call Driver_abortFlash("Reached the maximum cell number of 30 MIO")
+   endif
+
+
+ !===============================root size = domain size=================================!      
+ call gr_bh_gpu_Potential_Simple(device_ID,gpu_blocks,gr_bh_gpu_Mac_Angle, &
+                                    gr_bh_gpu_total_cell_count, &
+				    gr_bh_gpu_x_data,&
+                                    gr_bh_gpu_y_data,&
+                                    gr_bh_gpu_z_data,&
+                                    gr_bh_gpu_m_data,&
+                                    gr_bh_gpu_p_data, &
+                                    gr_bh_gpu_global_domdiagsq, & 
+                                    gr_bh_gpu_xmin_domain,&
+                                    gr_bh_gpu_ymin_domain,&
+                                    gr_bh_gpu_zmin_domain,&
+                                    gr_bh_gpu_xmax_domain,&
+                                    gr_bh_gpu_ymax_domain,&
+                                    gr_bh_gpu_zmax_domain, &                            
+                                    gr_bh_gpu_EwaldField,gr_bh_gpu_EwaldSize, &
+				    gr_bh_gpu_periodic,&
+                                    potential_memory_timings,& 
+                                    potential_kernel_timings)
+
+ !===============================root size = domain size END =================================! 
+ 
+!!=====================================================================
+ !    endif
+     
+   
+
+  
+
+
+
+return
+end subroutine gr_bh_gpu_CalcPotSelf
+
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Finalize.F90
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Finalize.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Finalize.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,338 @@
+!!****if* source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Finalize.F90
+!!
+!! NAME
+!!  gr_bh_gpu_Finalize
+!!
+!! SYNOPSIS
+!!
+!!  call gr_bh_gpu_Finalize()
+!!
+!! DESCRIPTION
+!!
+!!  This routine deallocates data fields for the GPU accelerated BH tree and prints/gathers final performance data.
+!!
+!! AUTHOR
+!!
+!!  © 2016 Gunther Lukat 
+!!
+!!  
+!!  
+!! ARGUMENTS
+!!
+!!
+!! RESULT
+!!  
+!!
+!! PARAMETERS
+!!
+!! NOTES
+!!
+!!
+!!***
+
+subroutine gr_bh_gpu_Finalize()
+use Grid_data, ONLY : gr_meshNumProcs, gr_meshComm,gr_meshMe
+use Logfile_interface, ONLY: Logfile_init, Logfile_stamp, Logfile_open, Logfile_close
+use gr_bh_gpu_data, ONLY : gpu_comm,&
+                            gr_bh_gpu_EwaldField,&
+                            gr_bh_gpu_roots_received,&                    
+                            essentials_memory_timings,potential_memory_timings,&
+			                essentials_kernel_timings,potential_kernel_timings,&  
+                            first_call,send_buffer, recv_buffer,send_request, recv_request
+
+use gr_bh_gpu_Interface, ONLY: gr_bh_gpu_Finalize_Buffer
+implicit none
+#include "Flash_mpi.h"
+#ifdef USE_VT
+include "VT.inc"
+#endif
+
+
+
+
+integer :: m,ierr,i,log_lun
+logical ::localWrite
+real ::  time_essential_mem_AVG,time_essential_mem_min,time_essential_mem_max
+
+real ::  time_essential_HtoD_AVG,time_essential_HtoD_min,time_essential_HtoD_max
+real ::  time_essential_HtoD_GB_AVG,time_essential_HtoD_GB_min,time_essential_HtoD_GB_max
+
+real ::  time_essential_DtoH_AVG,time_essential_DtoH_min,time_essential_DtoH_max
+real ::  time_essential_DtoH_GB_AVG,time_essential_DtoH_GB_min,time_essential_DtoH_GB_max
+        
+real ::  time_essential_build_AVG,time_essential_build_min, time_essential_build_max
+real ::  time_essential_sum_AVG, time_essential_sum_min,  time_essential_sum_max
+real ::  time_essential_find_AVG, time_essential_find_min,  time_essential_find_max       
+real ::  time_essential_init_AVG, time_essential_init_min,time_essential_init_max
+        
+
+real ::  time_calc_mem_AVG,time_calc_mem_min,time_calc_mem_max
+real ::  time_calc_HtoD_AVG,time_calc_HtoD_min,time_calc_HtoD_max
+real ::  time_calc_HtoD_GB_AVG,time_calc_HtoD_GB_min,time_calc_HtoD_GB_max
+real ::  time_calc_DtoH_AVG,time_calc_DtoH_min,time_calc_DtoH_max
+real ::  time_calc_DtoH_GB_AVG,time_calc_DtoH_GB_min,time_calc_DtoH_GB_max
+        
+
+real ::  time_calc_build_AVG,time_calc_build_min,time_calc_build_max
+real ::  time_calc_calc_AVG, time_calc_calc_min, time_calc_calc_max
+real ::  time_calc_sum_AVG, time_calc_sum_min, time_calc_sum_max
+real ::  time_calc_init_AVG, time_calc_init_min,time_calc_init_max
+ character(len=1024) :: filename
+
+!write(*,*)"calling FREE"
+
+#ifdef USE_VT
+call VTTRACEON()
+#endif
+
+
+
+if(allocated(gr_bh_gpu_EwaldField))deallocate(gr_bh_gpu_EwaldField)
+
+
+
+
+
+
+if(gr_meshNumProcs .gt. 1) then
+  call gr_bh_gpu_Finalize_Buffer(send_buffer, recv_buffer,send_request, recv_request)
+  call MPI_Comm_free(gpu_comm,ierr)
+endif
+
+
+
+essentials_memory_timings(2) = (essentials_memory_timings(2) /1073741824.0 ) /  (essentials_memory_timings(1)/1000)
+potential_memory_timings(2)  = (potential_memory_timings(2)  /1073741824.0 ) / ( potential_memory_timings(1)/1000) 
+
+essentials_memory_timings(4) = (essentials_memory_timings(4) /1073741824.0 ) / ( essentials_memory_timings(3)/1000)
+potential_memory_timings(4)  = (potential_memory_timings(4) /1073741824.0 ) / ( potential_memory_timings(3)/1000)
+!====================================================================================================================!
+!=====================================ESSENTIALS EVOLUTION===========================================================!
+!====================================================================================================================!
+
+ call MPI_Reduce(essentials_memory_timings(0), time_essential_mem_AVG, 1, FLASH_REAL, MPI_SUM, 0,gr_meshComm,ierr)
+ call MPI_Reduce(essentials_memory_timings(0), time_essential_mem_min, 1, FLASH_REAL, MPI_MIN, 0,gr_meshComm,ierr)
+ call MPI_Reduce(essentials_memory_timings(0), time_essential_mem_max, 1, FLASH_REAL, MPI_MAX, 0,gr_meshComm,ierr)
+ time_essential_mem_AVG= time_essential_mem_AVG/gr_meshNumProcs
+
+ call MPI_Reduce(essentials_memory_timings(1), time_essential_HtoD_AVG, 1, FLASH_REAL, MPI_SUM, 0,gr_meshComm,ierr)
+ call MPI_Reduce(essentials_memory_timings(1), time_essential_HtoD_min, 1, FLASH_REAL, MPI_MIN, 0,gr_meshComm,ierr)
+ call MPI_Reduce(essentials_memory_timings(1), time_essential_HtoD_max, 1, FLASH_REAL, MPI_MAX, 0,gr_meshComm,ierr)
+ time_essential_HtoD_AVG= time_essential_HtoD_AVG/gr_meshNumProcs
+
+ call MPI_Reduce(essentials_memory_timings(2), time_essential_HtoD_GB_AVG, 1, FLASH_REAL, MPI_SUM, 0,gr_meshComm,ierr)
+ call MPI_Reduce(essentials_memory_timings(2), time_essential_HtoD_GB_min, 1, FLASH_REAL, MPI_MIN, 0,gr_meshComm,ierr)
+ call MPI_Reduce(essentials_memory_timings(2), time_essential_HtoD_GB_max, 1, FLASH_REAL, MPI_MAX, 0,gr_meshComm,ierr)
+ time_essential_HtoD_GB_AVG= time_essential_HtoD_GB_AVG/gr_meshNumProcs
+
+
+ call MPI_Reduce(essentials_memory_timings(3), time_essential_DtoH_AVG, 1, FLASH_REAL, MPI_SUM, 0,gr_meshComm,ierr)
+ call MPI_Reduce(essentials_memory_timings(3), time_essential_DtoH_min, 1, FLASH_REAL, MPI_MIN, 0,gr_meshComm,ierr)
+ call MPI_Reduce(essentials_memory_timings(3), time_essential_DtoH_max, 1, FLASH_REAL, MPI_MAX, 0,gr_meshComm,ierr)
+ time_essential_DtoH_AVG= time_essential_DtoH_AVG/gr_meshNumProcs
+
+
+ call MPI_Reduce(essentials_memory_timings(4), time_essential_DtoH_GB_AVG, 1, FLASH_REAL, MPI_SUM, 0,gr_meshComm,ierr)
+ call MPI_Reduce(essentials_memory_timings(4), time_essential_DtoH_GB_min, 1, FLASH_REAL, MPI_MIN, 0,gr_meshComm,ierr)
+ call MPI_Reduce(essentials_memory_timings(4), time_essential_DtoH_GB_max, 1, FLASH_REAL, MPI_MAX, 0,gr_meshComm,ierr)
+ time_essential_DtoH_GB_AVG= time_essential_DtoH_GB_AVG/gr_meshNumProcs
+
+
+ call MPI_Reduce(essentials_kernel_timings(0), time_essential_init_AVG, 1, FLASH_REAL, MPI_SUM, 0,gr_meshComm,ierr)
+ call MPI_Reduce(essentials_kernel_timings(0), time_essential_init_min, 1, FLASH_REAL, MPI_MIN, 0,gr_meshComm,ierr)
+ call MPI_Reduce(essentials_kernel_timings(0), time_essential_init_max, 1, FLASH_REAL, MPI_MAX, 0,gr_meshComm,ierr)
+ time_essential_init_AVG=time_essential_init_AVG/gr_meshNumProcs
+
+
+ call MPI_Reduce(essentials_kernel_timings(1), time_essential_build_AVG, 1, FLASH_REAL, MPI_SUM, 0,gr_meshComm,ierr)
+ call MPI_Reduce(essentials_kernel_timings(1), time_essential_build_min, 1, FLASH_REAL, MPI_MIN, 0,gr_meshComm,ierr)
+ call MPI_Reduce(essentials_kernel_timings(1), time_essential_build_max, 1, FLASH_REAL, MPI_MAX, 0,gr_meshComm,ierr)
+ time_essential_build_AVG=time_essential_build_AVG/gr_meshNumProcs
+
+
+ call MPI_Reduce(essentials_kernel_timings(2), time_essential_sum_AVG, 1, FLASH_REAL, MPI_SUM, 0,gr_meshComm,ierr)
+ call MPI_Reduce(essentials_kernel_timings(2), time_essential_sum_min, 1, FLASH_REAL, MPI_MIN, 0,gr_meshComm,ierr)
+ call MPI_Reduce(essentials_kernel_timings(2), time_essential_sum_max, 1, FLASH_REAL, MPI_MAX, 0,gr_meshComm,ierr)
+ time_essential_sum_AVG=time_essential_sum_AVG/gr_meshNumProcs
+
+
+
+ call MPI_Reduce(essentials_kernel_timings(3), time_essential_find_AVG, 1, FLASH_REAL, MPI_SUM, 0,gr_meshComm,ierr)
+ call MPI_Reduce(essentials_kernel_timings(3), time_essential_find_min, 1, FLASH_REAL, MPI_MIN, 0,gr_meshComm,ierr)
+ call MPI_Reduce(essentials_kernel_timings(3), time_essential_find_max, 1, FLASH_REAL, MPI_MAX, 0,gr_meshComm,ierr)
+ time_essential_find_AVG=time_essential_find_AVG/gr_meshNumProcs
+
+
+
+
+!====================================================================================================================!
+!===============================================FORCE CALC Evolution=================================================!
+!====================================================================================================================!
+
+ call MPI_Reduce(potential_memory_timings(0), time_calc_mem_AVG, 1, FLASH_REAL, MPI_SUM, 0,gr_meshComm,ierr)
+ call MPI_Reduce(potential_memory_timings(0), time_calc_mem_min, 1, FLASH_REAL, MPI_MIN, 0,gr_meshComm,ierr)
+ call MPI_Reduce(potential_memory_timings(0), time_calc_mem_max, 1, FLASH_REAL, MPI_MAX, 0,gr_meshComm,ierr)
+ time_calc_mem_AVG=time_calc_mem_AVG/gr_meshNumProcs
+
+ call MPI_Reduce(potential_memory_timings(1), time_calc_HtoD_AVG, 1, FLASH_REAL, MPI_SUM, 0,gr_meshComm,ierr)
+ call MPI_Reduce(potential_memory_timings(1), time_calc_HtoD_min, 1, FLASH_REAL, MPI_MIN, 0,gr_meshComm,ierr)
+ call MPI_Reduce(potential_memory_timings(1), time_calc_HtoD_max, 1, FLASH_REAL, MPI_MAX, 0,gr_meshComm,ierr)
+ time_calc_HtoD_AVG=time_calc_HtoD_AVG/gr_meshNumProcs
+
+
+ call MPI_Reduce(potential_memory_timings(2), time_calc_HtoD_GB_AVG, 1, FLASH_REAL, MPI_SUM, 0,gr_meshComm,ierr)
+ call MPI_Reduce(potential_memory_timings(2), time_calc_HtoD_GB_min, 1, FLASH_REAL, MPI_MIN, 0,gr_meshComm,ierr)
+ call MPI_Reduce(potential_memory_timings(2), time_calc_HtoD_GB_max, 1, FLASH_REAL, MPI_MAX, 0,gr_meshComm,ierr)
+ time_calc_HtoD_GB_AVG=time_calc_HtoD_GB_AVG/gr_meshNumProcs
+
+
+
+ call MPI_Reduce(potential_memory_timings(3), time_calc_DtoH_AVG, 1, FLASH_REAL, MPI_SUM, 0,gr_meshComm,ierr)
+ call MPI_Reduce(potential_memory_timings(3), time_calc_DtoH_min, 1, FLASH_REAL, MPI_MIN, 0,gr_meshComm,ierr)
+ call MPI_Reduce(potential_memory_timings(3), time_calc_DtoH_max, 1, FLASH_REAL, MPI_MAX, 0,gr_meshComm,ierr)
+ time_calc_DtoH_AVG=time_calc_DtoH_AVG/gr_meshNumProcs
+
+
+ call MPI_Reduce(potential_memory_timings(4), time_calc_DtoH_GB_AVG, 1, FLASH_REAL, MPI_SUM, 0,gr_meshComm,ierr)
+ call MPI_Reduce(potential_memory_timings(4), time_calc_DtoH_GB_min, 1, FLASH_REAL, MPI_MIN, 0,gr_meshComm,ierr)
+ call MPI_Reduce(potential_memory_timings(4), time_calc_DtoH_GB_max, 1, FLASH_REAL, MPI_MAX, 0,gr_meshComm,ierr)
+ time_calc_DtoH_GB_AVG=time_calc_DtoH_GB_AVG/gr_meshNumProcs
+
+
+
+ call MPI_Reduce(potential_kernel_timings(0), time_calc_init_AVG, 1, FLASH_REAL, MPI_SUM, 0,gr_meshComm,ierr)
+ call MPI_Reduce(potential_kernel_timings(0), time_calc_init_min, 1, FLASH_REAL, MPI_MIN, 0,gr_meshComm,ierr)
+ call MPI_Reduce(potential_kernel_timings(0), time_calc_init_max, 1, FLASH_REAL, MPI_MAX, 0,gr_meshComm,ierr)
+ time_calc_init_AVG=time_calc_init_AVG/gr_meshNumProcs
+
+ call MPI_Reduce(potential_kernel_timings(1), time_calc_build_AVG, 1, FLASH_REAL, MPI_SUM, 0,gr_meshComm,ierr)
+ call MPI_Reduce(potential_kernel_timings(1), time_calc_build_min, 1, FLASH_REAL, MPI_MIN, 0,gr_meshComm,ierr)
+ call MPI_Reduce(potential_kernel_timings(1), time_calc_build_max, 1, FLASH_REAL, MPI_MAX, 0,gr_meshComm,ierr)
+ time_calc_build_AVG=time_calc_build_AVG/gr_meshNumProcs
+
+
+ call MPI_Reduce(potential_kernel_timings(2), time_calc_sum_AVG, 1, FLASH_REAL, MPI_SUM, 0,gr_meshComm,ierr)
+ call MPI_Reduce(potential_kernel_timings(2), time_calc_sum_min, 1, FLASH_REAL, MPI_MIN, 0,gr_meshComm,ierr)
+ call MPI_Reduce(potential_kernel_timings(2), time_calc_sum_max, 1, FLASH_REAL, MPI_MAX, 0,gr_meshComm,ierr)
+ time_calc_sum_AVG=time_calc_sum_AVG/gr_meshNumProcs
+
+ call MPI_Reduce(potential_kernel_timings(3), time_calc_calc_AVG, 1, FLASH_REAL, MPI_SUM, 0,gr_meshComm,ierr)
+ call MPI_Reduce(potential_kernel_timings(3), time_calc_calc_min, 1, FLASH_REAL, MPI_MIN, 0,gr_meshComm,ierr)
+ call MPI_Reduce(potential_kernel_timings(3), time_calc_calc_max, 1, FLASH_REAL, MPI_MAX, 0,gr_meshComm,ierr)
+ time_calc_calc_AVG=time_calc_calc_AVG/gr_meshNumProcs
+
+
+
+if (gr_meshMe .eq. 0) then
+ call Logfile_stampMessage("==============================================================================",.true.)
+ call Logfile_stampMessage("=================================GPU TIMERS===================================",.true.)
+ call Logfile_stampMessage("==============================================================================",.true.)
+ call Logfile_stampMessage("",.true.)
+ call Logfile_stampMessage("",.true.)
+ call Logfile_stampMessage("--------------------------- EVOLUTION Calc. Potential  -----------------------",.true.)
+ call Logfile_stampMessage("------------------------------------------------------------------------------",.true.)
+ write (filename,'(1X,A20,1X,A15,A15,1X,A15,1X)')"accounting unit","min/proc (s)" ,"max/proc (s)", "avg/proc (s)"
+ call Logfile_stampMessage(filename,.true.)
+ call Logfile_stampMessage("------------------------------------------------------------------------------",.true.)
+ call Logfile_stampMessage("",.true.)
+ write (filename,'(1X,A20,F13.4,F13.4,F13.4)')  "Init", time_calc_init_min/1000,&
+							time_calc_init_max/1000,&
+							(time_calc_init_AVG)/1000
+ call Logfile_stampMessage(filename,.true.)
+ write (filename,'(1X,A20,F13.4,F13.4,F13.4)')  "Treebuild", time_calc_build_min/1000,&
+							     time_calc_build_max/1000,&
+							     (time_calc_build_AVG)/1000
+ call Logfile_stampMessage(filename,.true.)
+ write (filename,'(1X,A20,F13.4,F13.4,F13.4)')  "Treewalk", time_calc_calc_min/1000,&
+							    time_calc_calc_max/1000,&
+                                                            (time_calc_calc_AVG)/1000
+ call Logfile_stampMessage(filename,.true.)
+ write (filename,'(1X,A20,F13.4,F13.4,F13.4)')  "Treesumm", time_calc_sum_min/1000,&
+								time_calc_sum_max/1000,&
+								(time_calc_sum_AVG)/1000
+ call Logfile_stampMessage(filename,.true.)
+ write (filename,'(1X,A20,F13.4,F13.4,F13.4)')  "Memory Alloc/Free", time_calc_mem_min/1000,&
+									time_calc_mem_max/1000,&
+									(time_calc_mem_AVG)/1000
+ call Logfile_stampMessage(filename,.true.)
+ write (filename,'(1X,A20,F13.4,F13.4,F13.4)')  "Memory HtoD", time_calc_HtoD_min/1000,&
+								time_calc_HtoD_max/1000,&
+								(time_calc_HtoD_AVG)/1000
+ call Logfile_stampMessage(filename,.true.)
+ write (filename,'(1X,A20,F13.4,F13.4,F13.4)')  "Memory DtoH", time_calc_DtoH_min/1000,&
+								time_calc_DtoH_max/1000,&
+								(time_calc_DtoH_AVG)/1000
+ call Logfile_stampMessage(filename,.true.)
+ call Logfile_stampMessage("------------------------------------------------------------------------------",.true.)
+ write (filename,'(1X,A20,1X,A15,A15,1X,A15,1X)')"accounting unit","min GB/s" ,"max GB/s", "avg GB/s"
+ call Logfile_stampMessage(filename,.true.)
+ call Logfile_stampMessage("------------------------------------------------------------------------------",.true.)
+ write (filename,'(1X,A20,F13.4,F13.4,F13.4)')  "Memory HtoD", time_calc_HtoD_GB_min ,&
+								time_calc_HtoD_GB_max,time_calc_HtoD_GB_AVG
+ call Logfile_stampMessage(filename,.true.) 
+ write (filename,'(1X,A20,F13.4,F13.4,F13.4)')  "Memory DtoH", time_calc_DtoH_GB_min,&
+								time_calc_DtoH_GB_max,time_calc_DtoH_GB_AVG
+ call Logfile_stampMessage(filename,.true.)
+ call Logfile_stampMessage("",.true.)
+ call Logfile_stampMessage("------------------------------------------------------------------------------",.true.)
+ call Logfile_stampMessage("",.true.)
+ call Logfile_stampMessage("",.true.)
+ call Logfile_stampMessage("",.true.)
+ call Logfile_stampMessage("--------------------------- EVOLUTION Calc. Essentials -----------------------",.true.)
+ call Logfile_stampMessage("------------------------------------------------------------------------------",.true.)
+ write (filename,'(1X,A20,1X,A15,A15,1X,A15,1X)')"accounting unit","min/proc (s)" ,"max/proc (s)", "avg/proc (s)"
+ call Logfile_stampMessage(filename,.true.)
+ call Logfile_stampMessage("------------------------------------------------------------------------------",.true.)
+  write (filename,'(1X,A20,F13.4,F13.4,F13.4)')  "Init", time_essential_init_min/1000,&
+							time_essential_init_max/1000,(time_essential_init_AVG)/1000
+ call Logfile_stampMessage(filename,.true.)
+  write (filename,'(1X,A20,F13.4,F13.4,F13.4)')  "Treebuild", time_essential_build_min/1000,&
+								time_essential_build_max/1000,&
+								(time_essential_build_AVG)/1000
+ call Logfile_stampMessage(filename,.true.)
+  write (filename,'(1X,A20,F13.4,F13.4,F13.4)')  "Treesum", time_essential_sum_min/1000,&
+							time_essential_sum_max/1000,&
+						       (time_essential_sum_AVG)/1000
+ call Logfile_stampMessage(filename,.true.)
+  write (filename,'(1X,A20,F13.4,F13.4,F13.4)')  "Treefind", time_essential_find_min/1000,&
+								time_essential_find_max/1000,&
+								(time_essential_find_AVG)/1000
+ call Logfile_stampMessage(filename,.true.)
+  write (filename,'(1X,A20,F13.4,F13.4,F13.4)')  "Memory Alloc/Free", time_essential_mem_min/1000,&
+									time_essential_mem_max/1000,&
+									(time_essential_mem_AVG)/1000
+ call Logfile_stampMessage(filename,.true.)
+  write (filename,'(1X,A20,F13.4,F13.4,F13.4)')  "Memory HtoD", time_essential_HtoD_min/1000,&
+								time_essential_HtoD_max/1000,&
+								(time_essential_HtoD_AVG)/1000
+ call Logfile_stampMessage(filename,.true.)
+  write (filename,'(1X,A20,F13.4,F13.4,F13.4)')  "Memory DtoH", time_essential_DtoH_min/1000,&
+								time_essential_DtoH_max/1000,&
+								(time_essential_DtoH_AVG)/1000
+ call Logfile_stampMessage(filename,.true.)
+ call Logfile_stampMessage("------------------------------------------------------------------------------",.true.)
+ write (filename,'(1X,A20,1X,A15,A15,1X,A15,1X)')"accounting unit","min GB/s" ,"max GB/s", "avg GB/s"
+ call Logfile_stampMessage(filename,.true.)
+ call Logfile_stampMessage("------------------------------------------------------------------------------",.true.)
+ write (filename,'(1X,A20,F13.4,F13.4,F13.4)')  "Memory HtoD", time_essential_HtoD_GB_min,&
+								time_essential_HtoD_GB_max,&
+								time_essential_HtoD_GB_AVG
+ call Logfile_stampMessage(filename,.true.)
+ write (filename,'(1X,A20,F13.4,F13.4,F13.4)')  "Memory DtoH", time_essential_DtoH_GB_min,&
+								time_essential_DtoH_GB_max,&
+								time_essential_DtoH_GB_AVG
+ call Logfile_stampMessage(filename,.true.)
+
+ call Logfile_stampMessage("",.true.)
+end if
+
+
+#ifdef USE_VT
+call VTTRACEOFF()
+call VTFINI(ierr)
+#endif
+
+
+
+return
+end subroutine gr_bh_gpu_Finalize
Index: source/Grid/GridSolvers/BHTree/GPU/KernelCode/Config
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/KernelCode/Config	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/KernelCode/Config	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1 @@
+REQUIRES hwaccel/GpuSupport/GpuSupportMain/CUDA/KernelCode
Index: source/Grid/GridSolvers/BHTree/GPU/KernelCode/Makefile
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/KernelCode/Makefile	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/KernelCode/Makefile	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,88 @@
+# Makefile for GPU-accelerated tree Poisson solver: objects from CUDA code
+#============================================================================
+#=================== BH-GPU Tree code makefile ==============================
+#============================================================================
+
+CUDA_KERNEL_OBJS += gr_bh_gpu_Kernel_Constants.o\
+	gr_bh_gpu_Init_Kernel.o\
+	gr_bh_gpu_MakeRootKernel.o\
+	gr_bh_gpu_Summarization_Kernel.o\
+	gr_bh_gpu_TreeBuilding_Kernel.o\
+	gr_bh_gpu_set_nodes.o
+
+CUDA_KERNEL_OBJS += gr_bh_gpu_Ewald_Extrapolate_Kernel.o\
+	gr_bh_gpu_Ewald_Real_Periodic_Kernel.o\
+	gr_bh_gpu_Ewald_Rec_Periodic_Kernel.o\
+	gr_bh_gpu_Periodic_InterpolateInField.o\
+	gr_bh_gpu_Ewald_Temp_Field_Kernel.o
+
+CUDA_KERNEL_OBJS += gr_bh_gpu_Potential_Calculate_Direkt_Kernel.o\
+	gr_bh_gpu_Potential_Calculate_ForeignTree_Kernel.o\
+	gr_bh_gpu_Potential_Calculate_Local_Kernel.o\
+	gr_bh_gpu_Essentials_Calculate_Nodes_Kernel.o
+
+#wrapper
+CUDA_KERNEL_OBJS += gr_bh_gpu_Essentials_Build_Init_Wrap.o\
+	gr_bh_gpu_Essentials_Calc_Essentials_Wrap.o\
+	gr_bh_gpu_Essentials_CleanUp_Device_Wrap.o\
+	gr_bh_gpu_Essentials_Send_Essentials_Wrap.o\
+	gr_bh_gpu_Essentials_Wait_And_Copy.o\
+	gr_bh_gpu_Essentials_Receive_Message.o\
+	gr_bh_gpu_Init_Allocate_MPI_Buffer.o\
+	gr_bh_gpu_Clear_Buffers.o\
+	gr_bh_gpu_Finalize_Buffer.o\
+	gr_bh_gpu_Potential_Simple.o\
+	gr_bh_gpu_Gather_Allocate_CellData.o\
+	gr_bh_gpu_GetGridData_SetValue.o\
+	gr_bh_gpu_CleanUp_CellData.o\
+	gr_bh_gpu_Save_Grid_Pot.o\
+	gr_bh_gpu_Potential_Init.o\
+	gr_bh_gpu_Potential_Local.o\
+	gr_bh_gpu_Potential_All.o\
+	gr_bh_gpu_Potential_Subset.o\
+	gr_bh_gpu_Potential_Copy.o\
+	gr_bh_gpu_Potential_Clean.o\
+	gr_bh_gpu_Essentials_Reallocate_CellData.o
+#EWALD
+CUDA_KERNEL_OBJS += gr_bh_gpu_Ewald_MakeField.o\
+	gr_bh_gpu_Ewald_Init.o
+
+# List some explicit dependencies here, since the dependency analysis of FLASH may
+# not be clever enough to detect dependencies on indirectly incuded header files.
+gr_bh_gpu_CleanUp_CellData.o \
+	gr_bh_gpu_Clear_Buffers.o \
+	gr_bh_gpu_Essentials_Build_Init_Wrap.o \
+	gr_bh_gpu_Essentials_Calc_Essentials_Wrap.o \
+	gr_bh_gpu_Essentials_Calculate_Nodes_Kernel.o \
+	gr_bh_gpu_Essentials_Reallocate_CellData.o \
+	gr_bh_gpu_Essentials_Receive_Message.o \
+	gr_bh_gpu_Essentials_Send_Essentials_Wrap.o \
+	gr_bh_gpu_Essentials_Wait_And_Copy.o \
+	gr_bh_gpu_Ewald_Extrapolate_Kernel.o \
+	gr_bh_gpu_Ewald_Init.o \
+	gr_bh_gpu_Ewald_MakeField.o \
+	gr_bh_gpu_Ewald_Real_Periodic_Kernel.o \
+	gr_bh_gpu_Ewald_Rec_Periodic_Kernel.o \
+	gr_bh_gpu_Ewald_Temp_Field_Kernel.o \
+	gr_bh_gpu_Finalize_Buffer.o \
+	gr_bh_gpu_Gather_Allocate_CellData.o \
+	gr_bh_gpu_GetGridData_SetValue.o \
+	gr_bh_gpu_Init_Allocate_MPI_Buffer.o \
+	gr_bh_gpu_Init_Kernel.o \
+	gr_bh_gpu_Kernel_Constants.o \
+	gr_bh_gpu_MakeRootKernel.o \
+	gr_bh_gpu_Periodic_InterpolateInField.o \
+	gr_bh_gpu_Potential_All.o \
+	gr_bh_gpu_Potential_Calculate_Direkt_Kernel.o \
+	gr_bh_gpu_Potential_Calculate_ForeignTree_Kernel.o \
+	gr_bh_gpu_Potential_Calculate_Local_Kernel.o \
+	gr_bh_gpu_Potential_Clean.o \
+	gr_bh_gpu_Potential_Copy.o \
+	gr_bh_gpu_Potential_Init.o \
+	gr_bh_gpu_Potential_Local.o \
+	gr_bh_gpu_Potential_Simple.o \
+	gr_bh_gpu_Potential_Subset.o \
+	gr_bh_gpu_Save_Grid_Pot.o \
+	gr_bh_gpu_set_nodes.o \
+	gr_bh_gpu_Summarization_Kernel.o \
+	gr_bh_gpu_TreeBuilding_Kernel.o : gr_bh_gpu_Kernel_Constants.cuh CudaError.cuh Flash.h constants.h
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_MakeRootKernel.cuh
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_MakeRootKernel.cuh	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_MakeRootKernel.cuh	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,9 @@
+#ifndef __MakeRootKernel_CUH__
+#define __MakeRootKernel_CUH__
+
+
+#include "gr_bh_gpu_Kernel_Constants.cuh"
+
+__global__ void MakeRootKernel(double *roots_minx,double *roots_maxx,double *roots_miny,double *roots_maxy,double *roots_minz,double *roots_maxz);
+
+#endif
Index: source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Temp_Field_Kernel.cuh
===================================================================
--- source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Temp_Field_Kernel.cuh	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/Grid/GridSolvers/BHTree/GPU/gr_bh_gpu_Ewald_Temp_Field_Kernel.cuh	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,11 @@
+#ifndef __EWALD_GEN_TEMP_CUH__
+#define __EWALD_GEN_TEMP_CUH__
+
+
+#include "gr_bh_gpu_Kernel_Constants.cuh"       
+
+__global__ void gen_temp_field(double *t_gridx,double *t_gridy,double *t_gridz);
+
+#endif
+
+
Index: source/hwaccel/GpuSupport/Config
===================================================================
--- source/hwaccel/GpuSupport/Config	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/hwaccel/GpuSupport/Config	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1 @@
+DEFAULT GpuSupportMain
Index: source/hwaccel/GpuSupport/GpuSupport_interface.F90
===================================================================
--- source/hwaccel/GpuSupport/GpuSupport_interface.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/hwaccel/GpuSupport/GpuSupport_interface.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,30 @@
+!!****h* source/hwaccel/GpuSupport/GpuSupport_interface
+!!
+!! NAME
+!!
+!!  GpuSupport_interface
+!!
+!!
+!! SYNOPSIS
+!!
+!!  use GpuSupport_interface
+!!
+!!
+!! DESCRIPTION
+!!
+!! This is the header file for the GpuSupport unit that defines its
+!! public interfaces.
+!!***
+Module GpuSupport_interface
+
+  use GpuSupport_cudaFortInterface, ONLY: GpuSupport_getDeviceProperties, &
+                                          GpuSupport_setDeviceMPIgroup
+
+  implicit none
+
+  interface
+     subroutine GpuSupport_init()
+     end subroutine GpuSupport_init
+  end interface
+
+end Module GpuSupport_interface
Index: source/hwaccel/GpuSupport/GpuSupport_init.F90
===================================================================
--- source/hwaccel/GpuSupport/GpuSupport_init.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/hwaccel/GpuSupport/GpuSupport_init.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,26 @@
+!!****f* source/hwaccel/GpuSupport/GpuSupport_init
+!!
+!! NAME
+!!
+!!  GpuSupport_init
+!!
+!! SYNOPSIS
+!!
+!!  call GpuSupport_init() 
+!!
+!! DESCRIPTION
+!!  
+!!  Initializes data structures of the GpuSupport unit (if necessary).
+!!  Currently does not do anything.
+!!
+!! ARGUMENTS
+!!
+!!  none
+!!
+!!***
+
+subroutine GpuSupport_init()
+
+   implicit none
+
+end subroutine GpuSupport_init
Index: source/hwaccel/GpuSupport/GpuSupportMain/Config
===================================================================
--- source/hwaccel/GpuSupport/GpuSupportMain/Config	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/hwaccel/GpuSupport/GpuSupportMain/Config	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1 @@
+DEFAULT CUDA
Index: source/hwaccel/GpuSupport/GpuSupportMain/CUDA/CudaError.cuh
===================================================================
--- source/hwaccel/GpuSupport/GpuSupportMain/CUDA/CudaError.cuh	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/hwaccel/GpuSupport/GpuSupportMain/CUDA/CudaError.cuh	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,42 @@
+#ifndef __CUDA_ERROR__
+#define __CUDA_ERROR__
+#include <stdio.h>
+#include <cuda.h>
+#include <cuda_runtime.h>
+
+#define CuErrChk(ans) { gpuAssert((ans), __FILE__, __LINE__); }
+
+static void gpuAssert(cudaError_t code, const char *file, int line, bool abort=true)
+{
+   if (code != cudaSuccess) 
+   {
+      fprintf(stderr,"GPUassert: %s %s %d\n", cudaGetErrorString(code), file, line);
+      if (abort) exit(code);
+   }
+}
+
+
+
+
+
+
+
+
+static  void CudaTest(char *msg)
+{
+#ifdef GPU_DEBUG
+  cudaError_t e;
+  
+  cudaDeviceSynchronize();
+  if (cudaSuccess != (e = cudaGetLastError())) {
+     printf("======= CUDA ERROR ======\n");
+    fprintf(stderr, "======= CUDA ERROR ======\n");
+    fprintf(stderr, "%s: %d\n", msg, e);
+    fprintf(stderr, "%s\n", cudaGetErrorString(e));
+    fprintf(stderr, "======= ========== ======\n");
+    exit(-1);
+  }
+#endif
+}
+
+#endif
Index: source/hwaccel/GpuSupport/GpuSupportMain/CUDA/Config
===================================================================
--- source/hwaccel/GpuSupport/GpuSupportMain/CUDA/Config	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/hwaccel/GpuSupport/GpuSupportMain/CUDA/Config	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,3 @@
+LIBRARY CUDA
+
+DEFAULT KernelCode
Index: source/hwaccel/GpuSupport/GpuSupportMain/CUDA/KernelCode/CudaGeneralFunctions.cuh
===================================================================
--- source/hwaccel/GpuSupport/GpuSupportMain/CUDA/KernelCode/CudaGeneralFunctions.cuh	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/hwaccel/GpuSupport/GpuSupportMain/CUDA/KernelCode/CudaGeneralFunctions.cuh	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,165 @@
+#ifndef _CUDA_GENERAL_FLASH_
+#define _CUDA_GENERAL_FLASH_
+#include <stdio.h>
+#include "mpi.h"
+#include <cuda.h>
+#include <cuda_runtime.h>
+#include <thrust/device_ptr.h>
+#include <thrust/fill.h>
+#include "CudaError.cuh"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+
+/*******************************************************************/
+/*****************Gpu DEVICE PROPERTIES******************************/
+/*******************************************************************/
+
+void get_device_properties_(int *device_ID,int *multiProcessorCount);
+  
+void get_device_count_(int *device_count, int *err);
+
+void set_device_(int *device_ID, int *err);
+
+void device_reset_(int *err);
+
+/*******************************************************************/
+/*****************Errror handling fortran***************************/
+/*******************************************************************/
+
+void print_last_error_string_(char *msg,int *err);
+
+
+/*******************************************************************/
+/*****************Stream handling******************************/
+/*******************************************************************/
+
+
+void set_streams_(cudaStream_t **streams,int *num, int *err);
+
+void del_streams_(cudaStream_t **streams,int *num, int *err);
+
+void wait_stream_(cudaStream_t **streams,int *num, int *err);
+
+
+/*******************************************************************/
+/*****************ALLOCATION OF PINNED MEMORY***********************/
+/*******************************************************************/
+
+
+void cuda_pinned_allocate_d_(double **device_ptr,int *elements, int *err);
+
+void cuda_pinned_free_d_(double **device_ptr,int *err);
+
+
+void cuda_pinned_allocate_f_(float **device_ptr,int *elements, int *err);
+
+void cuda_pinned_free_f_(float **device_ptr,int *err);
+
+
+void cuda_pinned_allocate_i_(int **device_ptr,int *elements, int *err);
+
+void cuda_pinned_free_i_(int **device_ptr,int *err);
+/*******************************************************************/
+/*****************COPY TO/FROM  PINNED MEMORY***********************/
+/*******************************************************************/
+void cuda_pinned_copy_to_d_(double **pinned_ptr,double *paged_ptr,int *elements,int *offset_pinned, int *offset_paged);
+void cuda_pinned_copy_to_f_(float **pinned_ptr,float *paged_ptr,int *elements,int *offset_pinned, int *offset_paged);
+void cuda_pinned_copy_to_i_(int **pinned_ptr,int *paged_ptr,int *elements,int *offset_pinned, int *offset_paged);
+
+
+void cuda_pinned_copy_from_d_(double **pinned_ptr,double *paged_ptr,int *elements,int *offset_pinned, int *offset_paged);
+void cuda_pinned_copy_from_f_(float **pinned_ptr,float *paged_ptr,int *elements,int *offset_pinned, int *offset_paged);
+void cuda_pinned_copy_from_i_(int **pinned_ptr,int *paged_ptr,int *elements,int *offset_pinned, int *offset_paged);
+
+/*******************************************************************/
+/***************** cPTR ARITHMETICS*********************************/
+/*******************************************************************/
+void *get_cptr_d_(double **device_ptr,int *elements);
+void *get_cptr_f_(float **device_ptr,int *elements);
+void *get_cptr_i_(int **device_ptr,int *elements);
+
+
+/*******************************************************************/
+/*****************ASYN COPY OF MEMORY******************************/
+/*******************************************************************/
+
+
+void devicecopyasync_d_(double **device_pointer,double **fortval,
+		        int *size ,int *dev_offset, 
+			int *host_offset,int *flag,
+		        cudaStream_t **stream,int *num, int*err);
+
+
+void devicecopyasync_f_(float **device_pointer,float *fortval,
+		        int *size,int *dev_offset ,
+			int *host_offset, int *flag,
+		        cudaStream_t **stream,int *num, int*err);
+ 
+
+void devicecopyasync_i_(int **device_pointer,int *fortval,
+		        int *size,int *dev_offset ,
+			int *host_offset, int *flag,
+		        cudaStream_t **stream,int *num, int*err);
+
+/*******************************************************************/
+/*****************ALLOCATION OF DEVICE MEMORY******************************/
+/*******************************************************************/
+
+void devicemalloc_d_(double **device_pointer,int *size, int *err);
+
+void devicemalloc_f_(float **device_pointer,int *size, int *err);
+            
+void devicemalloc_i_(int **device_pointer,int *size, int*err);
+  
+
+void devicefree_d_(double **device_pointer, int *err);
+              
+void devicefree_f_(float **device_pointer, int *err);
+             
+void devicefree_i_(int **device_pointer, int*err);
+            
+/*******************************************************************/
+/*********************COPY OPERATIONS*******************************/
+/*******************************************************************/
+
+void devicecopy_d_(double **device_pointer,double *fortval,int *size ,int *offset,int *flag, int*err);
+           
+void devicecopy_f_(float **device_pointer,float *fortval,int *size,int *offset ,int *flag, int*err);
+
+void devicecopy_i_(int **device_pointer,int *fortval,int *size ,int *offset,int *flag, int*err);
+
+
+
+
+/*******************************************************************/
+/*********************FILL OPERATIONS*******************************/
+/*******************************************************************/
+/*******************************************************************/
+
+void devicefill_i_(int **device_pointer,int *size ,int *val);
+     
+void devicefill_f_(float **device_pointer,int *size ,float *val);
+
+void devicefill_d_(double **device_pointer,int *size ,double *val);
+    
+
+/*******************************************************************/
+/*****************HELPER OPERATIONS******************************/
+/*******************************************************************/
+void get_smallest_d_(double *tiny);
+
+
+
+
+
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
Index: source/hwaccel/GpuSupport/GpuSupportMain/CUDA/KernelCode/CudaGeneralFunctions.cu
===================================================================
--- source/hwaccel/GpuSupport/GpuSupportMain/CUDA/KernelCode/CudaGeneralFunctions.cu	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/hwaccel/GpuSupport/GpuSupportMain/CUDA/KernelCode/CudaGeneralFunctions.cu	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,468 @@
+
+#include "CudaGeneralFunctions.cuh"
+
+/*******************************************************************/
+/*******************************************************************/
+/*********************GPU BASIC ACCESSING***************************/
+/*******************************************************************/
+/*       THIS FILE IS INTENDED TO BE USED TOGETHER WITH            */ 
+/*       THE FORTRAN INTERFACE                                     */
+/*******************************************************************/
+
+
+
+
+/*******************************************************************/
+/*****************Gpu DEVICE PROPERTIES******************************/
+/*******************************************************************/
+
+
+void get_device_properties_(int *device_ID,int *multiProcessorCount)
+{
+  
+
+  
+   
+   int devid, len;
+   char name[MPI_MAX_PROCESSOR_NAME]; 
+   MPI_Get_processor_name(name, &len);
+   devid = *device_ID;
+
+  if(*device_ID == -1)
+	  devid =0;
+
+   cudaDeviceProp deviceProp;
+   CuErrChk(cudaGetDeviceProperties(&deviceProp, devid));
+       
+  
+   
+
+   if ((deviceProp.major == 9999) && (deviceProp.minor == 9999)) {
+     fprintf(stderr, "There is no CUDA capable device\n");
+     exit(-1);
+   }
+   
+   if (deviceProp.major < 2) {
+     fprintf(stderr, "Need at least compute capability 2.0\n");
+     exit(-1);
+   }
+ 
+   if (deviceProp.warpSize != 32) {
+     fprintf(stderr, "Warp size must be %d\n", deviceProp.warpSize);
+     exit(-1);
+   }
+
+  *multiProcessorCount = deviceProp.multiProcessorCount; 
+   
+#ifdef GPU_DEBUG
+    printf("  Host name : %s\n", name);
+    printf("  Device id : %d\n", devid);
+    printf("  Device name: %s\n", deviceProp.name);
+    printf("  Memory Clock Rate (KHz): %d\n",deviceProp.memoryClockRate);
+    printf("  Memory Bus Width (bits): %d\n",deviceProp.memoryBusWidth);
+    printf("  Peak Memory Bandwidth (GB/s): %f\n\n",2.0*deviceProp.memoryClockRate*(deviceProp.memoryBusWidth/8)/1.0e6);
+    printf("  Compute capability (major.minor): %d.%d\n",deviceProp.major,deviceProp.minor);
+    printf("  Total Global Memory (GB): %10.2f\n",((((float)deviceProp.totalGlobalMem)/1024/1024)/1024));
+#endif  
+  
+   
+
+}
+
+void get_device_count_(int *device_count,int *err)
+     {
+       *err = cudaGetDeviceCount(device_count);
+     }
+
+void set_device_(int *device_ID, int *err)
+{
+	     *err = cudaSetDevice(*device_ID);
+	     
+}
+
+void device_reset_(int *err)
+{
+	 *err =cudaDeviceReset();
+	     
+}
+
+/*******************************************************************/
+/********************ERROR TEST*************************************/
+/*******************************************************************/
+
+
+void print_last_error_string_(char *msg,int *err)
+{
+     CudaTest(msg);
+}
+
+/*******************************************************************/
+/************************* STREAMS **********************************/
+/*******************************************************************/
+
+void set_streams_(cudaStream_t **streams,int *num, int *err)
+{
+	     
+              *streams = (cudaStream_t *)malloc(sizeof(cudaStream_t)* *num);  
+ 	      for (int i = 0; i < *num; i++)
+                  CuErrChk(cudaStreamCreate(&(*streams)[i]));
+    }
+
+void del_streams_(cudaStream_t **streams,int *num, int *err)
+{
+ 	      for (int i = 0; i < *num; i++)
+                  CuErrChk(cudaStreamDestroy((*streams)[i]));
+
+	      free(*streams);
+            }
+
+void wait_stream_(cudaStream_t **streams,int *num, int *err)
+{
+ 	        CuErrChk(cudaStreamSynchronize((*streams)[*num]));
+    }
+
+
+/*******************************************************************/
+/*****************ALLOCATION OF PINNED MEMORY***********************/
+/*******************************************************************/
+
+void cuda_pinned_allocate_d_(double **device_ptr,int *elements, int *err)
+{
+ 	 CuErrChk( cudaMallocHost((void**) device_ptr, sizeof(double) * *elements)); 
+        }
+
+void cuda_pinned_free_d_(double **device_ptr,int *err)
+{
+ 	 CuErrChk(  cudaFreeHost(*device_ptr));
+        }
+
+
+void cuda_pinned_allocate_f_(float **device_ptr,int *elements, int *err)
+{
+ 	CuErrChk(  cudaMallocHost((void**) device_ptr, sizeof(float) * *elements)); 
+        }
+
+void cuda_pinned_free_f_(float **device_ptr,int *err)
+{
+ 	 CuErrChk(  cudaFreeHost(*device_ptr));
+        }
+
+
+void cuda_pinned_allocate_i_(int **device_ptr,int *elements, int *err)
+		{
+
+ 	 CuErrChk(  cudaMallocHost((void**) device_ptr, sizeof(int) * *elements)); 
+        }
+
+void cuda_pinned_free_i_(int **device_ptr,int *err)
+{
+ 	 CuErrChk(   cudaFreeHost(*device_ptr));
+   }
+
+/*******************************************************************/
+/*****************COPY TO/FROM  PINNED MEMORY***********************/
+/*******************************************************************/
+void cuda_pinned_copy_to_d_(double **pinned_ptr,double *paged_ptr,int *elements,int *offset_pinned, int *offset_paged)
+{
+   
+	std::copy( paged_ptr  + *offset_paged, 
+		   paged_ptr  + *offset_paged + *elements, 
+		  *pinned_ptr + *offset_pinned );
+
+}
+
+void cuda_pinned_copy_to_f_(float **pinned_ptr,float *paged_ptr,int *elements,int *offset_pinned, int *offset_paged)
+{
+ 	
+	std::copy( paged_ptr  + *offset_paged, 
+		   paged_ptr  + *offset_paged + *elements, 
+		  (*pinned_ptr) + *offset_pinned );
+        }
+
+void cuda_pinned_copy_to_i_(int **pinned_ptr,int *paged_ptr,int *elements,int *offset_pinned, int *offset_paged)
+{
+ 	     
+	std::copy( paged_ptr  + *offset_paged, 
+		   paged_ptr  + *offset_paged + *elements, 
+		  (*pinned_ptr) + *offset_pinned );
+        }
+
+
+
+
+void cuda_pinned_copy_from_d_(double **pinned_ptr,double *paged_ptr,int *elements,int *offset_pinned, int *offset_paged)
+{
+ 	 
+	std::copy((*pinned_ptr) + *offset_pinned, 
+		  (*pinned_ptr) + *offset_pinned + *elements, 
+		  paged_ptr +  *offset_paged );
+        }
+
+void cuda_pinned_copy_from_f_(float **pinned_ptr,float *paged_ptr,int *elements,int *offset_pinned, int *offset_paged)
+{
+ 	 
+	std::copy((*pinned_ptr) + *offset_pinned, 
+		  (*pinned_ptr) + *offset_pinned + *elements, 
+		  paged_ptr +  *offset_paged );
+        }
+
+void cuda_pinned_copy_from_i_(int **pinned_ptr,int *paged_ptr,int *elements,int *offset_pinned, int *offset_paged)
+{
+ 	 
+	std::copy((*pinned_ptr) + *offset_pinned, 
+		  (*pinned_ptr) + *offset_pinned + *elements, 
+		  paged_ptr +  *offset_paged );
+        }
+
+
+/*******************************************************************/
+/***************** cPTR ARITHMETICS*********************************/
+/*******************************************************************/
+void *get_cptr_d_(double **device_ptr,int *elements)
+{
+ 	      return (*device_ptr + *elements);
+        }
+
+void *get_cptr_f_(float **device_ptr,int *elements)
+{
+ 	      return (*device_ptr + *elements);
+        }
+
+void *get_cptr_i_(int **device_ptr,int *elements)
+{
+ 	      return (*device_ptr + *elements);
+        }
+
+
+/*******************************************************************/
+/*****************ASYN COPY OF MEMORY******************************/
+/*******************************************************************/
+void devicecopyasync_d_(double **device_pointer,double **fortval,
+		        int *size ,int *dev_offset, 
+			int *host_offset,int *flag,
+		        cudaStream_t **stream,int *num, int*err)
+                      {
+
+                      cudaMemcpyKind kind = (cudaMemcpyKind) *flag;
+                      //if (*flag== 0)//HOSTtoHOST
+                      if (*flag== 1)//HOSTtoDEVICE
+                          CuErrChk(   cudaMemcpyAsync(*device_pointer+*dev_offset,
+				       *fortval+*host_offset,
+				       sizeof(double)*(*size),
+				       kind,
+				       (*stream)[*num])); 
+
+                      if (*flag== 2)//DEVICEtoHOST
+	                  CuErrChk(   cudaMemcpyAsync( *fortval + *host_offset,
+				         *device_pointer + *dev_offset,
+					sizeof(double)*(*size),
+					kind,
+					(*stream)[*num]));
+	      
+	      
+                       //if (*flag== 3)//DEVICEtoDEVICE
+                      }
+
+void devicecopyasync_f_(float **device_pointer,float **fortval,
+		        int *size, int *dev_offset ,
+			int *host_offset, int *flag,
+		        cudaStream_t **stream, int *num, int*err)
+                     {
+
+                     cudaMemcpyKind kind = (cudaMemcpyKind) *flag;
+                     //if (*flag== 0)//HOSTtoHOST
+                     if (*flag== 1)//HOSTtoDEVICE
+                        *err = cudaMemcpyAsync(*device_pointer + *dev_offset,
+					       *fortval + *host_offset,
+					       sizeof(float)*(*size),
+					       kind,
+					       (*stream)[*num]);
+
+                     if (*flag== 2)//DEVICEtoHOST
+                        *err = cudaMemcpyAsync(*fortval + *host_offset,
+					       *device_pointer + *dev_offset,
+					       sizeof(float)*(*size),
+					       kind,(*stream)[*num]);
+                      //if (*flag== 3)//DEVICEtoDEVICE
+                      }
+
+void devicecopyasync_i_(int **device_pointer,int **fortval,
+		        int *size, int *dev_offset ,
+			int *host_offset, int *flag,
+		        cudaStream_t **stream, int *num, int*err)
+                     {
+
+                     cudaMemcpyKind kind = (cudaMemcpyKind) *flag;
+                     //if (*flag== 0)//HOSTtoHOST
+                     if (*flag== 1)//HOSTtoDEVICE
+                        *err = cudaMemcpyAsync(*device_pointer + *dev_offset,
+					       *fortval + *host_offset,
+					       sizeof(int)*(*size),
+					       kind,
+					       (*stream)[*num]);
+
+                     if (*flag== 2)//DEVICEtoHOST
+                        *err = cudaMemcpyAsync(*fortval + *host_offset,
+					       *device_pointer + *dev_offset,
+					       sizeof(int)*(*size),
+					       kind,(*stream)[*num]);
+                      //if (*flag== 3)//DEVICEtoDEVICE
+                      }
+
+
+
+/*******************************************************************/
+/*****************ALLOCATION OF MEMORY******************************/
+/*******************************************************************/
+void devicemalloc_d_(double **device_pointer,int *size, int *err)
+{
+              *err = cudaMalloc((void **)device_pointer, sizeof(double) * (*size )); 
+            }
+
+void devicemalloc_f_(float **device_pointer,int *size, int *err)
+{
+             *err = cudaMalloc((void **)device_pointer, sizeof(float) * (*size )); 
+                }
+
+void devicemalloc_i_(int **device_pointer,int *size, int*err)
+{
+              *err = cudaMalloc((void **)device_pointer, sizeof(int) * (*size ));
+ }
+
+
+/*******************************************************************/
+/*****************FREEING OF MEMORY*********************************/
+/*******************************************************************/
+void devicefree_d_(double **device_pointer, int *err)
+{
+              *err = cudaFree(*device_pointer);
+               }
+
+void devicefree_f_(float **device_pointer, int *err)
+{
+	
+              *err = cudaFree(*device_pointer); 
+                }
+
+void devicefree_i_(int **device_pointer, int*err)
+{
+              *err = cudaFree(*device_pointer);
+ }
+
+
+/*******************************************************************/
+/*********************COPY OPERATIONS*******************************/
+/*******************************************************************/
+void devicecopy_d_(double **device_pointer,double *fortval,int *size ,int *offset,int *flag, int*err)
+{
+              cudaMemcpyKind kind = (cudaMemcpyKind) *flag;
+              //if (*flag== 0)//HOSTtoHOST
+              if (*flag== 1)//HOSTtoDEVICE
+                 *err = cudaMemcpy(*device_pointer + *offset,fortval, sizeof(double) * (*size ), kind);
+              if (*flag== 2)//DEVICEtoHOST
+                 *err = cudaMemcpy(fortval,*device_pointer + *offset, sizeof(double) * (*size ), kind);
+              //if (*flag== 3)//DEVICEtoDEVICE
+            
+}
+
+void devicecopy_f_(float **device_pointer,float *fortval,int *size,int *offset ,int *flag, int*err)
+{
+              cudaMemcpyKind kind = (cudaMemcpyKind) *flag;
+              //if (*flag== 0)//HOSTtoHOST
+              if (*flag== 1)//HOSTtoDEVICE
+                 *err = cudaMemcpy(*device_pointer + *offset,fortval, sizeof(float) * (*size ), kind);
+              if (*flag== 2)//DEVICEtoHOST
+                 *err = cudaMemcpy(fortval,*device_pointer + *offset, sizeof(float) * (*size ), kind);
+              //if (*flag== 3)//DEVICEtoDEVICE
+            }
+
+void devicecopy_i_(int **device_pointer,int *fortval,int *size ,int *offset,int *flag, int*err)
+{
+              cudaMemcpyKind kind = (cudaMemcpyKind) *flag;
+              //if (*flag== 0);//HOSTtoHOST
+              if (*flag== 1)//HOSTtoDEVICE
+                 *err = cudaMemcpy(*device_pointer + *offset,fortval, sizeof(int) * (*size ), kind);
+              if (*flag== 2)//DEVICEtoHOST
+                 *err = cudaMemcpy(fortval,*device_pointer + *offset, sizeof(int) * (*size ), kind);
+              //if (*flag== 3);//DEVICEtoDEVICE
+            }
+
+
+/*******************************************************************/
+/*********************FILL OPERATIONS*******************************/
+/*******************************************************************/
+/*******************************************************************/
+/*
+void devicefill_i_(int **device_pointer,int *size ,int *val, int*err)
+{
+      //thrust::device_ptr<int> dev_ptr = thrust::device_pointer_cast(*device_pointer); 
+      //thrust::fill(dev_ptr, dev_ptr + *size, (int) *val);
+     }
+
+
+void devicefill_f_(float **device_pointer,int *size ,float *val, int*err)
+{
+       //thrust::device_ptr<float> dev_ptr = thrust::device_pointer_cast(*device_pointer); 
+       //thrust::fill(dev_ptr, dev_ptr + *size, (float) *val);
+      }
+
+
+void devicefill_d_(double **device_pointer,int *size ,double *val, int*err)
+{
+       //thrust::device_ptr<double> dev_ptr = thrust::device_pointer_cast(*device_pointer); 
+       //thrust::fill(dev_ptr, dev_ptr + *size, (double) *val);
+       }
+
+
+*/
+/*******************************************************************/
+/*****************HELPER OPERATIONS******************************/
+/*******************************************************************/
+__global__ void calc_tinydouble(double *dval) 
+{
+
+    int i = threadIdx.x + blockIdx.x*blockDim.x;
+    if (i==0) {
+        
+
+       
+        double xd, oldxd; 
+
+   
+      
+        xd = 1.; oldxd = 2.;
+
+       
+        while (xd != 0.)  {
+            oldxd /= 2.;
+            xd /= 2.;
+        }
+
+        *dval = oldxd;
+        
+
+    }
+    return;
+}
+
+
+void get_smallest_d_(double *tiny)
+{
+           
+             double *dval;
+             cudaMalloc(&dval, 1*sizeof(double));
+             CudaTest((char*)"Malloc failed in get_TINY");
+             calc_tinydouble<<<1,32>>>(dval);
+             CudaTest((char*)"CALC CONST failed  in get_TINY"); 
+             cudaMemcpy(tiny, dval, 1*sizeof(double), cudaMemcpyDeviceToHost);
+             CudaTest((char*)"memcpy failed in get_TINY");  
+             cudaFree(dval);	  
+	    
+	     
+
+    }
+
+
+
+
+
Index: source/hwaccel/GpuSupport/GpuSupportMain/CUDA/KernelCode/Makefile
===================================================================
--- source/hwaccel/GpuSupport/GpuSupportMain/CUDA/KernelCode/Makefile	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/hwaccel/GpuSupport/GpuSupportMain/CUDA/KernelCode/Makefile	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,3 @@
+CUDA_KERNEL_OBJS += CudaGeneralFunctions.o
+
+CudaGeneralFunctions.o : CudaError.cuh
Index: source/hwaccel/GpuSupport/GpuSupportMain/CUDA/GpuSupport_cudaFortInterface.F90
===================================================================
--- source/hwaccel/GpuSupport/GpuSupportMain/CUDA/GpuSupport_cudaFortInterface.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/hwaccel/GpuSupport/GpuSupportMain/CUDA/GpuSupport_cudaFortInterface.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,837 @@
+!!****if* source/hwaccel/GpuSupport/GpuSupportMain/CUDA/GpuSupport_cudaFortInterface
+!!
+!! NAME
+!!
+!!  GpuSupport_cudaFortInterface
+!!
+!!
+!! SYNOPSIS
+!!
+!!  use GpuSupport_cudaFortInterface
+!!
+!!
+!! DESCRIPTION
+!!
+!!  Fortran module containing data and interfaces for
+!!  general use of CUDA code from Fortran.
+!!  Contains subroutine GpuSupport_setDeviceMPIgroup.
+!!
+!!
+!!***
+
+Module GpuSupport_cudaFortInterface
+  use, intrinsic :: iso_c_binding
+
+
+  implicit none
+
+    
+    integer,parameter:: cudaMemcpyHostToHost=0, &
+                        cudaMemcpyHostToDevice=1, &
+                        cudaMemcpyDeviceToHost=2, &
+                        cudaMemcpyDeviceToDevice=3
+
+     integer,parameter:: cudaSuccess=0 , &
+     cudaErrorMissingConfiguration=1, &
+     cudaErrorMemoryAllocation=2, &
+     cudaErrorInitializationError=3, &
+     cudaErrorLaunchFailure=4, &
+     cudaErrorPriorLaunchFailure=5, &
+     cudaErrorLaunchTimeout=6, &
+     cudaErrorLaunchOutOfResources=7, &
+     cudaErrorInvalidDeviceFunction=8, &
+     cudaErrorInvalidConfiguration=9, &
+     cudaErrorInvalidDevice=10, &
+     cudaErrorInvalidValue=11, &
+     cudaErrorInvalidPitchValue=12, &
+     cudaErrorInvalidSymbol=13, &
+     cudaErrorMapBufferObjectFailed=14, &
+     cudaErrorUnmapBufferObjectFailed=15, &
+     cudaErrorInvalidHostPointer=16, &
+     cudaErrorInvalidDevicePointer=17, &
+     cudaErrorInvalidTexture=18, &
+     cudaErrorInvalidTextureBinding=19, &
+     cudaErrorInvalidChannelDescriptor=20, &
+     cudaErrorInvalidMemcpyDirection=21, &
+     cudaErrorAddressOfConstant=22, &
+     cudaErrorTextureFetchFailed=23, &
+     cudaErrorTextureNotBound=24, &
+     cudaErrorSynchronizationError=25, &
+     cudaErrorInvalidFilterSetting=26, &
+     cudaErrorInvalidNormSetting=27, &
+     cudaErrorMixedDeviceExecution=28, &
+     cudaErrorCudartUnloading=29, &
+     cudaErrorUnknown=30, &
+     cudaErrorNotYetImplemented=31, &
+     cudaErrorMemoryValueTooLarge=32, &
+     cudaErrorInvalidResourceHandle=33, &
+     cudaErrorNotReady=34, &
+     cudaErrorInsufficientDriver=35, &
+     cudaErrorSetOnActiveProcess=36, &
+     cudaErrorInvalidSurface=37, &
+     cudaErrorNoDevice=38, &
+     cudaErrorECCUncorrectable=39, &
+     cudaErrorSharedObjectSymbolNotFound=40 , &
+     cudaErrorSharedObjectInitFailed=41, &
+     cudaErrorUnsupportedLimit=42, &
+     cudaErrorDuplicateVariableName=43, &
+     cudaErrorDuplicateTextureName=44, &
+     cudaErrorDuplicateSurfaceName=45, &
+     cudaErrorDevicesUnavailable=46, &
+     cudaErrorStartupFailure=127, &
+     cudaErrorApiFailureBase=10000
+
+!/*******************************************************************/
+!/*****************Gpu DEVICE PROPERTIES******************************/
+!/*******************************************************************/
+
+
+
+
+
+
+
+
+interface
+    subroutine  GpuSupport_get_device_count(device_count,err) bind(C,name='get_device_count_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+        integer, intent(in) :: device_count
+        integer, intent(inout)::err
+    end subroutine  GpuSupport_get_device_count
+  end interface
+  
+
+
+
+interface
+    subroutine GpuSupport_getDeviceProperties( device_ID,&
+            mpcount)bind(C, name="get_device_properties_")
+     use, intrinsic :: ISO_C_BINDING
+     implicit none
+     integer, intent (inout)          :: device_ID,mpcount
+     
+     
+    end subroutine GpuSupport_getDeviceProperties
+end interface
+
+
+interface
+    subroutine GpuSupport_set_device(device_ID,err) bind(C,name='set_device_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+        integer, intent(in) :: device_ID
+        integer, intent(inout)::err
+    end subroutine  GpuSupport_set_device
+  end interface
+  
+
+interface
+    subroutine  GpuSupport_device_reset(err) bind(C,name='device_reset_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+        integer, intent(inout)::err
+    end subroutine  GpuSupport_device_reset
+  end interface
+  
+
+
+
+
+!/*******************************************************************/
+!/********************ERROR TEST*************************************/
+!/*******************************************************************/
+ interface
+    subroutine GpuSupport_print_last_error_string(str,err) bind(C,name='print_last_error_string_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+        character(c_char), intent(in) :: str
+       integer, intent(inout)::err
+    end subroutine GpuSupport_print_last_error_string
+  end interface
+  
+
+
+
+
+
+!/*******************************************************************/
+!/*****************   PINNED MEMORY   *******************************/
+!/*******************************************************************/
+ 
+
+
+ interface
+    subroutine  GpuSupport_cuda_pinned_allocate_d(device_ptr,num,err)&
+            bind(C,name='cuda_pinned_allocate_d_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+       type(c_ptr) , intent(inout) :: device_ptr
+       integer, intent(inout)::err
+       integer, intent(in)::num
+    end subroutine  GpuSupport_cuda_pinned_allocate_d
+  end interface
+ 
+
+ interface
+    subroutine  GpuSupport_cuda_pinned_free_d(device_ptr,err)&
+            bind(C,name='cuda_pinned_free_d_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+       type(c_ptr) , intent(inout) :: device_ptr
+       integer, intent(inout)::err
+       
+    end subroutine  GpuSupport_cuda_pinned_free_d
+  end interface
+ 
+
+ interface
+    subroutine  GpuSupport_cuda_pinned_allocate_f(device_ptr,num,err)&
+            bind(C,name='cuda_pinned_allocate_f_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+       type(c_ptr) , intent(inout) :: device_ptr
+       integer, intent(inout)::err
+       integer, intent(in)::num
+    end subroutine  GpuSupport_cuda_pinned_allocate_f
+  end interface
+ 
+
+ interface
+    subroutine  GpuSupport_cuda_pinned_free_f(device_ptr,err)&
+            bind(C,name='cuda_pinned_free_f_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+       type(c_ptr) , intent(inout) :: device_ptr
+       integer, intent(inout)::err
+       
+    end subroutine  GpuSupport_cuda_pinned_free_f
+  end interface
+ 
+ interface
+    subroutine  GpuSupport_cuda_pinned_allocate_i(device_ptr,num,err)&
+            bind(C,name='cuda_pinned_allocate_i_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+       type(c_ptr) , intent(inout) :: device_ptr
+       integer, intent(inout)::err
+       integer, intent(in)::num
+    end subroutine  GpuSupport_cuda_pinned_allocate_i
+  end interface
+ 
+
+ interface
+    subroutine  GpuSupport_cuda_pinned_free_i(device_ptr,err)&
+            bind(C,name='cuda_pinned_free_i_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+       type(c_ptr) , intent(inout) :: device_ptr
+       integer, intent(inout)::err
+       
+    end subroutine  GpuSupport_cuda_pinned_free_i
+  end interface
+ 
+!/*******************************************************************/
+!/*****************COPY TO/FROM PINNED MEMORY************************/
+!/*******************************************************************/
+
+
+ interface
+    subroutine  GpuSupport_cuda_pinned_copy_to_d(pinned_ptr,paged_ptr,elements,pinned_offset,paged_offset)&
+            bind(C,name='cuda_pinned_copy_to_d_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+       type(c_ptr) , intent(inout) :: pinned_ptr
+       integer, intent(in):: elements,pinned_offset,paged_offset
+       real(8), intent(inout) :: paged_ptr(elements)
+
+    end subroutine GpuSupport_cuda_pinned_copy_to_d
+  end interface
+ 
+
+ interface
+    subroutine  GpuSupport_cuda_pinned_copy_to_f(pinned_ptr,paged_ptr,elements,pinned_offset,paged_offset)&
+            bind(C,name='cuda_pinned_copy_to_f_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+       type(c_ptr) , intent(inout) :: pinned_ptr
+       integer, intent(in):: elements,pinned_offset,paged_offset
+       real(4), intent(inout) :: paged_ptr(elements)
+
+    end subroutine GpuSupport_cuda_pinned_copy_to_f
+  end interface
+ 
+ 
+  interface
+    subroutine  GpuSupport_cuda_pinned_copy_to_i(pinned_ptr,paged_ptr,elements,pinned_offset,paged_offset)&
+            bind(C,name='cuda_pinned_copy_to_i_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+       type(c_ptr) , intent(inout) :: pinned_ptr
+       integer, intent(in):: elements,pinned_offset,paged_offset
+       integer, intent(inout) :: paged_ptr(elements)
+
+    end subroutine GpuSupport_cuda_pinned_copy_to_i
+  end interface
+
+
+
+
+
+
+  interface
+    subroutine  GpuSupport_cuda_pinned_copy_from_d(pinned_ptr,paged_ptr,elements,pinned_offset,paged_offset)&
+            bind(C,name='cuda_pinned_copy_from_d_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+       type(c_ptr) , intent(inout) :: pinned_ptr
+       integer, intent(in):: elements,pinned_offset,paged_offset
+       real(8), intent(inout) :: paged_ptr(elements)
+
+    end subroutine GpuSupport_cuda_pinned_copy_from_d
+  end interface
+ 
+
+
+
+  interface
+    subroutine  GpuSupport_cuda_pinned_copy_from_f(pinned_ptr,paged_ptr,elements,pinned_offset,paged_offset)&
+            bind(C,name='cuda_pinned_copy_from_f_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+       type(c_ptr) , intent(inout) :: pinned_ptr
+       integer :: elements,pinned_offset,paged_offset
+       real(4), intent(inout) :: paged_ptr(elements)
+
+    end subroutine GpuSupport_cuda_pinned_copy_from_f
+  end interface
+ 
+
+
+  interface
+    subroutine  GpuSupport_cuda_pinned_copy_from_i(pinned_ptr,paged_ptr,elements,pinned_offset,paged_offset)&
+            bind(C,name='cuda_pinned_copy_from_i_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+       type(c_ptr) , intent(inout) :: pinned_ptr
+       integer :: elements,pinned_offset,paged_offset
+       integer, intent(inout) :: paged_ptr(elements)
+
+    end subroutine GpuSupport_cuda_pinned_copy_from_i
+  end interface
+ 
+
+
+!/*******************************************************************/
+!/*****************   PTR ARITHMETIK  *******************************/
+!/*******************************************************************/
+ 
+
+
+ interface
+    type(c_ptr) function  GpuSupport_get_cptr_d(device_ptr,offset)&
+             bind(C,name='get_cptr_d_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+       integer, intent(in)::offset
+       type(c_ptr) , intent(in) :: device_ptr
+       type(c_ptr) :: get_cptr
+    end function  GpuSupport_get_cptr_d
+  end interface
+ 
+
+ interface
+    type(c_ptr) function  GpuSupport_get_cptr_i(device_ptr,offset)&
+            bind(C,name='get_cptr_i_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+       integer, intent(in)::offset
+       type(c_ptr) , intent(in) :: device_ptr
+       type(c_ptr) :: get_cptr
+    end function  GpuSupport_get_cptr_i
+  end interface
+ 
+
+
+ interface
+	 type(c_ptr) function  GpuSupport_get_cptr_f(device_ptr,offset)&
+		              bind(C,name='get_cptr_f_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+       integer, intent(in)::offset
+       type(c_ptr) , intent(in) :: device_ptr
+       type(c_ptr) :: get_cptr
+    end function  GpuSupport_get_cptr_f
+  end interface
+ 
+
+
+
+
+
+
+!/*******************************************************************/
+!/*****************   Streams   *************************************/
+!/*******************************************************************/
+ interface
+    subroutine  GpuSupport_set_streams(streams,num,err) bind(C,name='set_streams_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+       type(c_ptr) , intent(inout) :: streams
+       integer, intent(in)::num
+       integer, intent(inout)::err
+    end subroutine  GpuSupport_set_streams
+  end interface
+
+
+ interface
+    subroutine  GpuSupport_del_streams(streams,num,err) bind(C,name='del_streams_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+       type(c_ptr) , intent(inout) :: streams
+       integer, intent(in)::num   
+       integer, intent(inout)::err
+    end subroutine  GpuSupport_del_streams
+  end interface
+
+
+ interface
+    subroutine  GpuSupport_wait_stream(streams,num,err) bind(C,name='wait_stream_')
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+       type(c_ptr) , intent(inout) :: streams
+       integer, intent(in)::num
+       integer, intent(inout)::err
+    end subroutine  GpuSupport_wait_stream
+  end interface
+!/*******************************************************************/
+
+
+
+
+!/*******************************************************************/
+!/*******************Async Copy Operatiions**************************/
+!/*******************************************************************/
+
+   interface
+     subroutine  GpuSupport_devicecopyasync_d(devPtr,arr,size,&
+	                         dev_offset,host_offset,&
+				 flag,stream,num,err) bind(C, name="devicecopyasync_d_")
+        use, intrinsic :: ISO_C_BINDING
+        implicit none
+        type(c_ptr), intent(inout) :: devPtr
+        integer, intent(in)        :: size
+        integer, intent(in)        :: dev_offset,host_offset
+	real(8), intent(inout) :: arr(0:size-1)
+        integer, intent(in)        :: flag
+	type(c_ptr), intent(inout) :: stream
+	integer, intent(in)        :: num 
+        integer, intent(inout)     :: err 
+      end subroutine  GpuSupport_devicecopyasync_d
+   end interface
+
+   interface
+     subroutine  GpuSupport_devicecopyasync_f(devPtr,arr,size,&
+		                  dev_offset,host_offset,&
+				  flag,stream,num,err) bind(C, name="devicecopyasync_f_")
+        use, intrinsic :: ISO_C_BINDING
+        implicit none
+        type(c_ptr) ,intent(inout):: devPtr
+        integer, intent(in)        :: size
+        integer, intent(in)        :: dev_offset, host_offset
+        real(4),    intent(in)        :: arr(0:size-1)
+        integer, intent(in)        :: flag
+        type(c_ptr), intent(inout) :: stream
+        integer, intent(in)        :: num 
+        integer, intent(inout)     :: err 
+      end subroutine  GpuSupport_devicecopyasync_f
+   end interface
+
+   interface
+     subroutine  GpuSupport_devicecopyasync_i(devPtr,arr,size,&
+             dev_offset,host_offset,&
+             flag,stream,num,err) bind(C, name="devicecopyasync_i_")
+        use, intrinsic :: ISO_C_BINDING
+        implicit none
+        type(c_ptr) ,intent(inout):: devPtr
+        integer, intent(in)        :: size
+        integer, intent(in)        :: dev_offset,host_offset
+        integer,    intent(in)        :: arr(0:size-1)
+        integer, intent(in)        :: flag
+        type(c_ptr), intent(inout) :: stream
+        integer, intent(in)        :: num 
+        integer, intent(inout)     :: err 
+      end subroutine  GpuSupport_devicecopyasync_i
+   end interface
+  
+
+
+
+
+
+
+
+
+
+!/*******************************************************************/
+!/*****************ALLOCATION OF MEMORY******************************/
+!/*******************************************************************/
+   interface !malloc double
+     subroutine  GpuSupport_devicemalloc_d(devPtr,size,err) bind(C, name="devicemalloc_d_")
+        use, intrinsic :: ISO_C_BINDING
+        implicit none
+        type(c_ptr) , intent(inout) :: devPtr
+        integer, intent(in)        :: size
+        integer, intent(inout)     :: err 
+      end subroutine  GpuSupport_devicemalloc_d
+   end interface
+
+
+   interface !malloc float
+     subroutine  GpuSupport_devicemalloc_f(devPtr,size,err) bind(C, name="devicemalloc_f")
+        use, intrinsic :: ISO_C_BINDING
+        implicit none
+        type(c_ptr), intent(inout) :: devPtr
+        integer, intent(in)        :: size
+        integer, intent(inout)     :: err 
+      end subroutine  GpuSupport_devicemalloc_f
+   end interface
+
+
+   interface !malloc int
+     subroutine  GpuSupport_devicemalloc_i(devPtr,size,err) bind(C, name="devicemalloc_i")
+        use, intrinsic :: ISO_C_BINDING
+        implicit none
+        type(c_ptr), intent(inout) :: devPtr
+        integer, intent(in)        :: size
+        integer, intent(inout)     :: err 
+      end subroutine  GpuSupport_devicemalloc_i
+   end interface
+
+
+
+!/*******************************************************************/
+!/*****************FREEING OF MEMORY*********************************/
+!/*******************************************************************/
+ interface !free double
+     subroutine  GpuSupport_devicefree_d(devPtr,err) bind(C, name="devicefree_d_")
+        use, intrinsic :: ISO_C_BINDING
+        implicit none
+        type(c_ptr),intent(inout)  :: devPtr
+        integer, intent(inout)     :: err 
+      end subroutine  GpuSupport_devicefree_d
+   end interface
+
+
+   interface !free float
+     subroutine devicefree_f(devPtr,err) bind(C, name="devicefree_f_")
+        use, intrinsic :: ISO_C_BINDING
+        implicit none
+        type(c_ptr),intent(inout)  :: devPtr
+        integer, intent(inout)     :: err 
+      end subroutine devicefree_f
+   end interface
+
+
+   interface !free int
+      subroutine devicefree_i(devPtr,err) bind(C, name="devicefree_i_")
+        use, intrinsic :: ISO_C_BINDING
+        implicit none
+        type(c_ptr),intent(inout)  :: devPtr
+        integer, intent(inout)     :: err 
+      end subroutine devicefree_i
+   end interface
+
+
+!/*******************************************************************/
+!/*********************FILL OPERATIONS*******************************/
+!/*******************************************************************/
+
+ !  interface
+ !    subroutine devicefill_d(devPtr,size,val,err) bind(C, name="devicefill_d_")
+ !       use, intrinsic :: ISO_C_BINDING
+ !       implicit none
+ !       type(c_ptr) ,intent(inout) :: devPtr
+ !       integer, intent(in)        :: size
+!	real, intent(in)           :: val
+!        integer, intent(inout)     :: err 
+!      end subroutine devicefill_d
+!   end interface
+
+ !  interface
+ !    subroutine devicefill_f(devPtr,size,val,err) bind(C, name="devicefill_f_")
+ !       use, intrinsic :: ISO_C_BINDING
+ !       implicit none
+ !       type(c_ptr) ,intent(inout) :: devPtr
+ !       integer, intent(in)        :: size
+!	real, intent(in)           :: val 
+!        integer, intent(inout)     :: err 
+!      end subroutine devicefill_f
+!   end interface
+
+
+ !  interface
+ !    subroutine devicefill_i(devPtr,size,val,err) bind(C, name="devicefill_i_")
+ !       use, intrinsic :: ISO_C_BINDING
+ !       implicit none
+ !       type(c_ptr) ,intent(inout) :: devPtr
+ !       integer, intent(in)        :: size
+ !       integer, intent(in)        :: val 
+ !       integer, intent(inout)     :: err 
+ !     end subroutine devicefill_i
+ !  end interface
+
+
+
+
+!/*******************************************************************/
+!/*********************COPY OPERATIONS*******************************/
+!/*******************************************************************/
+
+   interface
+     subroutine GpuSupport_devicecopy_d(devPtr,arr,size,offset,flag,err) bind(C, name="devicecopy_d_")
+        use, intrinsic :: ISO_C_BINDING
+        implicit none
+        type(c_ptr) ,intent(inout):: devPtr
+        integer, intent(in)        :: size
+        integer, intent(in)        :: offset
+        real,    intent(in)        :: arr(0:size-1)
+        integer, intent(in)        :: flag
+        integer, intent(inout)     :: err 
+      end subroutine GpuSupport_devicecopy_d
+   end interface
+
+   interface
+     subroutine GpuSupport_devicecopy_f(devPtr,arr,size,offset,flag,err) bind(C, name="devicecopy_f_")
+        use, intrinsic :: ISO_C_BINDING
+        implicit none
+        type(c_ptr) ,intent(inout):: devPtr
+        integer, intent(in)        :: size
+        integer, intent(in)        :: offset
+        real,    intent(in)        :: arr(0:size-1)
+        integer, intent(in)        :: flag
+        integer, intent(inout)     :: err 
+      end subroutine GpuSupport_devicecopy_f
+   end interface
+
+   interface
+     subroutine GpuSupport_devicecopy_i(devPtr,arr,size,offset,flag,err) bind(C, name="devicecopy_i_")
+        use, intrinsic :: ISO_C_BINDING
+        implicit none
+        type(c_ptr) ,intent(inout):: devPtr
+        integer, intent(in)        :: size
+        integer, intent(in)        :: offset
+        real,    intent(in)        :: arr(0:size-1)
+        integer, intent(in)        :: flag
+        integer, intent(inout)     :: err 
+      end subroutine GpuSupport_devicecopy_i
+   end interface
+
+
+  interface
+    subroutine get_smallest_d(smallest) bind(C, name="get_smallest_d_")
+        use, intrinsic :: ISO_C_BINDING
+        implicit none
+	    real, intent(inout) :: smallest
+    end subroutine get_smallest_d
+  end interface
+
+!/*******************************************************************/
+!/***********  CREATE A DEVICE COMMUNICATER *************************/
+!/*******************************************************************/
+!   interface
+ !   subroutine set_device_mpigroup(base_rank,base_commSize,base_comm,&
+!	                        gpu_rank,gpu_commSize,gpu_comm,&
+!				device_ID,DeviceCount,device_name,&
+!				len_name,globalmem,minor,major,mpcount)
+ !    implicit none
+ !    integer, intent(in)::base_rank,base_commSize,base_comm
+ !    integer, intent(inout)::gpu_rank,gpu_commSize,gpu_comm
+ !    
+ !    
+ !   end subroutine set_device_mpigroup
+ !  end interface
+!/*******************************************************************/
+
+
+
+
+
+
+contains
+
+
+subroutine GpuSupport_setDeviceMPIgroup(base_commSize,base_rank,base_comm,&
+        gpu_commSize_out,gpu_rank_out,gpu_comm_out,&
+        device_ID_out,deviceCount_out,host_name_out)
+
+use GpuSupport_data, ONLY: gpu_set,gpu_rank,gpu_comm,gpu_commSize,&
+                            gpu_deviceCount,gpu_deviceID,gpu_deviceSmpCount, gpu_hostName
+
+implicit none
+#include "Flash_mpi.h"
+
+
+ integer, intent(in)                    :: base_rank,base_commSize,base_comm
+ integer, intent(inout)                 :: gpu_rank_out,gpu_commSize_out,gpu_comm_out
+ integer, intent(inout)                 :: device_ID_out,deviceCount_out
+ character(len=MPI_MAX_PROCESSOR_NAME),intent(inout) :: host_name_out 
+ character(len=256) :: device_name 
+ integer :: namelen,ierr,memsize,len_name
+ integer :: i,j,k,NodeCount,Nodesize, MinDevice, MaxDevice, noderank
+ integer,allocatable :: members(:), gpu_ranks(:)
+ character(len=MPI_MAX_PROCESSOR_NAME) :: all_host_names(0:base_commSize-1) 
+ character(len=MPI_MAX_PROCESSOR_NAME) :: unique_host_names(0:base_commSize-1) 
+ integer :: color,color2,grprank,noderankt
+ integer :: all_device_counts(0:base_commSize-1)
+ integer :: unique_colors(0:base_commSize-1)
+ integer :: gpu2_rank,gpu2_comm,gpu2_commSize
+ integer :: minor,major, mpcount
+ integer :: globalmem
+
+ if (.not. gpu_set) then
+     !write(*,*)"THIS IS NOT SET :!!!!"
+     gpu_set = .true.
+
+    len_name =256
+    call get_device_count(gpu_deviceCount,ierr)
+            
+
+    call MPI_Allgather(gpu_deviceCount,1,MPI_INT,&
+                    all_device_counts,1,MPI_INT,&
+                    base_comm,ierr);
+
+ 
+
+    call MPI_Get_processor_name(gpu_hostName, namelen,ierr);
+    all_host_names(base_rank) = gpu_hostName;
+
+    call MPI_Allgather(gpu_hostName,MPI_MAX_PROCESSOR_NAME,FLASH_CHAR,&
+                    all_host_names,MPI_MAX_PROCESSOR_NAME,FLASH_CHAR,&
+                    base_comm,ierr);
+
+    call MPI_Allgather(gpu_deviceCount,1,MPI_INT,&
+                    all_device_counts,1,MPI_INT,&
+                    base_comm,ierr);
+
+ 
+
+  ! get the minimum count of devices
+    MinDevice=minval(all_device_counts)
+    MaxDevice=maxval(all_device_counts)
+#ifdef GPU_DEBUG
+  write(*,*)"rank ",base_rank,"has",MaxDevice,"devices"
+#endif
+  if(base_rank == 0) then
+     if (MinDevice .lt. 1) &
+          call Driver_abortFlash("ERROR: DeviceCount is 0 in GpuSupport_setDeviceMPIgroup() of GpuSupport_cudaFortInterface.F90")
+  
+     if(MaxDevice .gt. MinDevice)then
+      write(*,*) "The number of GPU devices per compute node is set to ",MinDevice,&
+                  "as a result of the device count on node :",all_host_names(minloc(all_device_counts))
+     endif
+   endif
+  
+  call MPI_Barrier(base_comm,ierr)
+
+
+
+  !DEBUG 
+  
+!  MinDevice = 1
+  deviceCount_out = MinDevice
+
+
+
+  NodeCount = 0
+  unique_host_names(0)=all_host_names(0)
+  if(gpu_hostName .eq.unique_host_names(NodeCount) )then
+         color=NodeCount
+   endif
+
+
+  outer: do i=1,base_commSize-1
+     do j=0,NodeCount
+        if (unique_host_names(j) .eq.  all_host_names(i)) then
+           ! Found a match so start looking again
+           cycle outer
+        end if
+     end do
+     ! No match found so add it to the output
+     NodeCount = NodeCount + 1
+     unique_host_names(NodeCount) = all_host_names(i)
+     if(gpu_hostName .eq.unique_host_names(NodeCount) )then
+       color=NodeCount
+     endif
+  end do outer
+!  write(*,*)"Nodecount",Nodecount,color,gpu_hostName
+  NodeCount = NodeCount+1
+  NodeSize = base_commSize/NodeCount
+  
+  
+  
+
+ call MPI_Comm_split(base_comm, color,base_rank,gpu_comm,ierr)
+ call MPI_Comm_rank(gpu_comm, gpu_rank,ierr)
+ call MPI_Comm_size(gpu_comm, gpu_commSize,ierr)
+! write(*,*)"My old rank was" ,gr_meshMe,"of" ,gr_meshNumProcs,&
+!"and my new rank is",gpu_rank,&
+!"of",gpu_commSize,"in grp",&
+!unique_host_names(color), "with color2",color
+gpu_deviceID = 0
+
+ if(gpu_deviceCount .ge. 2)then
+    noderank = base_rank  - (color * NodeSize)
+    color2   = mod(noderank,gpu_deviceCount)
+    gpu_deviceID = color2
+    device_ID_out = color2
+    call MPI_Comm_split(gpu_comm, color2,gpu_rank,gpu2_comm,ierr)
+    call MPI_Comm_rank(gpu2_comm, gpu2_rank,ierr)
+    call MPI_Comm_size(gpu2_comm, gpu2_commSize,ierr)
+
+
+#ifdef GPU_DEBUG
+  write(*,*)"My old rank was" ,base_rank, "of" ,base_commSize,&
+            "and my new rank is",gpu2_rank,&
+            "of",gpu2_commSize,"in grp",&
+             unique_host_names(color), "with color",color2
+#endif
+ 
+ call  MPI_Comm_free(gpu_comm,ierr)
+ gpu_comm = gpu2_comm
+ gpu_rank = gpu2_rank
+ gpu_commSize = gpu2_commSize
+
+
+endif
+
+
+#ifdef GPU_DEBUG 
+ do i = 0, base_commSize-1
+  if(base_rank .eq. i)then     
+     call GpuSupport_getDeviceProperties(gpu_deviceID,mpcount)
+   
+     
+
+
+  endif
+  call MPI_Barrier(base_comm,ierr)
+ end do
+#endif 
+ 
+
+endif
+
+gpu_comm_out = gpu_comm
+gpu_commSize_out=gpu_commSize
+gpu_rank_out = gpu_rank
+device_ID_out = gpu_deviceID
+
+return
+
+
+
+end subroutine GpuSupport_setDeviceMPIgroup
+
+
+end module GpuSupport_cudaFortInterface 
Index: source/hwaccel/GpuSupport/GpuSupportMain/CUDA/Makefile
===================================================================
--- source/hwaccel/GpuSupport/GpuSupportMain/CUDA/Makefile	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/hwaccel/GpuSupport/GpuSupportMain/CUDA/Makefile	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,2 @@
+GpuSupport += GpuSupport_data.o\
+			   KernelCode.o
Index: source/hwaccel/GpuSupport/GpuSupportMain/CUDA/GpuSupport_data.F90
===================================================================
--- source/hwaccel/GpuSupport/GpuSupportMain/CUDA/GpuSupport_data.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/hwaccel/GpuSupport/GpuSupportMain/CUDA/GpuSupport_data.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,22 @@
+module  GpuSupport_data
+use, intrinsic :: iso_c_binding
+implicit none
+#include "Flash_mpi.h"
+
+
+      !MPI COMMUNICATOR FIELDS
+      !Typically there is one CommGrp per GPU Device
+      ! On a 1 GPU per Node scenario, this leads to one CommGrp per Node
+
+      logical, save :: gpu_set = .false.
+      integer, save :: gpu_rank         !THE MPI GPU RANK
+      integer, save :: gpu_commSize     !THE GPU MPI COMMUNICATOR SIZE
+      integer, save :: gpu_comm         !THE GPU MPI COMMUNICATOR
+
+      !GPU DEVICE SPECIFIC FIELDS
+      integer, save :: gpu_deviceCount !The number of devices on the host
+      integer, save :: gpu_deviceSmpCount ! the number of smp for the selected device
+      integer, save :: gpu_deviceID =-1! the device ID default = 0
+      character(len=MPI_MAX_PROCESSOR_NAME), save :: gpu_hostName 
+
+end module GpuSupport_data
Index: source/hwaccel/GpuSupport/GpuSupport_cudaFortInterface.F90
===================================================================
--- source/hwaccel/GpuSupport/GpuSupport_cudaFortInterface.F90	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/hwaccel/GpuSupport/GpuSupport_cudaFortInterface.F90	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,93 @@
+!!****f* source/hwaccel/GpuSupport/GpuSupport_cudaFortInterface
+!!
+!! NAME
+!!
+!!  GpuSupport_cudaFortInterface
+!!
+!!
+!! SYNOPSIS
+!!
+!!  use GpuSupport_cudaFortInterface
+!!
+!!  call GpuSupport_getDeviceProperties(integer(INOUT)::device_ID,
+!!                                      integer(INOUT)::mpcount)
+!!
+!!  call GpuSupport_setDeviceMPIgroup(integer(in) :: base_commSize,
+!!                                    integer(in) :: base_rank,
+!!                                    integer(in) :: base_comm,
+!!                                 integer(INOUT) :: gpu_commSize_out,
+!!                                 integer(INOUT) :: gpu_rank_out,
+!!                                 integer(INOUT) :: gpu_comm_out,
+!!                                 integer(INOUT) :: device_ID_out,
+!!                                 integer(INOUT) :: deviceCount_out,
+!!       character(len=MPI_MAX_PROCESSOR_NAME)(INOUT)::host_name_out)
+!!
+!! DESCRIPTION
+!!
+!!  Fortran module containing data and interfaces for
+!!  general use of CUDA code from Fortran.
+!!  Contains subroutine GpuSupport_setDeviceMPIgroup.
+!!
+!!
+!!***
+
+!!  This is a stub version.
+
+Module GpuSupport_cudaFortInterface
+  use, intrinsic :: iso_c_binding
+
+
+  implicit none
+
+!/*******************************************************************/
+!/*****************Gpu DEVICE PROPERTIES******************************/
+!/*******************************************************************/
+
+
+  interface
+     subroutine GpuSupport_getDeviceProperties( device_ID,&
+            mpcount)bind(C, name="get_device_properties_")
+       use, intrinsic :: ISO_C_BINDING
+       implicit none
+       integer, intent (inout)          :: device_ID,mpcount
+
+     end subroutine GpuSupport_getDeviceProperties
+  end interface
+
+
+!/*******************************************************************/
+!/***********  CREATE A DEVICE COMMUNICATER *************************/
+!/*******************************************************************/
+!   interface
+ !   subroutine set_device_mpigroup(base_rank,base_commSize,base_comm,&
+!	                        gpu_rank,gpu_commSize,gpu_comm,&
+!				device_ID,DeviceCount,device_name,&
+!				len_name,globalmem,minor,major,mpcount)
+ !    implicit none
+ !    integer, intent(in)::base_rank,base_commSize,base_comm
+ !    integer, intent(inout)::gpu_rank,gpu_commSize,gpu_comm
+ !    
+ !    
+ !   end subroutine set_device_mpigroup
+ !  end interface
+!/*******************************************************************/
+
+contains
+
+  subroutine GpuSupport_setDeviceMPIgroup(base_commSize,base_rank,base_comm,&
+        gpu_commSize_out,gpu_rank_out,gpu_comm_out,&
+        device_ID_out,DeviceCount_out,host_name_out)
+
+    implicit none
+#include "Flash_mpi.h"
+
+    integer, intent(in)                    :: base_rank,base_commSize,base_comm
+    integer, intent(inout)                 :: gpu_rank_out,gpu_commSize_out,gpu_comm_out
+    integer, intent(inout)                 :: device_ID_out,DeviceCount_out
+    character(len=MPI_MAX_PROCESSOR_NAME),intent(inout) :: host_name_out 
+
+    ! stub version, no executable code.
+
+end subroutine GpuSupport_setDeviceMPIgroup
+
+end module GpuSupport_cudaFortInterface 
Index: source/hwaccel/GpuSupport/Makefile
===================================================================
--- source/hwaccel/GpuSupport/Makefile	(.../tags/FLASH4.4_release)	(revision 0)
+++ source/hwaccel/GpuSupport/Makefile	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,2 @@
+GpuSupport += GpuSupport_interface.o GpuSupport_init.o\
+			   GpuSupport_cudaFortInterface.o
Index: sites/EXAMPLE_GPU/Makefile.h
===================================================================
--- sites/EXAMPLE_GPU/Makefile.h	(.../tags/FLASH4.4_release)	(revision 0)
+++ sites/EXAMPLE_GPU/Makefile.h	(.../branches/flash4_lukat)	(revision 25574)
@@ -0,0 +1,206 @@
+# FLASH makefile definitions for the Intel ifc compilers on Linux
+
+#----------------------------------------------------------------------------
+# Set the HDF/HDF5 library paths -- these need to be updated for your system
+#----------------------------------------------------------------------------
+
+HDF5_PATH   = 
+SZ_PATH     = 
+
+
+#----------------------------------------------------------------------------
+# Set the CUDA library path --  needs to be updated for your system
+#----------------------------------------------------------------------------
+
+
+CUDA_PATH   = 
+
+
+#----------------------------------------------------------------------------
+# Compiler VARIANTS 
+#----------------------------------------------------------------------------
+
+# INTEL COMPILERS and nvcc
+
+FORT_COMP = ifort 
+CC_COMP   = icc
+CXX_COMP  = icpc
+NVCC_COMP = nvcc
+
+#MPI wrapper compilers for intel
+MPI_FORCOMP = mpiifort
+MPI_CCCOMP  = mpiicc
+MPI_CXXCOMP = mpiicpc
+MPI_NVCCOMP = nvcc
+
+
+#----------------------------------------------------------------------------
+# Compiler
+# Use the mpi wrappers or specify MPI includes and libs in 
+# MPI_INC_CC, MPI_INC_F, MPI_INC_CX
+#----------------------------------------------------------------------------
+
+
+FCOMP   = $(MPI_FORCOMP)
+CCOMP   = $(MPI_CCCOMP)
+CPPCOMP = $(MPI_CXXCOMP) 
+CUCOMP  = $(MPI_NVCCOMP)
+GPU_CPPC  = $(MPI_CXXCOMP)
+
+#----------------------------------------------------------------------------
+# Compilation flags
+#
+#  Three sets of compilation/linking flags are defined: one for optimized
+#  code, one for testing, and one for debugging.  The default is to use the 
+#  _OPT version.  Specifying -debug to setup will pick the _DEBUG version,
+#  these should enable bounds checking.  Specifying _TEST is used for 
+#  flash_test, and is set for quick code generation, and (sometimes) 
+#  profiling.  The Makefile generated by setup will assign the generic token 
+#  (ex. FFLAGS) to the proper set of flags (ex. FFLAGS_OPT).
+#----------------------------------------------------------------------------
+#INCLUDE FLAGS
+HDF5_INC  = -I $(HDF5_PATH)/include
+TRACE_INC = 
+
+
+#ifort
+FFLAGS_OPT   =   -c -r8 -i4 -O3 -xHost -mcmodel=large -inline-level=2 $(HDF5_INC) 
+FFLAGS_DEBUG =   -c -r8 -i4 -g  -traceback  -mcmodel=large -debug all 
+FFLAGS_TEST  =   -c -r8 -i4 -g  -traceback  -mcmodel=large -debug all -D TRACE=1 $(TRACE_INC)
+
+
+#icc
+CFLAGS_OPT    =   -c  -O3  -xHost  -mcmodel=large -inline-level=2  $(HDF5_INC) 
+CFLAGS_DEBUG  =   -c  -g  -traceback -mcmodel=large  -debug all  $(HDF5_INC) 
+CFLAGS_TEST   =   -c  -g  -traceback -mcmodel=large  -debug all  $(HDF5_INC)  $(TRACE_INC) -D TRACE=1
+
+
+#gpucxx
+GPU_CFLAGS_OPT   = "-O3 -xHost -inline-level=2"
+GPU_CFLAGS_DEBUG = "-g -traceback -debug all -mcmodel=large -D GPU_DEBUG"
+GPU_CFLAGS_TEST  = "-g -traceback -mcmodel=large -debug all -D TRACE -D GPU_DEBUG"
+
+#nvcc
+CU_FLAGS_OPT = -c -rdc=true -maxrregcount 32 -O3\
+	       --generate-code arch=compute_30,code=sm_35\
+	       --generate-code arch=compute_30,code=sm_30\
+	       --generate-code arch=compute_20,code=sm_20\
+	       --generate-code arch=compute_20,code=sm_21\
+	       -ccbin=$(GPU_CPPC) -Xcompiler $(GPU_CFLAGS_OPT)
+
+		  
+CU_FLAGS_DEBUG =  -c -g -G -lineinfo -v -rdc=true -maxrregcount 32\
+		  --ptxas-options=-v -res-usage -O0\
+		  --generate-code arch=compute_30,code=sm_35\
+		  --generate-code arch=compute_30,code=sm_30\
+		  --generate-code arch=compute_20,code=sm_20\
+		  --generate-code arch=compute_20,code=sm_21\
+		  -ccbin=$(GPU_CPPC) -Xcompiler $(GPU_CFLAGS_DEBUG)\
+		  -D GPU_DEBUG
+
+
+CU_FLAGS_TEST =  -c -g -G -lineinfo -v -rdc=true -maxrregcount 32\
+		 --ptxas-options=-v -res-usage -O0\
+		 --generate-code arch=compute_30,code=sm_35\
+		 --generate-code arch=compute_30,code=sm_30\
+		 --generate-code arch=compute_20,code=sm_20\
+		 --generate-code arch=compute_20,code=sm_21\
+		 -ccbin=$(GPU_CPPC) -Xcompiler $(GPU_CFLAGS_TEST)\
+		 -D GPU_DEBUG -D TRACE
+
+
+
+
+
+#---------------------------------------------------------------------------
+#LINKER
+#USE THE FORTRAN COMPILER AS LINKER
+#--------------------------------------------------------------------------
+
+
+LINK    =  $(FCOMP)
+
+#------------------------------------------------------------------------
+#DEVICE CODE LINKER USE NVCC
+#-------------------------------------------------------------------------
+GPU_LINK = $(CUCOMP)
+
+#----------------------------------------------------------------------------
+# Linker flags
+#
+#  There is a seperate version of the linker flags for each of the _OPT, 
+#  _DEBUG, and _TEST cases.
+#----------------------------------------------------------------------------
+
+LFLAGS_OPT   =   -i4 -r8 -o 
+LFLAGS_DEBUG =   -i4 -r8 -o
+LFLAGS_TEST  =   -i4 -r8 -o
+
+
+GPU_L_FLAGS_OPT = -O3\
+		  --generate-code arch=compute_30,code=sm_35\
+		  --generate-code arch=compute_30,code=sm_30\
+		  --generate-code arch=compute_20,code=sm_20\
+		  --generate-code arch=compute_20,code=sm_21\
+		  -dlink -maxrregcount 32 -rdc=true -ccbin=$(GPU_CPPC) -Xcompiler $(GPU_CFLAGS_OPT) -o
+
+
+GPU_L_FLAGS_DEBUG =  -g -lineinfo -v -O0 -dlink\
+		     --generate-code arch=compute_30,code=sm_35\
+		     --generate-code arch=compute_30,code=sm_30\
+		     --generate-code arch=compute_20,code=sm_20\
+		     --generate-code arch=compute_20,code=sm_21\
+		     -rdc=true -maxrregcount 32 --ptxas-options=-v  -ccbin=$(GPU_CPPC) -Xcompiler $(GPU_CFLAGS_DEBUG) -o
+
+
+GPU_L_FLAGS_TEST =  -g -lineinfo -v -O0 -dlink\
+		    --generate-code arch=compute_30,code=sm_35\
+		    --generate-code arch=compute_30,code=sm_30\
+		    --generate-code arch=compute_20,code=sm_20\
+		    --generate-code arch=compute_20,code=sm_21\
+		    -rdc=true -maxrregcount 32 --ptxas-options=-v -ccbin=$(GPU_CPPC) -Xcompiler $(GPU_CFLAGS_TEST) -o
+
+
+
+#----------------------------------------------------------------------------
+# Library specific linking
+#
+#  If a FLASH module has a 'LIBRARY xxx' line in its Config file, we need to
+#  create a macro in this Makefile.h for LIB_xxx, which will be added to the
+#  link line when FLASH is built.  This allows us to switch between different
+#  (incompatible) libraries.  We also create a _OPT, _DEBUG, and _TEST
+#  library macro to add any performance-minded libraries (like fast math),
+#  depending on how FLASH was setup.
+#----------------------------------------------------------------------------
+
+LIB_MPI   =   
+LIB_CUDA  =   -L$(CUDA_PATH)/lib64 -L$(CUDA_PATH)/extras/CUPTI/lib64/ -lcudart -lcuda -lstdc++ 
+LIB_HDF5  =   -L$(HDF5_PATH)/lib -lhdf5  -L$(SZ_PATH)/lib -lz 
+LIB_OPT   =    
+LIB_DEBUG =   
+LIB_TEST  =  
+
+
+
+#----------------------------------------------------------------------------
+# Additional machine-dependent object files
+#
+#  Add any machine specific files here -- they will be compiled and linked
+#  when FLASH is built.
+#----------------------------------------------------------------------------
+
+MACHOBJ = 
+
+
+#----------------------------------------------------------------------------
+# Additional commands
+#---------------------------------------------------------------------------- 
+
+MV = mv -f
+AR = ar -r
+RM = rm -f
+CD = cd
+RL = ranlib
+ECHO = echo
+
+
Index: bin/genFiles.py
===================================================================
--- bin/genFiles.py	(.../tags/FLASH4.4_release)	(revision 25574)
+++ bin/genFiles.py	(.../branches/flash4_lukat)	(revision 25574)
@@ -559,6 +559,8 @@
         if type(setupVarDict[key]) == types.BooleanType and setupVarDict[key]:
             useopenmp = 1
             break
+    
+    usecuda = GVars.usecuda
 
     if GVars.build_tau:
         shutil.copyfile(os.path.join(GVars.tauDir,'select.tau'),
Index: bin/setup_shortcuts.txt
===================================================================
--- bin/setup_shortcuts.txt	(.../tags/FLASH4.4_release)	(revision 25574)
+++ bin/setup_shortcuts.txt	(.../branches/flash4_lukat)	(revision 25574)
@@ -195,3 +195,6 @@
 mcheck:use_mcheck=True:--defines=MCHECK:--with-unit=source/monitors/Debugger/DebuggerMain/mcheck
 dmalloc:use_dmalloc=True:--defines=DMALLOC:--with-unit=source/monitors/Debugger/DebuggerMain/dmalloc
 ddt:use_ddt=True:--defines=DDT:--with-unit=source/monitors/Debugger/DebuggerMain/dmalloc
+
+#Support CUDA for GPU.
+cuda:-cuda:-unit=hwaccel/GpuSupport/GpuSupportMain/CUDA
Index: bin/linkFiles.py
===================================================================
--- bin/linkFiles.py	(.../tags/FLASH4.4_release)	(revision 25574)
+++ bin/linkFiles.py	(.../branches/flash4_lukat)	(revision 25574)
@@ -39,11 +39,11 @@
     self.objDir = objDir
     # In case of noClobber dont touch files with following extension initially
     self.noClobberExt = ['.so', '.a', '.o', '.unit', '.F90', '.c', '.fh', 
-                    ".F",".h",".mod",'.py'] 
+                    ".F",".h",".mod",'.py','.cu','.cuh','.mk'] 
     # extensions of files are never deleted (unless the source is deleted)
     self.derExt = [".o",".mod"]
     # precedence for extensions in each group
-    groups = [['.f90', '.F90', '.f', '.c', '.F', '.C', '.cxx'], '.py', '.fh', ['.h','.hxx']]
+    groups = [['.f90', '.F90', '.f', '.c', '.F', '.C', '.cxx', '.cu'], '.py', '.fh', ['.h','.hxx','.cuh']]
     #put extensions that aren't grouped into a one-item list (save typing)
     self.groupdict = {} # dictionary mapping an extension to list of related extensions
     self.exts = []
Index: bin/Makefile.tpl
===================================================================
--- bin/Makefile.tpl	(.../tags/FLASH4.4_release)	(revision 25574)
+++ bin/Makefile.tpl	(.../branches/flash4_lukat)	(revision 25574)
@@ -12,6 +12,7 @@
 ## useopenmp    -> 0 or 1 depending on whether certain setup variables with names like 
 ##                 thread{BlockList,WithinBlock,RayTrace,ProtonTrace} are of Boolean
 ##                 type and True.
+## usecuda      -> 0 or 1 depending on whether we want to compile with CUDA
 ## includeTAU   -> ???
 ## pdbvar
 ## pdbrule
@@ -36,8 +37,8 @@
 REORDERFLAG = %(indexReorder)d
 FLASHBINARY = true
 USEOPENMP = %(useopenmp)d
+USECUDA = %(usecuda)d
 
-
 # -s => Do not print the actual command being executed
 # -r => Do not use any implicit rules (speeds up performance)
 # -d => Print debugging information on how make is making decisions
@@ -53,7 +54,6 @@
 
 default: $(EXE)
 
-
 #\tMachine-dependent include file
 
 CDEFINES = %(cDefines)s
@@ -65,6 +65,9 @@
 PRINT_F77_FLAGS = @$(ECHO) F77: $(FCOMP) $(FFLAGS) $(FDEFINES)
 PRINT_F90_FLAGS = @$(ECHO) F90: $(FCOMP) $(FFLAGS) $(F90FLAGS) $(FDEFINES)
 PRINT_C_FLAGS = @$(ECHO) C: $(CCOMP) $(CFLAGS) $(CDEFINES)
+PRINT_CU_FLAGS = @$(ECHO) CU: $(CUCOMP) $(CU_FLAGS) $(CDEFINES)
+PRINT_GPU_CXX_FLAGS = @$(ECHO) GPU_CXX: $(GPU_CPPC) $(GPU_CFLAGS)
+PRINT_GPU_L_FLAGS = @$(ECHO) GPULinker: $(GPU_LINK) $(GPU_L_FLAGS) KernelCode.o \(Object files of Gpu-units\)
 PRINT_L_FLAGS = @$(ECHO) Linker: $(LINK) $(LFLAGS) $(EXE) \(Object files of units\) $(LIB) $(CONFIG_LIB)
 PRINT_FLAGS_FILE = setup_flags
 SUCCESS_FILE = .success
@@ -126,7 +129,8 @@
 
 .SUFFIXES:
 
-.SUFFIXES: .f .F .f90 .F90 .c .C .cxx .o
+.SUFFIXES: .f .F .f90 .F90 .c .C .cxx .o .cu
+
   
 %%.o : %%.f
 \t$(ECHO-COMPILING) 
@@ -145,15 +149,23 @@
 else
 \t-$(if $(wildcard $*.mod),if [ -w $*.mod -a -s $(shell echo $*|tr A-Z a-z).mod -a \( $(shell echo $*|tr A-Z a-z).mod -nt $*.mod \) ] ;then ln -f -v $(shell echo $*|tr A-Z a-z).mod $*.mod;else test -f $@ -a -s $@ -a -w $@&&touch $@||:;fi)
 endif
+
 %%.o : %%.c
 \t$(ECHO-COMPILING) 
 \t$(CCOMP) $(CFLAGS) $(CDEFINES) $<
+
 %%.o : %%.C
 \t$(ECHO-COMPILING) 
 \t$(CPPCOMP) $(CFLAGS) $(CDEFINES) $<
+
 %%.o : %%.cxx
 \t$(ECHO-COMPILING) 
 \t$(CPPCOMP) $(CFLAGS) $(CDEFINES) $<
+
+%%.o : %%.cu
+\t$(ECHO-COMPILING) 
+\t$(CUCOMP) $(CU_FLAGS) $(CDEFINES) $<
+
 %%API.c %%API.h %%API-bridges.F90: %%.int
 \t$(ECHO-PROCESSING) 
 \t./int2API.py $<
@@ -202,7 +214,7 @@
 Makefile.Depend: 
 \t@$(ECHO) Calculating dependencies
 \t./setup_depends.py %(dependFlags)s $(FFLAGS) $(F90FLAGS) $(CFLAGS) *.f *.f90 *.F90 *.F 
-\t./setup_addcdepends.py $(CFLAGS) *.c
+\t./setup_addcdepends.py $(CFLAGS) *.c *.cu
 
 .PHONY: reorder
 # run reorder script or kill the script if we dont need it
@@ -235,6 +247,7 @@
 
 %(includeList)s
 ALL_OBJ_FILES = \\%(includeMacros)s
+       $(CUDA_KERNEL_OBJS) \
        $(MISC_OBJS) 
 
 #Remove duplicate names (side effect is sorting)
@@ -253,7 +266,12 @@
 \t$(PRINT_F90_FLAGS)
 \t$(PRINT_C_FLAGS)
 \t$(PRINT_L_FLAGS)
+ifeq ("$(USECUDA)","1")
+\t$(PRINT_CU_FLAGS)
+\t$(PRINT_GPU_CXX_FLAGS)
+\t$(PRINT_GPU_L_FLAGS)
 endif
+endif
 \t@$(RM) $(PRINT_FLAGS_FILE)
 \t@$(RM) -f $(SUCCESS_FILE)
 \t@$(TOUCH) $(PRINT_FLAGS_FILE)
@@ -263,6 +281,14 @@
 \t@$(ECHO)  >> $(PRINT_FLAGS_FILE)
 \t$(PRINT_C_FLAGS) >> $(PRINT_FLAGS_FILE)
 \t@$(ECHO)  >> $(PRINT_FLAGS_FILE)
+ifeq ("$(USECUDA)","1")
+\t$(PRINT_CU_FLAGS) >> $(PRINT_FLAGS_FILE)
+\t@$(ECHO)  >> $(PRINT_FLAGS_FILE)
+\t$(PRINT_GPU_CXX_FLAGS) >> $(PRINT_FLAGS_FILE)
+\t@$(ECHO)  >> $(PRINT_FLAGS_FILE)
+\t$(PRINT_GPU_L_FLAGS) >> $(PRINT_FLAGS_FILE)
+\t@$(ECHO)  >> $(PRINT_FLAGS_FILE)
+endif
 \t$(PRINT_L_FLAGS) >> $(PRINT_FLAGS_FILE)
 \t@$(ECHO)  >> $(PRINT_FLAGS_FILE)
 \t@$(ECHO) Data Object files: >> $(PRINT_FLAGS_FILE)
@@ -273,11 +299,29 @@
 
 .PHONY: data code clean distclean
 
+ifeq ("$(USECUDA)","1")
+KernelCode.o: $(CUDA_KERNEL_OBJS)
+\t@$(ECHO) Linking in Devicecode to KernelCode.o
+\t$(GPU_LINK) $(GPU_L_FLAGS) $@ $^
+else
+KernelCode.o: $(CUDA_KERNEL_OBJS)
+\t@$(ECHO) Sorry, I am unable to build $@ mentioned in $(call MKFILES,$@).
+\t@$(ECHO) You may need to run ./setup with a flag for requesting CUDA support
+\t@$(ECHO) in order to use the $(call UNITNAMES,$(call MKFILES,$@)) unit.
+\t@$(RM) -f $(SUCCESS_FILE)
+\t@exit 1
+endif
+
+
 data: $(DATA_OBJS)
 
 code: data $(CODE_OBJS) 
 
+ifeq ("$(USECUDA)","0")
 $(EXE) : printflags reorder data code
+else ifeq ("$(USECUDA)","1")
+$(EXE) : printflags reorder KernelCode.o data code
+endif
 \t@$(ECHO) Linking in Units and Libraries 
 \t$(LINK) $(LFLAGS) $(EXE) $(ALL_OBJS) $(LIB) $(CONFIG_LIB)
 \t@$(ECHO) SUCCESS
Index: bin/parseCmd.py
===================================================================
--- bin/parseCmd.py	(.../tags/FLASH4.4_release)	(revision 25574)
+++ bin/parseCmd.py	(.../branches/flash4_lukat)	(revision 25574)
@@ -18,7 +18,7 @@
 WITHOUT_ARGS = ["auto","1d","2d","3d","portable",
                 "makehide", "curvilinear",
                 "opt","debug","test", "index-reorder", "strictparams",
-                "fbs","help", "noclobber", "nofbs"] # nofbs disables fixedBlockSize
+                "fbs","cuda","help", "noclobber", "nofbs"] # nofbs disables fixedBlockSize
 # options with arguments
 WITH_ARGS = ["maxblocks","nxb","nyb","nzb","verbose","site","ostype",
              "defines", "objdir","with-unit", "unit", "with-library",
@@ -47,6 +47,7 @@
             -defines=<defines> -unitsfile=<filename>
             -datafiles=<wildcard> -parfile=<filename>
             -fbs -nofbs -tau=<makefile>
+            -cuda
 
    (Misc Options)
             -makehide -noclobber -portable -help
@@ -241,6 +242,7 @@
         elif arg == '--debug':            GVars.buildFlag = "DEBUG"
         elif arg == '--test':             GVars.buildFlag = "TEST"
         elif arg == '--opt':              GVars.buildFlag = "OPT"
+        elif arg == '--cuda':             GVars.usecuda = 1
         elif arg == '--maxblocks':          GVars.maxblocks = int(val)
         elif arg == '--makehide':           GVars.makedisplay = 0
         elif arg == '--makefile':           GVars.makefileext = "."+val
@@ -436,6 +438,9 @@
     if GVars.nxb == None: 	 
        GVars.nxb = 8 	 
 
+    if GVars.usecuda == None:
+        GVars.usecuda =0
+
     defines = {}
     defines["N_DIM"] = GVars.dimension
     defines["MAXBLOCKS"] = GVars.maxblocks
Index: bin/globals.py
===================================================================
--- bin/globals.py	(.../tags/FLASH4.4_release)	(revision 25574)
+++ bin/globals.py	(.../branches/flash4_lukat)	(revision 25574)
@@ -81,7 +81,7 @@
 SHORTCUT_CHAR = "+"
 
 # for processing internal and external libraries
-COMPILERS = ('FFLAGS', 'CFLAGS', 'LFLAGS', 'LIB')
+COMPILERS = ('FFLAGS', 'CFLAGS', 'LFLAGS', 'CU_FLAGS','GPU_L_FLAGS','GPU_CFLAGS','LIB')
 DEFLTFLAG = 'OPT'
 
 # unit prefixes which setup knows about
@@ -226,6 +226,7 @@
     nxb          = None  # domain size
     nyb          = None
     nzb          = None
+    usecuda      = None
     build_site   = None 
     build_tau    = None
     build_os     = None
