From 69445742d2428d42944b178e3b93a966390b91fe Mon Sep 17 00:00:00 2001 From: Ming Zeng Date: Wed, 5 Aug 2020 13:47:00 +0200 Subject: [PATCH 1/3] Add makefile for Maxwell (cluster of DESY). --- source/make.MAXWELL | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 source/make.MAXWELL diff --git a/source/make.MAXWELL b/source/make.MAXWELL new file mode 100644 index 0000000..291bfc4 --- /dev/null +++ b/source/make.MAXWELL @@ -0,0 +1,41 @@ +JSONFORTRAN_HOME=/home/zming/src/git/json-fortran/lib +JSON_LIB=$(JSONFORTRAN_HOME) +JSON_INC=$(JSONFORTRAN_HOME)/mod + +# name of the compiler +FC_MAXWELL = mpif90 -c -I$(JSON_INC) -fopenmp +CC_MAXWELL = mpicc -c +LINKER_MAXWELL = mpif90 -fopenmp -O3 + +OPTS_DBL_MAXWELL = -O3 -r8 +OPTS_SGL_MAXWELL = -O3 + +FORMAT_FREE_MAXWELL = -stand f03 -FR -Tf +FORMAT_FIXED_MAXWELL = -FI + +# hdf5 libraries +#HDF5_HOME = /opt/hdf5/hdf5-1.10.6 +#HDF5_HOME = /data/netapp/fla/plasma/software-max/hdf5-1.12.0/intel.16.0.3.openmpi.1.10.3 +HDF5_HOME = /software/openmpi-intel/3.1.6/hdf5-1.10.6 +HDF5_LIB = $(HDF5_HOME)/lib +HDF5_INC =$(HDF5_HOME)/include + +HDF_LIBPATH_MAXWELL= -L$(HDF5_LIB) \ + -L/data/netapp/fla/plasma/software-max/szip-2.1/lib \ + -L/usr/lib64 \ + -lhdf5_fortran -lhdf5_hl -lhdf5 -lz -lsz +HDF_INCLUDE_PATH_MAXWELL= -I$(HDF5_INC) -I${HDF5_LIB} \ + -I/data/netapp/fla/plasma/software-max/szip-2.1/include \ + -lhdf5hl_fortran -lhdf5_fortran -lhdf5_hl -lhdf5 + +# other libs +OTHER_LIBS_MAXWELL = -L$(JSON_LIB) -ljsonfortran + +# memory options +MEM_OPTIONS_MAXWELL = + +# options for linking +# use static libs +STATIC_LINK_MAXWELL = +# use shared libs +SHARED_LINK_MAXWELL = From 6ca31802b10a7b51836ed2295ee4467641fc2797 Mon Sep 17 00:00:00 2001 From: Ming Zeng Date: Thu, 6 Aug 2020 19:49:10 +0200 Subject: [PATCH 2/3] Implement beam profile 4, read beam from a particle raw h5 file from OSIRIS or QuickPIC output. --- source/fdist3d_class.f03 | 172 +++++++++++++++++++++++++++++++++++- source/hdf5io_class.f03 | 82 ++++++++++++++++- source/simulation_class.f03 | 7 +- 3 files changed, 253 insertions(+), 8 deletions(-) diff --git a/source/fdist3d_class.f03 b/source/fdist3d_class.f03 index b3ac45f..00fab79 100644 --- a/source/fdist3d_class.f03 +++ b/source/fdist3d_class.f03 @@ -1,5 +1,5 @@ ! fdist3d class for QuickPIC Open Source 1.0 -! update: 04/18/2016 +! update: 08/06/2020 module fdist3d_class @@ -9,13 +9,14 @@ module fdist3d_class use ufield3d_class use part3d_lib use input_class + use hdf5io_class implicit none private public :: fdist3d, fdist3d_000, fdist3d_001, fdist3d_002, fdist3d_100 - public :: fdist3d_003 + public :: fdist3d_003, fdist3d_004 type, abstract :: fdist3d @@ -117,7 +118,7 @@ end subroutine ab_init_fdist3d end type fdist3d_002 ! type, extends(fdist3d) :: fdist3d_003 -! Read external particle data files +! Read external particle data files in text format private integer :: npt @@ -129,6 +130,26 @@ end subroutine ab_init_fdist3d procedure, private :: dist3d => dist3d_003 end type fdist3d_003 +! + type, extends(fdist3d) :: fdist3d_004 +! Read external particle data files in a h5 file of OSIRIS or QuickPIC particle raw data + private + + real :: offset_x, offset_y, offset_z, dx, dy, dz, box_min_x, box_min_y, box_min_z +! format can be 0 (OSIRIS, 1,2,3 correspond to z,x,y) or 1 (QuickPIC, 1,2,3 correspond to x,y,z) + integer :: format +! The actual number of particles of a macro particle is (cell_volumn*n0_per_cc*1e6)*q +! which should be constant between codes, thus +! q_ratio = (cell_volumn_in_old_sim*n0_in_old_sim)/(cell_volumn_in_new_sim*n0_in_new_sim) +! /raw_sampling_ratio. + real :: q_ratio + character(len=:), allocatable :: file + + contains + procedure, private :: init_fdist3d => init_fdist3d_004 + procedure, private :: dist3d => dist3d_004 + + end type fdist3d_004 ! type, extends(fdist3d) :: fdist3d_100 ! Ring profile @@ -877,6 +898,149 @@ subroutine dist3d_003(this,part3d,npp,fd) call this%err%werrfl2(class//sname//' ended') end subroutine dist3d_003 +! + subroutine init_fdist3d_004(this,input,i) + + implicit none + + class(fdist3d_004), intent(inout) :: this + type(input_json), intent(inout), pointer :: input + integer, intent(in) :: i +! local data + real :: max + real :: alx, aly, alz + integer :: indx, indy, indz + character(len=20) :: sn,s1 + character(len=18), save :: sname = 'init_fdist3d_004:' + + this%sp => input%sp + this%err => input%err + this%p => input%pp + + call this%err%werrfl2(class//sname//' started') + + write (sn,'(I3.3)') i + s1 = 'beam('//trim(sn)//')' + + call input%get('simulation.indx',indx) + call input%get('simulation.indy',indy) + call input%get('simulation.indz',indz) + call input%get('simulation.box.x(1)',this%box_min_x) + call input%get('simulation.box.x(2)',max) + call input%get(trim(s1)//'.offset(1)',this%offset_x) + alx = (max-this%box_min_x) + this%dx=alx/real(2**indx) + call input%get('simulation.box.y(1)',this%box_min_y) + call input%get('simulation.box.y(2)',max) + call input%get(trim(s1)//'.offset(2)',this%offset_y) + aly = (max-this%box_min_y) + this%dy=aly/real(2**indy) + call input%get('simulation.box.z(1)',this%box_min_z) + call input%get('simulation.box.z(2)',max) + call input%get(trim(s1)//'.offset(3)',this%offset_z) + alz = (max-this%box_min_z) + this%dz=alz/real(2**indz) + + call input%get(trim(s1)//'.profile', this%npf) + call input%get(trim(s1)//'.npmax', this%npmax) + call input%get(trim(s1)//'.evolution', this%evol) + call input%get(trim(s1)//'.file_name', this%file) + call input%get(trim(s1)//'.format', this%format) + call input%get(trim(s1)//'.q_ratio', this%q_ratio) + + + call this%err%werrfl2(class//sname//' ended') + + end subroutine init_fdist3d_004 +! + subroutine dist3d_004(this,part3d,npp,fd) + + implicit none + + class(fdist3d_004), intent(inout) :: this + real, dimension(:,:), pointer, intent(inout) :: part3d + integer, intent(inout) :: npp + class(ufield3d), intent(in), pointer :: fd +! local data1 + +! edges(1) = lower boundary in y of particle partition +! edges(2) = upper boundary in y of particle partition +! edges(3) = lower boundary in z of particle partition +! edges(4) = upper boundary in z of particle partition + real, dimension(:,:), pointer :: part + integer :: np_h5, nx, ny, nz, i + real, dimension(4) :: edges + integer, dimension(2) :: noff + integer :: np_count + integer :: ierr + character(len=18), save :: sname = 'dist3d_004:' + real, dimension(:), pointer :: x, y, z, ux, uy, uz, q + + call this%err%werrfl2(class//sname//' started') + + ierr = 0 + nx = fd%getnd1(); ny = fd%getnd2(); nz = fd%getnd3() + part => part3d + noff = fd%getnoff() + edges(1) = noff(1); edges(3) = noff(2) + edges(2) = edges(1) + fd%getnd2p() + edges(4) = edges(3) + fd%getnd3p() + + if (0==this%format) then +! OSIRIS particle raw format + call read_particle_raw(trim(this%file), np_h5, z, x, y, uz, ux, uy, q, ierr) + do i = 1, np_h5 + x(i) = (x(i) + this%offset_x - this%box_min_x)/this%dx + y(i) = (y(i) + this%offset_y - this%box_min_y)/this%dy + z(i) = (this%offset_z - z(i) - this%box_min_z)/this%dz + end do + else +! QuickPIC particle raw format + call read_particle_raw(trim(this%file), np_h5, x, y, z, ux, uy, uz, q, ierr) + do i = 1, np_h5 + x(i) = (x(i) + this%offset_x)/this%dx + y(i) = (y(i) + this%offset_y)/this%dy + z(i) = (z(i) + this%offset_z)/this%dz + end do + end if + + if (np_h5>this%npmax) then + print *, "Warning! Number of particles in the h5 file is ", np_h5, "," + print *, "larger than npmax ", this%npmax, "." + print *, "This may cause overflow." + end if + np_count = 1 + do i = 1, np_h5 + if ((x(i)>=1) .and. x(i)<=(nx-1) .and. (y(i) >= edges(1))& + & .and. (y(i) < edges(2)) .and. (z(i) >= edges(3)) .and. (z(i) < edges(4))) then + part(1, np_count) = x(i) + part(2, np_count) = y(i) + part(3, np_count) = z(i) + part(4, np_count) = ux(i) + part(5, np_count) = uy(i) + part(6, np_count) = uz(i) + part(7, np_count) = q(i)*this%q_ratio + np_count = np_count + 1 + end if + end do + + deallocate(x) + deallocate(y) + deallocate(z) + deallocate(ux) + deallocate(uy) + deallocate(uz) + deallocate(q) + + npp = np_count - 1 + if (ierr /= 0) then + write (erstr,*) 'Read beam raw h5 file error' + call this%err%equit(class//sname//erstr) + endif + + call this%err%werrfl2(class//sname//' ended') + + end subroutine dist3d_004 ! subroutine init_fdist3d_100(this,input,i) @@ -1067,4 +1231,4 @@ subroutine dist3d_100(this,part3d,npp,fd) end subroutine dist3d_100 ! - end module fdist3d_class \ No newline at end of file + end module fdist3d_class diff --git a/source/hdf5io_class.f03 b/source/hdf5io_class.f03 index 3ae15b1..e516acb 100644 --- a/source/hdf5io_class.f03 +++ b/source/hdf5io_class.f03 @@ -1,5 +1,5 @@ ! hdf5io module for QuickPIC Open Source 1.0 -! update: 04/18/2016 +! update: 08/06/2020 module hdf5io_class @@ -14,6 +14,7 @@ module hdf5io_class public :: hdf5file, pwfield, pwfield_pipe, wfield_pipe, pwpart_pipe, pwpart,& &wpart,rpart + public :: read_h5_dataset, read_particle_raw type hdf5file @@ -66,7 +67,11 @@ module hdf5io_class interface pwpart module procedure pwpart_2d end interface - + + interface read_h5_dataset + module procedure read_h5_dataset_real + end interface + contains subroutine init_hdf5file(this,filename,timeunits,ty,n,t,dt,axisname,& @@ -1419,6 +1424,79 @@ subroutine rpart(pp,perr,file,part,npp,ierr) end if end subroutine rpart +! + subroutine read_h5_dataset_real( parentID, name, buf, ndims, dims ) + !--------------------------------------------------------------------------- + ! Determine the size of a dataset, allocate the same size and read dataset to a 1D array buf. + ! Also allocate a 1d array dims with the size of ndims, and save the dimensions of the dataset to dims. + ! After calling this subroutine, one may optinal reshape buf to the same shape as dataset + ! Reshape reference: https://software.intel.com/en-us/forums/intel-fortran-compiler/topic/269622 + implicit none + + integer(hid_t), intent(in) :: parentID + character( len = * ), intent(in) :: name + + real, dimension(:), pointer, intent(out) :: buf + integer, intent(out) :: ndims + integer(hsize_t), dimension(:), pointer, intent(out) :: dims + + integer(hsize_t), dimension(:), pointer :: maxdims + integer(hsize_t) :: npoints + integer(hid_t) :: dataspaceID, datasetID + integer :: ierr + integer(hid_t) :: treal + + treal = detect_precision() + + call h5dopen_f(parentID, name, datasetID, ierr) + call h5dget_space_f(datasetID, dataspaceID, ierr) + call h5sget_simple_extent_ndims_f(dataspaceID, ndims, ierr) + allocate(dims(ndims), maxdims(ndims)) + call h5sget_simple_extent_dims_f(dataspaceID, dims, maxdims, ierr) + deallocate(maxdims) + call h5sget_simple_extent_npoints_f(dataspaceID, npoints, ierr) + allocate(buf(npoints)) + call h5dread_f(datasetID, treal, buf, dims, ierr) + ! close the dataset + call h5sclose_f( dataspaceID, ierr ) + call h5dclose_f( datasetID, ierr ) + end subroutine read_h5_dataset_real +! + subroutine read_particle_raw(filename, n_particles, x1, x2, x3, p1, p2, p3, q, ierr) + ! Read macro particle raw data file for beam initialization. File should be a h5 file in OSIRIS format or QuickPIC format. + implicit none + + character( len = * ), intent(in) :: filename + integer, intent(out) :: n_particles + real, dimension(:), intent(out), pointer :: x1, x2, x3, p1, p2, p3, q + + integer(hid_t) :: treal + integer(hid_t) :: file_id + integer :: ndims + integer(hsize_t), dimension(:), pointer :: dims + integer, intent(out) :: ierr + + call h5open_f(ierr) + call h5fopen_f(filename, H5F_ACC_RDONLY_F, file_id, ierr) + call read_h5_dataset( file_id, 'x1', x1, ndims, dims ) + deallocate(dims) + call read_h5_dataset( file_id, 'x2', x2, ndims, dims ) + deallocate(dims) + call read_h5_dataset( file_id, 'x3', x3, ndims, dims ) + deallocate(dims) + call read_h5_dataset( file_id, 'p1', p1, ndims, dims ) + deallocate(dims) + call read_h5_dataset( file_id, 'p2', p2, ndims, dims ) + deallocate(dims) + call read_h5_dataset( file_id, 'p3', p3, ndims, dims ) + deallocate(dims) + call read_h5_dataset( file_id, 'q', q, ndims, dims ) + n_particles = dims(1) + deallocate(dims) + call h5fclose_f(file_id, ierr) + call h5close_f(ierr) + + end subroutine read_particle_raw ! function detect_precision() integer(hid_t) :: detect_precision diff --git a/source/simulation_class.f03 b/source/simulation_class.f03 index 993408c..f6f48e8 100644 --- a/source/simulation_class.f03 +++ b/source/simulation_class.f03 @@ -1,5 +1,5 @@ ! simulation module for QuickPIC Open Source 1.0 -! update: 01/09/2018 +! update: 08/06/2020 module simulation_class @@ -420,6 +420,9 @@ subroutine init_sim_beams(this,input) case (3) allocate(fdist3d_003::this%pf(i)%p) call this%pf(i)%p%new(input,i) + case (4) + allocate(fdist3d_004::this%pf(i)%p) + call this%pf(i)%p%new(input,i) case (100) allocate(fdist3d_100::this%pf(i)%p) call this%pf(i)%p%new(input,i) @@ -1317,4 +1320,4 @@ function ntag() end function ntag ! - end module simulation_class \ No newline at end of file + end module simulation_class From adfcf97b53500e2b98132ca53c64d3e48dbef039 Mon Sep 17 00:00:00 2001 From: Ming Zeng Date: Mon, 19 Jul 2021 14:31:55 +0800 Subject: [PATCH 3/3] Fix crashing on some supercomputers. Also add make.TIANHE. --- source/fpois2d_lib.f03 | 2 +- source/make.TIANHE | 35 +++++++++++++++++++++++++++++++++++ source/part2d_class.f03 | 2 +- 3 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 source/make.TIANHE diff --git a/source/fpois2d_lib.f03 b/source/fpois2d_lib.f03 index 177b316..db8fabd 100644 --- a/source/fpois2d_lib.f03 +++ b/source/fpois2d_lib.f03 @@ -79,7 +79,7 @@ subroutine PBPOISD22N_QP(cu,dcu,amu,bxy,bz,isign,ffd,ax,ay,affp& real, dimension(3,nyv,kxp2+1,j2blok), intent(inout) :: cu, amu real, dimension(nyv,kxp2+1,j2blok), intent(inout) :: bz complex, dimension(nyd,kxp2,j2blok), intent(in) :: ffd - end subroutine + end subroutine end interface ! interface diff --git a/source/make.TIANHE b/source/make.TIANHE new file mode 100644 index 0000000..cd034a3 --- /dev/null +++ b/source/make.TIANHE @@ -0,0 +1,35 @@ +JSON_INC=/PARA2/grid305/git/json-fortran/lib/mod +JSON_LIB=/PARA2/grid305/git/json-fortran/lib + +FC_TIANHE = mpif90 -c -I$(JSON_INC) -fopenmp +CC_TIANHE = mpicc -c +LINKER_TIANHE = mpif90 -fopenmp -O3 + +OPTS_DBL_TIANHE = -O3 -r8 +OPTS_SGL_TIANHE = -O3 + +FORMAT_FREE_TIANHE = -stand f03 -FR -Tf +FORMAT_FIXED_TIANHE = -FI + +# hdf5 libraries + +#HDF5_LIB =/WORK/app/hdf5/1.10.6-gcc-5.4.0/lib +#HDF5_INC =/WORK/app/hdf5/1.10.6-gcc-5.4.0/include +HDF5_ROOT = /WORK/app/hdf5/1.8.21-icc-18 +HDF5_LIB =$(HDF5_ROOT)/lib +HDF5_INC =$(HDF5_ROOT)/include + +HDF_LIBPATH_TIANHE= -L$(HDF5_LIB) -lhdf5_fortran -lhdf5_hl -lhdf5 -lz +HDF_INCLUDE_PATH_TIANHE= -I$(HDF5_INC) -I${HDF5_LIB} -lhdf5hl_fortran -lhdf5_fortran -lhdf5_hl -lhdf5 + +# other libs +OTHER_LIBS_TIANHE = -L$(JSON_LIB) -ljsonfortran + +# memory options +MEM_OPTIONS_TIANHE = + +# options for linking +# use static libs +STATIC_LINK_TIANHE = +# use shared libs +SHARED_LINK_TIANHE = diff --git a/source/part2d_class.f03 b/source/part2d_class.f03 index bcbc286..bddfaa5 100755 --- a/source/part2d_class.f03 +++ b/source/part2d_class.f03 @@ -64,7 +64,7 @@ module part2d_class integer, dimension(:,:), pointer :: ncl => null() integer, dimension(:,:,:), pointer :: ihole => null() integer, dimension(:), pointer :: kpic => null() - real :: xtras = 0.2 + real :: xtras = 4.0 logical :: list = .true. contains