My Community

Code: Select all

# Default precompiler options
CPP_OPTIONS = -DHOST=\"LinuxNV\" \
              -DMPI -DMPI_INPLACE -DMPI_BLOCK=8000 -Duse_collective \
              -DscaLAPACK \
              -DCACHE_SIZE=4000 \
              -Davoidalloc \
              -Dvasp6 \
              -Dtbdyn \
              -Dqd_emulate \
              -Dfock_dblbuf \
              -D_OPENMP \
              -DACC_OFFLOAD \
              -DNVCUDA \
              -DUSENCCL

CPP         = nvfortran -Mpreprocess -Mfree -Mextend -E $(CPP_OPTIONS) $*$(FUFFIX)  > $*$(SUFFIX)

# N.B.: you might need to change the cuda-version here
#       to one that comes with your NVIDIA-HPC SDK
CC          = mpicc  -acc -gpu=cc60,cc70,cc80,cc90,cuda12.5 -mp
FC          = mpif90 -acc -gpu=cc60,cc70,cc80,cc90,cuda12.5 -mp
FCL         = mpif90 -acc -gpu=cc60,cc70,cc80,cc90,cuda12.5 -mp -c++libs

FREE        = -Mfree

FFLAGS      = -Mbackslash -Mlarge_arrays

OFLAG       = -fast

DEBUG       = -Mfree -O0 -traceback

LLIBS       = -cudalib=cublas,cusolver,cufft,nccl -cuda

# Redefine the standard list of O1 and O2 objects
SOURCE_O1  := pade_fit.o minimax_dependence.o wave_window.o
SOURCE_O2  := pead.o

# For what used to be vasp.5.lib
CPP_LIB     = $(CPP)
FC_LIB      = $(FC)
CC_LIB      = $(CC)
CFLAGS_LIB  = -O -w
FFLAGS_LIB  = -O1 -Mfixed
FREE_LIB    = $(FREE)

OBJECTS_LIB = linpack_double.o

# For the parser library
CXX_PARS    = nvc++ --no_warnings

##
## Customize as of this point! Of course you may change the preceding
## part of this file as well if you like, but it should rarely be
## necessary ...
##
# When compiling on the target machine itself , change this to the
# relevant target when cross-compiling for another architecture
VASP_TARGET_CPU ?= -tp cascadelake
FFLAGS     += $(VASP_TARGET_CPU)

# Specify your NV HPC-SDK installation (mandatory)
#... first try to set it automatically
#NVROOT      =$(shell which nvfortran | awk -F /compilers/bin/nvfortran '{ print $$1 }')

# If the above fails, then NVROOT needs to be set manually
NVHPC      ?= /shared/EL9/explorer/nvidia-hpc-sdk/24.7
NVVERSION   = 24.7
NVROOT      = /shared/EL9/explorer/nvidia-hpc-sdk/24.7/Linux_x86_64/24.7

## Improves performance when using NV HPC-SDK >=21.11 and CUDA >11.2
OFLAG_IN   = -fast -Mwarperf
SOURCE_IN  := nonlr.o

# Software emulation of quadruple precsion (mandatory)
QD         ?= $(NVROOT)/compilers/extras/qd
LLIBS      += -L$(QD)/lib -lqdmod -lqd
INCS       += -I$(QD)/include/qd

# Intel MKL for FFTW, BLAS, LAPACK, and scaLAPACK
MKLROOT    ?= /shared/EL9/explorer/intel-oneapi/2025.0.1/mkl/2025.0
MKLLIBS     = -Mmkl
#MKLLIBS     = -lmkl_intel_lp64 -lmkl_pgi_thread -lmkl_core -pgf90libs -mp -lpthread -lm -ldl

# If you want to use scaLAPACK from MKL
LLIBS_MKL   = -L$(MKLROOT)/lib -lmkl_scalapack_lp64 -lmkl_blacs_openmpi_lp64 $(MKLLIBS)

# Use a separate scaLAPACK installation (optional but recommended in combination with OpenMPI)
# Comment out the two lines below if you want to use scaLAPACK from MKL instead
#SCALAPACK_ROOT ?= /path/to/your/scalapack/installation
#LLIBS_MKL   = -L$(SCALAPACK_ROOT)/lib -lscalapack $(MKLLIBS)

LLIBS      += $(LLIBS_MKL)

INCS       += -I$(MKLROOT)/include/fftw

# Use cusolvermp (optional)
# supported as of NVHPC-SDK 24.1 (and needs CUDA-11.8)
#CPP_OPTIONS+= -DCUSOLVERMP -DCUBLASMP
#LLIBS      += -cudalib=cusolvermp,cublasmp -lnvhpcwrapcal

# HDF5-support (optional but strongly recommended, and mandatory for some features)
#CPP_OPTIONS+= -DVASP_HDF5
#HDF5_ROOT  ?= /shared/EL9/explorer/HDF5/1.14.6
#LLIBS      += -L$(HDF5_ROOT)/lib -lhdf5_fortran
#INCS       += -I$(HDF5_ROOT)/include

# For the VASP-2-Wannier90 interface (optional)
CPP_OPTIONS    += -DVASP2WANNIER90
WANNIER90_ROOT ?= /projects/bansil/programs/WANNIER90/wannier90-3.1.0nvidiavaspinterfacenompi
LLIBS          += -L$(WANNIER90_ROOT) -lwannier

# For the fftlib library (hardly any benefit for the OpenACC GPU port, especially in combination with MKL's FFTs)
#CPP_OPTIONS+= -Dsysv
#FCL        += fftlib.o
#CXX_FFTLIB  = nvc++ -mp --no_warnings -std=c++11 -DFFTLIB_USE_MKL -DFFTLIB_THREADSAFE
#INCS_FFTLIB = -I./include -I$(MKLROOT)/include/fftw
#LIBS       += fftlib
#LLIBS      += -ldl

# For machine learning library vaspml (experimental)
#CPP_OPTIONS += -Dlibvaspml
#CPP_OPTIONS += -DVASPML_USE_CBLAS
#CPP_OPTIONS += -DVASPML_DEBUG_LEVEL=3
#CXX_ML      = mpic++ -mp
#CXXFLAGS_ML = -O3 -std=c++17 -Wall -Wextra
#INCLUDE_ML  =

# Add -gpu=tripcount:host to compiler commands for NV HPC-SDK > 25.1
#NVFORTRAN_VERSION := $(shell nvfortran --version | sed -n '2s/^nvfortran \([0-9.]*\).*/\1/p')
# define greater_or_equal
#$(shell printf '%s\n%s\n' '$(1)' '$(2)' | sort -V | head -n1 | grep -q '$(2)' && echo true || echo false)
#endef
#ifeq ($(call greater_or_equal,$(NVFORTRAN_VERSION),25.1),true)
#    CC  += -gpu=tripcount:host
#    FC  += -gpu=tripcount:host
#endif

Code: Select all

ar: creating libdmy.a
ar: creating libparser.a
NVFORTRAN-W-0006-Input file empty (crayhip.f90)
NVFORTRAN/x86-64 Linux 24.7-0: compilation completed with warnings
NVFORTRAN-W-0006-Input file empty (intelmkl.f90)
NVFORTRAN/x86-64 Linux 24.7-0: compilation completed with warnings
NVFORTRAN-W-0006-Input file empty (vaspml.f90)
NVFORTRAN/x86-64 Linux 24.7-0: compilation completed with warnings
NVFORTRAN-W-0155-Mismatched data type for member kvector (bandgap_tools.F: 394)
NVFORTRAN-W-0155-Mismatched data type for member kvector (bandgap_tools.F: 401)
  0 inform,   2 warnings,   0 severes, 0 fatal for find_band_edges_current_kpoint
NVFORTRAN-W-0921-Redefinition of symbol RANK_SUFFIX (./ml_reader.inc: 5)
/usr/bin/ld: /shared/EL9/explorer/intel-oneapi/2025.0.1/mkl/2025.0/lib/libmkl_intel_thread.so: undefined reference to `__kmpc_end_masked'
/usr/bin/ld: /shared/EL9/explorer/intel-oneapi/2025.0.1/mkl/2025.0/lib/libmkl_intel_thread.so: undefined reference to `__kmpc_masked'
pgacclnk: child process exit status 1: /usr/bin/ld
make[2]: *** [makefile:153: vasp] Error 2
cp: cannot stat 'vasp': No such file or directory
make[1]: *** [makefile:150: all] Error 1
make: *** [makefile:17: std] Error 2

Code: Select all

nvfortran -Mpreprocess -Mfree -Mextend -E -DHOST=\"LinuxNV\" -DMPI -DMPI_INPLACE -DMPI_BLOCK=8000 -Duse_collective -DscaLAPACK -DCACHE_SIZE=4000 -Davoidalloc -Dvasp6 -Dtbdyn -Dqd_emulate -Dfock_dblbuf -D_OPENMP -DACC_OFFLOAD -DNVCUDA -DUSENCCL -DVASP2WANNIER90 rpa_high.F > rpa_high.f90 -DNGZhalf
mpif90 -acc -gpu=cc60,cc70,cc80,cc90,cuda12.5 -mp -Mfree -Mbackslash -Mlarge_arrays -tp cascadelake -fast -I/shared/EL9/explorer/nvidia-hpc-sdk/24.7/Linux_x86_64/24.7/compilers/extras/qd/include/qd -I/shared/EL9/explorer/intel-oneapi/2025.0.1/mkl/2025.0/include/fftw  -c rpa_high.f90
nvfortran -Mpreprocess -Mfree -Mextend -E -DHOST=\"LinuxNV\" -DMPI -DMPI_INPLACE -DMPI_BLOCK=8000 -Duse_collective -DscaLAPACK -DCACHE_SIZE=4000 -Davoidalloc -Dvasp6 -Dtbdyn -Dqd_emulate -Dfock_dblbuf -D_OPENMP -DACC_OFFLOAD -DNVCUDA -DUSENCCL -DVASP2WANNIER90 main.F > main.f90 -DNGZhalf
mpif90 -acc -gpu=cc60,cc70,cc80,cc90,cuda12.5 -mp -Mfree -Mbackslash -Mlarge_arrays -tp cascadelake -Mfree -O0 -traceback -I/shared/EL9/explorer/nvidia-hpc-sdk/24.7/Linux_x86_64/24.7/compilers/extras/qd/include/qd -I/shared/EL9/explorer/intel-oneapi/2025.0.1/mkl/2025.0/include/fftw  -c main.f90
mpif90 -acc -gpu=cc60,cc70,cc80,cc90,cuda12.5 -mp -c++libs -o vasp c2f_interface.o simd.o base.o string.o tutor.o version.o build_info.o command_line.o vhdf5_base.o incar_reader.o reader_base.o openmp_struct.o openacc_struct.o offload_struct.o mpi.o mpi_shmem.o main_mpi.o mathtools.o profiling.o bse_struct.o mgrid_struct.o pot_struct.o hamil_struct.o radial_struct.o pseudo_struct.o wave_struct.o nl_struct.o mkpoints_struct.o bandgap_struct.o poscar_struct.o esf_struct.o afqmc_struct.o minimax_struct.o setex_struct.o locproj_struct.o fock_glb.o chi_glb.o smart_allocate.o xml.o constant.o plugins.o ml_ff_c2f_interface.o ml_ff_prec.o ml_ff_string.o ml_ff_tutor.o ml_ff_constant.o ml_ff_mpi_help.o ml_ff_neighbor.o ml_ff_taglist.o ml_ff_struct.o ml_ff_mpi_shmem.o vdwforcefield_glb.o jacobi.o scala_struct.o ini.o scala.o nvcuda.o crayhip.o intelmkl.o openmp.o openacc.o offload.o scalapack_wrappers.o blas_wrappers.o lapack_wrappers.o asa.o lattice.o poscar.o fft_comm.o fftw.o fft_wrappers.o fft_base.o mgrid.o libmbd.o ml_asa2.o ml_ff_mpi.o ml_ff_helper.o ml_ff_logfile.o ml_ff_math.o ml_ff_iohandle.o ml_ff_memory.o ml_ff_abinitio.o ml_ff_ff2.o ml_ff_ff3.o ml_ff_ff.o ml_ff_mlff.o vaspml.o ldalib.o wpbe.o ggalib.o mbj.o mggalib.o vdw_nl.o xc_driver.o setex.o pseudo.o radial.o gridq.o coulomb_cutoff.o ebs.o symlib.o gauss_quad.o m_unirnk.o mkpoints.o random.o wave.o wave_mpi.o wave_high.o bext.o spinsym.o symmetry.o lattlib.o nonl.o nonlr.o nonl_high.o dfast.o choleski2.o mix.o hamil.o constrmag.o cl_shift.o relativistic.o LDApU.o paw_base.o tau_mu.o fexcg.o egrad.o pawsym.o pawfock.o pawlhf.o diis.o rhfatm.o hyperfine.o fock_ace.o mkpoints_full.o charge.o us.o extpot.o paw.o Lebedev-Laikov.o stockholder.o pot_electrostat.o dipol.o solvation.o scpc.o fermi_energy.o tet.o dos.o elf.o hamil_rot.o chain.o dyna.o fileio.o vhdf5.o bandgap_tools.o pot.o sphpro.o core_rel.o aedens.o wavpre.o wavpre_noio.o broyden.o dynbr.o reader.o writer.o xml_writer.o brent.o stufak.o opergrid.o stepver.o fast_aug.o fock_multipole.o fock.o fock_dbl.o fock_frc.o supercell.o mkpoints_change.o subrot_cluster.o sym_grad.o mymath.o npt_dynamics.o subdftd3.o subdftd4.o internals.o dynconstr.o dimer_heyden.o dvvtrajectory.o vdwforcefield.o nmr.o pead.o k-proj.o subrot.o subrot_scf.o paircorrection.o rpa_force.o ml_reader.o ml_interface_writer.o ml_interface.o coulomb_cutoff_gradients.o force.o pwlhf.o gw_model.o optreal.o steep.o rmm-diis.o davidson.o david_full.o david_inner.o root_find.o lcao_bare.o locproj.o electron_common.o electron.o rot.o electron_all.o shm.o pardens.o optics.o constr_cell_relax.o stm.o finite_diff.o elpol.o hamil_lr.o rmm-diis_lr.o subrot_lr.o lr_helper.o hamil_lrf.o elinear_response.o ilinear_response.o linear_optics.o setlocalpp.o wannier.o electron_OEP.o electron_lhf.o twoelectron4o.o minimax_ini.o minimax_dependence.o minimax_functions1D.o minimax_functions2D.o minimax_varpro.o minimax.o umco.o mlwf.o ratpol.o pade_fit.o screened_2e.o wave_cacher.o crpa.o chi_base.o wpot.o local_field.o ump2.o ump2kpar.o fcidump.o ump2no.o bse_te.o bse_lanczos.o bse.o bse_driver.o time_propagation.o esf.o acfdt.o afqmc.o rpax.o chi.o dmft.o GG_base.o acfdt_GG.o greens_orbital.o lt_mp2.o rnd_orb_mp2.o greens_real_space.o chi_GG.o chi_super.o sydmat.o rmm-diis_mlr.o linear_response_NMR.o wannier_interpol.o wave_interpolate.o wave_rotator.o wave_window.o wap.o elphon_potential_struct.o elphon_base.o elphon_triplets.o elphon_potential.o elphon_accumulators.o elphon_kgrid.o transport.o elphon_common.o elphon_mels.o elphon_selfen_ph.o elphon_driver.o linear_response.o auger.o dmatrix.o phonon.o elphon_derivative.o wannier_mats.o elphon.o core_con_mat.o embed.o rpa_high.o  main.o  -Llib -ldmy -Lparser -lparser -cudalib=cublas,cusolver,cufft,nccl -cuda -L/shared/EL9/explorer/nvidia-hpc-sdk/24.7/Linux_x86_64/24.7/compilers/extras/qd/lib -lqdmod -lqd -L/shared/EL9/explorer/intel-oneapi/2025.0.1/mkl/2025.0/lib -lmkl_scalapack_lp64 -lmkl_blacs_openmpi_lp64 -Mmkl -L/projects/bansil/programs/WANNIER90/wannier90-3.1.0nvidiavaspinterfacenompi -lwannier
make[2]: Leaving directory '/projects/bansil/programs/VASP651gpuexplorer/vasp.6.5.1/build/std'
make[1]: Leaving directory '/projects/bansil/programs/VASP651gpuexplorer/vasp.6.5.1/build/std'

Code: Select all

MKLLIBS     = -Mmkl
#MKLLIBS     = -lmkl_intel_lp64 -lmkl_pgi_thread -lmkl_core -pgf90libs -mp -lpthread -lm -ldl

Code: Select all

#MKLLIBS     = -Mmkl
MKLLIBS     = -lmkl_intel_lp64 -lmkl_pgi_thread -lmkl_core -pgf90libs -mp -lpthread -lm -ldl