Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update cuda version for doppio to use cuda 10.0 #141

Open
wants to merge 4 commits into
base: gpu
Choose a base branch
from
Open

update cuda version for doppio to use cuda 10.0 #141

Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
308cf7d
update cuda version for doppio to use cuda 10.0
xinshengqin Feb 7, 2020
3422408
use environment variable CUDA_VERSION to set which version of CUDA to…
xinshengqin Feb 7, 2020
2bc51c7
Let Makefile.common.GPU use bash environment variables CUDA_VERSION a…
xinshengqin Feb 11, 2020
b02122e
update Makefile.common.GPU to use CUDA-relevant variables from a loca…
xinshengqin Mar 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 12 additions & 71 deletions src/Makefile.common.GPU
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,39 +1,15 @@
# CPP compiler
CC ?= g++

# for using some pre-set compiler options on specific machines
# you can add your own here
host_name := $(shell hostname -f)
ifeq ($(findstring titan, $(host_name)), titan)
machine_name := titan
else ifeq ($(findstring doppio, $(host_name)), doppio)
machine_name := doppio
else ifeq ($(findstring rhea, $(host_name)), rhea)
machine_name := rhea
endif


# if FC is set to pgi, we use pgi compiler
ifeq ($(FC),pgi)
ifeq ($(machine_name), titan)
CLAW_FC ?= ftn
else ifeq ($(machine_name), rhea)
CLAW_FC ?= pgfortran
else
CLAW_FC ?= pgfortran
endif
CLAW_FC ?= pgfortran
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Usually FC is set to the fortran compiler command. Here you seem to be setting it to pgi and then using that to set CLAW_FC to the compiler command. Why not just set FC ?= pgfortran in the application makefile, which would be more consistent with other usage?

I see from the old version that on some machines the pgi compiler name is ftn but when running on such a machine the user should just set FC to ftn?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I was just following what's done in regular Makefile.common where we have FC and then CLAW_FC ?= FC. Then in order to stress that you need a special GPU-enable pgi compiler, I set it to pgi to distinguish it from regular pgfortran. "ftn" is just an alias to pgfortran on one of the machine I have used.

I can change this if that's better. I don't fully understand why do we have both FC and CLAW_FC though. Why not just FC?

else
CLAW_FC ?= $(FC)
endif
# if CC is set to pgi, we use pgi compiler
ifeq ($(CC),pgi)
ifeq ($(machine_name), titan)
CLAW_CC ?= CC
else ifeq ($(machine_name), rhea)
CLAW_CC ?= pgc++
else
CLAW_CC ?= pgc++
endif
CLAW_CC ?= pgc++
else
CLAW_CC ?= $(CC)
endif
Expand Down Expand Up @@ -128,7 +104,7 @@ ALL_INCLUDE += $(addprefix -I,$(INCLUDE))
ALL_INCLUDE += $(addprefix -I,$(MODULE_PATHS)) $(addprefix -L,$(LIB_PATHS))
ifeq ($(USE_CUDA), TRUE)
ifndef CUDA_PATH
$(error CUDA_PATH Environment variable is undefined)
$(error CUDA_PATH is not defined)
endif
ALL_INCLUDE += -I$(CUDA_PATH)/include
ALL_LIBRARIES += -L$(CUDA_PATH)/lib64
Expand All @@ -146,28 +122,9 @@ ifeq ($(USE_CUDA),TRUE)
$(error You must set FC to pgi to use CUDA)
endif
ALL_DEFINES += -DCUDA
ifeq ($(machine_name), titan)
CUDA_VERSION ?= 9.1
CUDA_CC ?= cc35
else ifeq ($(machine_name), doppio)
CUDA_VERSION ?= 10.0
CUDA_CC ?= cc60
else ifeq ($(machine_name), rhea)
CUDA_VERSION ?= 8.0
CUDA_CC ?= cc35
else # default compilation flags if the machine is none of the above
CUDA_VERSION ?= 10.0
endif
ifeq ($(origin CUDA_CC),undefined) # check if CUDA_CC is defined in bash environment
# don't specify compute capability if CUDA_CC is not set
ALL_FFLAGS += -Mcuda=cuda$(CUDA_VERSION),maxregcount:80 -Mcuda=lineinfo -Mcuda=rdc
ALL_LFLAGS += -Mcuda=cuda$(CUDA_VERSION),maxregcount:80 -Mcuda=lineinfo -Mcuda=rdc
ALL_CPPFLAGS += -Mcuda=cuda$(CUDA_VERSION),maxregcount:80 -Mcuda=lineinfo -Mcuda=rdc
else
ALL_FFLAGS += -Mcuda=cuda$(CUDA_VERSION),$(CUDA_CC),maxregcount:80 -Mcuda=lineinfo -Mcuda=rdc
ALL_LFLAGS += -Mcuda=cuda$(CUDA_VERSION),$(CUDA_CC),maxregcount:80 -Mcuda=lineinfo -Mcuda=rdc
ALL_CPPFLAGS += -Mcuda=cuda$(CUDA_VERSION),$(CUDA_CC),maxregcount:80 -Mcuda=lineinfo -Mcuda=rdc
endif
ALL_FFLAGS += -Mcuda=cuda$(CUDA_VERSION),$(CUDA_CC),maxregcount:80 -Mcuda=lineinfo -Mcuda=rdc
ALL_LFLAGS += -Mcuda=cuda$(CUDA_VERSION),$(CUDA_CC),maxregcount:80 -Mcuda=lineinfo -Mcuda=rdc
ALL_CPPFLAGS += -Mcuda=cuda$(CUDA_VERSION),$(CUDA_CC),maxregcount:80 -Mcuda=lineinfo -Mcuda=rdc
ALL_LIBRARIES += -lcudafor -lcuda
ifeq ($(VERBOSE), TRUE)
ALL_FFLAGS += -Mcuda=ptxinfo
Expand Down Expand Up @@ -257,7 +214,7 @@ else ifeq ($(CLAW_FC),ifort)
MODULE_FLAG = -module
OMP_FLAG = -openmp
# TODO: add flags for the intel compiler
else ifeq ($(CLAW_FC),$(filter $(CLAW_FC),pgfortran ftn)) # CLAW_FC is either pgfortran or ftn
else ifeq ($(CLAW_FC),pgfortran)
# Note that there shoud be a space after this flag
MODULE_FLAG = -module
ifeq ($(DEBUG),TRUE)
Expand All @@ -273,9 +230,6 @@ else ifeq ($(CLAW_FC),$(filter $(CLAW_FC),pgfortran ftn)) # CLAW_FC is either pg
ifeq ($(VERBOSE), TRUE)
ALL_FFLAGS += -Manno -Mkeepasm
endif
ifeq ($(machine_name), rhea)
ALL_LIBRARIES += -L/ccs/compilers/gcc/rhel6-x86_64/5.3.0/lib64
endif
else
$(error Unrecognized compiler: $(CLAW_FC))
endif
Expand Down Expand Up @@ -304,19 +258,11 @@ ifeq ($(USE_CUDA), TRUE)
ifneq ($(DEBUG),TRUE)
ALL_NVCCFLAGS += -O3 -Xcompiler='-O3'
endif
ifeq ($(machine_name), titan)
ALL_NVCCFLAGS += --generate-code arch=compute_35,code=compute_35
# below are equivalant to the line above
# ALL_NVCCFLAGS += --gpu-architecture=compute_35
# ALL_NVCCFLAGS += --gpu-code=sm_35
else ifeq ($(machine_name), doppio)
ALL_NVCCFLAGS += --generate-code arch=compute_60,code=compute_60
ALL_NVCCFLAGS += --generate-code arch=compute_61,code=compute_61
else ifeq ($(machine_name), rhea)
ALL_NVCCFLAGS += --generate-code arch=compute_35,code=compute_35
else
ALL_NVCCFLAGS += --generate-code arch=compute_35,code=compute_35
endif
# uncomment these lines and change them accordingly to obtain potential
# better optimization if you know the CUDA compute capability for the
# machine this code is compiled for
# ALL_NVCCFLAGS += --generate-code arch=compute_60,code=compute_60
# ALL_NVCCFLAGS += --generate-code arch=compute_61,code=compute_61
ifeq ($(VERBOSE), TRUE)
ALL_NVCCFLAGS += -keep --resource-usage -Xptxas=--verbose
ALL_NVCCFLAGS += -keep-dir built.tmp
Expand All @@ -330,11 +276,6 @@ $(shell mkdir -p built.tmp)
endif
endif

ifeq ($(machine_name), doppio)
# TODO: this is just a workaround for imcompatibility of pgi 18.4 and ubuntu 18.4 on Doppio
# We should get rid of this eventually
ALL_CPPFLAGS += -D__CUDACC__
endif

# We may want to set MAKELEVEL here as it is not always set but we know we are
# not the first level (the original calling Makefile should be MAKELEVEL = 0)
Expand Down