Raise number of threads used by GASNet by default + add warning if exceeded

runtime/src/comm/gasnet/comm-gasnet-ex.c

@@ -714,7 +714,15 @@ int chpl_comm_addr_gettable(c_nodeid_t node, void* start, size_t len)
 
 
 int32_t chpl_comm_getMaxThreads(void) {
-  return GASNETI_MAX_THREADS-1;
+  uint64_t maxGasnetThreads = gex_System_QueryMaxThreads();
+  if (maxGasnetThreads > INT32_MAX) {
+    if (chpl_nodeID == 0) {
+      chpl_warning("GASNet's maximum number of threads exceeds Chapel's "
+                   "'int32_t' value, so capping it at INT32_MAX.", 0, 0);
+    }
+    maxGasnetThreads = INT32_MAX;
+  }
+  return maxGasnetThreads - 1; // reserve one thread for the primordial thread
 }
 
 //

runtime/src/tasks/qthreads/tasks-qthreads.c

@@ -502,7 +502,18 @@ static void setupAvailableParallelism(int32_t maxThreads) {
     if (hwpar > 0) {
         // Limit the parallelism to the maximum imposed by the comm layer.
         if (0 < maxThreads && maxThreads < hwpar) {
-            hwpar = maxThreads;
+          if (chpl_nodeID == 0) {
+            char msg[1024];
+            snprintf(msg, sizeof(msg),
+                     "The CHPL_COMM setting is limiting the number of threads "
+                     "to %d, rather than the hardware's preference of %d.%s",
+                     maxThreads, hwpar,
+                     (strcmp(CHPL_COMM, "gasnet") ? "" :
+                      "  To increase this limit, set 'GASNET_MAX_THREADS' to "
+                      "a larger value in your environment."));
+            chpl_warning(msg, 0, 0);
+          }
+          hwpar = maxThreads;
         }
 
         //

test/parallel/taskPool/figueroa/ManyThreads.comm-gasnet.tasks-fifo.good

@@ -1,3 +1,3 @@
 total is 4501500
-warning: CHPL_RT_NUM_THREADS_PER_LOCALE = 300 is too large; limit is NNNN
+warning: CHPL_RT_NUM_THREADS_PER_LOCALE = 2000 is too large; limit is NNNN
 warning: Setting number of threads in CHPL_TASKS=fifo can lead to deadlock

test/parallel/taskPool/figueroa/ManyThreads.comm-gasnet.tasks-qthreads.good

@@ -1,3 +1,3 @@
 total is 4501500
-warning: CHPL_RT_NUM_THREADS_PER_LOCALE = 300 is too large; limit is NNNN
+warning: CHPL_RT_NUM_THREADS_PER_LOCALE = 2000 is too large; limit is NNNN
 warning: QTHREADS: Reduced numThreadsPerLocale=NNNN to NNNN to prevent oversubscription of the system.

test/parallel/taskPool/figueroa/ManyThreads.execenv

@@ -1,4 +1,4 @@
-CHPL_RT_NUM_THREADS_PER_LOCALE=300
+CHPL_RT_NUM_THREADS_PER_LOCALE=2000
 CHPL_RT_NUM_THREADS_PER_LOCALE_QUIET=no
 CHPL_RT_CALL_STACK_SIZE=256K
 QT_GUARD_PAGES=false

third-party/gasnet/Makefile

@@ -135,7 +135,7 @@ gasnet-config: FORCE
 	mkdir -p $(GASNET_BUILD_DIR)
 	cd $(GASNET_SUBDIR) && ./Bootstrap -T
 	$(XTRA_CONFIG_COMMAND)
-	$(ENSURE_GASNET_COMPATIBLE_COMPILER) cd $(GASNET_BUILD_DIR) && $(CHPL_GASNET_ENV_VARS) $(GASNET_SUBDIR)/$(CHPL_GASNET_CFG_SCRIPT) --prefix=$(GASNET_INSTALL_DIR) $(CHPL_GASNET_CFG_OPTIONS) --disable-seq --enable-par --disable-parsync $(CHPL_GASNET_CFG_OPTIONS)
+	$(ENSURE_GASNET_COMPATIBLE_COMPILER) cd $(GASNET_BUILD_DIR) && $(CHPL_GASNET_ENV_VARS) $(GASNET_SUBDIR)/$(CHPL_GASNET_CFG_SCRIPT) --prefix=$(GASNET_INSTALL_DIR) --disable-seq --enable-par --disable-parsync --with-max-pthreads-per-node=65535 $(CHPL_GASNET_CFG_OPTIONS)
 
 gasnet-build: FORCE
 	$(ENSURE_GASNET_COMPATIBLE_COMPILER) cd $(GASNET_BUILD_DIR) && $(MAKE) all
@@ -158,4 +158,6 @@ gasnet: gasnet-config gasnet-build gasnet-install
 
 FORCE:
 
+.PHONY: FORCE
+
 .NOTPARALLEL: