cpu: support darwin/arm64 CPU feature detection

Support ARM64 features detection. The CPU features which are supported by
Apple Silicon M1 are assumed as the minimal set of features for Go programs
running on darwin/arm64.

The ARM64 supporting features are referred to
https://en.wikichip.org/wiki/arm/armv8#ARMv8_Extensions_and_Processor_Features

cpu/cpu_android_arm64.go

@@ -0,0 +1,98 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build arm64
+// +build android
+
+package cpu
+
+// HWCAP/HWCAP2 bits. These are exposed by Linux.
+const (
+	hwcap_FP       = 1 << 0
+	hwcap_ASIMD    = 1 << 1
+	hwcap_EVTSTRM  = 1 << 2
+	hwcap_AES      = 1 << 3
+	hwcap_PMULL    = 1 << 4
+	hwcap_SHA1     = 1 << 5
+	hwcap_SHA2     = 1 << 6
+	hwcap_CRC32    = 1 << 7
+	hwcap_ATOMICS  = 1 << 8
+	hwcap_FPHP     = 1 << 9
+	hwcap_ASIMDHP  = 1 << 10
+	hwcap_CPUID    = 1 << 11
+	hwcap_ASIMDRDM = 1 << 12
+	hwcap_JSCVT    = 1 << 13
+	hwcap_FCMA     = 1 << 14
+	hwcap_LRCPC    = 1 << 15
+	hwcap_DCPOP    = 1 << 16
+	hwcap_SHA3     = 1 << 17
+	hwcap_SM3      = 1 << 18
+	hwcap_SM4      = 1 << 19
+	hwcap_ASIMDDP  = 1 << 20
+	hwcap_SHA512   = 1 << 21
+	hwcap_SVE      = 1 << 22
+	hwcap_ASIMDFHM = 1 << 23
+)
+
+func osInit() {
+	if err := readHWCAP(); err != nil {
+		// failed to read /proc/self/auxv, try reading registers directly
+		readARM64Registers()
+		return
+	}
+
+	// HWCap was populated by the runtime from the auxiliary vector.
+	// Use HWCap information since reading aarch64 system registers
+	// is not supported in user space on older linux kernels.
+	ARM64.HasFP = isSet(hwCap, hwcap_FP)
+	ARM64.HasASIMD = isSet(hwCap, hwcap_ASIMD)
+	ARM64.HasEVTSTRM = isSet(hwCap, hwcap_EVTSTRM)
+	ARM64.HasAES = isSet(hwCap, hwcap_AES)
+	ARM64.HasPMULL = isSet(hwCap, hwcap_PMULL)
+	ARM64.HasSHA1 = isSet(hwCap, hwcap_SHA1)
+	ARM64.HasSHA2 = isSet(hwCap, hwcap_SHA2)
+	ARM64.HasCRC32 = isSet(hwCap, hwcap_CRC32)
+	ARM64.HasFPHP = isSet(hwCap, hwcap_FPHP)
+	ARM64.HasASIMDHP = isSet(hwCap, hwcap_ASIMDHP)
+	ARM64.HasASIMDRDM = isSet(hwCap, hwcap_ASIMDRDM)
+	ARM64.HasJSCVT = isSet(hwCap, hwcap_JSCVT)
+	ARM64.HasFCMA = isSet(hwCap, hwcap_FCMA)
+	ARM64.HasLRCPC = isSet(hwCap, hwcap_LRCPC)
+	ARM64.HasDCPOP = isSet(hwCap, hwcap_DCPOP)
+	ARM64.HasSHA3 = isSet(hwCap, hwcap_SHA3)
+	ARM64.HasSM3 = isSet(hwCap, hwcap_SM3)
+	ARM64.HasSM4 = isSet(hwCap, hwcap_SM4)
+	ARM64.HasASIMDDP = isSet(hwCap, hwcap_ASIMDDP)
+	ARM64.HasSHA512 = isSet(hwCap, hwcap_SHA512)
+	ARM64.HasSVE = isSet(hwCap, hwcap_SVE)
+	ARM64.HasASIMDFHM = isSet(hwCap, hwcap_ASIMDFHM)
+
+	// The Samsung S9+ kernel reports support for atomics, but not all cores
+	// actually support them, resulting in SIGILL. See issue #28431.
+	// TODO(elias.naur): Only disable the optimization on bad chipsets on android.
+	ARM64.HasATOMICS = false
+
+	// Check to see if executing on a NeoverseN1 and in order to do that,
+	// check the AUXV for the CPUID bit. The getMIDR function executes an
+	// instruction which would normally be an illegal instruction, but it's
+	// trapped by the kernel, the value sanitized and then returned. Without
+	// the CPUID bit the kernel will not trap the instruction and the process
+	// will be terminated with SIGILL.
+	if ARM64.HasCPUID {
+		midr := getMIDR()
+		part_num := uint16((midr >> 4) & 0xfff)
+		implementor := byte((midr >> 24) & 0xff)
+
+		if implementor == 'A' && part_num == 0xd0c {
+			ARM64.IsNeoverseN1 = true
+		}
+		if implementor == 'A' && part_num == 0xd40 {
+			ARM64.IsZeus = true
+		}
+	}
+}
+
+func isSet(hwc uint, value uint) bool {
+	return hwc&value != 0
+}

cpu/cpu_arm64.go

@@ -4,8 +4,6 @@
 
 package cpu
 
-import "runtime"
-
 const cacheLineSize = 64
 
 func initOptions() {
@@ -38,25 +36,8 @@ func initOptions() {
 }
 
 func archInit() {
-	switch runtime.GOOS {
-	case "freebsd":
-		readARM64Registers()
-	case "linux", "netbsd":
-		doinit()
-	default:
-		// Most platforms don't seem to allow reading these registers.
-		//
-		// OpenBSD:
-		// See https://golang.org/issue/31746
-		setMinimalFeatures()
-	}
-}
-
-// setMinimalFeatures fakes the minimal ARM64 features expected by
-// TestARM64minimalFeatures.
-func setMinimalFeatures() {
-	ARM64.HasASIMD = true
-	ARM64.HasFP = true
+	setMinimalFeatures()
+	osInit()
 }
 
 func readARM64Registers() {
@@ -170,3 +151,12 @@ func parseARM64SystemRegisters(isar0, isar1, pfr0 uint64) {
 func extractBits(data uint64, start, end uint) uint {
 	return (uint)(data>>start) & ((1 << (end - start + 1)) - 1)
 }
+
+// setMinimalFeatures fakes the minimal ARM64 features expected by
+// TestARM64minimalFeatures.
+func setMinimalFeatures() {
+	ARM64.HasASIMD = true
+	ARM64.HasFP = true
+}
+
+func getMIDR() uint64

cpu/cpu_arm64.s

@@ -30,3 +30,9 @@ TEXT ·getpfr0(SB),NOSPLIT,$0-8
 	WORD	$0xd5380400
 	MOVD	R0, ret+0(FP)
 	RET
+
+// func getMIDR() uint64
+TEXT ·getMIDR(SB), NOSPLIT, $0-8
+	MRS	MIDR_EL1, R0
+	MOVD	R0, ret+0(FP)
+	RET

cpu/cpu_darwin_arm64.go

@@ -0,0 +1,133 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build arm64
+// +build darwin
+// +build !ios
+
+package cpu
+
+import (
+	"fmt"
+	"strings"
+	"syscall"
+	"unsafe"
+)
+
+func osInit() {
+	ARM64.HasFP = sysctlEnabled("hw.optional.floatingpoint")
+	ARM64.HasASIMD = sysctlEnabled("hw.optional.neon")
+	ARM64.HasCRC32 = sysctlEnabled("hw.optional.armv8_crc32")
+	ARM64.HasATOMICS = sysctlEnabled("hw.optional.armv8_1_atomics")
+	ARM64.HasFPHP = sysctlEnabled("hw.optional.neon_hpfp")
+	ARM64.HasASIMDHP = sysctlEnabled("hw.optional.floatingpoint")
+	ARM64.HasSHA3 = sysctlEnabled("hw.optional.armv8_2_sha3")
+	ARM64.HasSHA512 = sysctlEnabled("hw.optional.armv8_2_sha512")
+	ARM64.HasASIMDFHM = sysctlEnabled("hw.optional.armv8_2_fhm")
+
+	// There are no hw.optional sysctl values for the below features on Mac OS 11.0
+	// to detect their supported state dynamically. Assume the CPU features that
+	// Apple Silicon M1 supports to be available as a minimal set of features
+	// to all Go programs running on darwin/arm64.
+	ARM64.HasEVTSTRM = true
+	ARM64.HasAES = true
+	ARM64.HasPMULL = true
+	ARM64.HasSHA1 = true
+	ARM64.HasSHA2 = true
+	ARM64.HasCPUID = true
+	ARM64.HasASIMDRDM = true
+	ARM64.HasJSCVT = true
+	ARM64.HasFCMA = true
+	ARM64.HasLRCPC = true
+	ARM64.HasDCPOP = true
+	ARM64.HasSM3 = true
+	ARM64.HasSM4 = true
+	ARM64.HasASIMDDP = true
+	ARM64.HasSVE = true
+}
+
+// The following is minimal copy of functionality from x/sys/unix so the cpu package can call
+// sysctl without depending on x/sys/unix.
+
+func sysctlEnabled(name string, args ...int) bool {
+	mib, err := nametomib(name)
+	if err != nil {
+		return false
+	}
+
+	for _, a := range args {
+		mib = append(mib, _C_int(a))
+	}
+
+	// Find size.
+	n := uintptr(0)
+	if err := sysctl(mib, nil, &n, nil, 0); err != nil {
+		return false
+	}
+
+	return true
+}
+
+type _C_int int32
+
+func sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) {
+	var _zero uintptr
+	var _p0 unsafe.Pointer
+	if len(mib) > 0 {
+		_p0 = unsafe.Pointer(&mib[0])
+	} else {
+		_p0 = unsafe.Pointer(&_zero)
+	}
+	_, _, errno := syscall.Syscall6(
+		syscall.SYS___SYSCTL,
+		uintptr(_p0),
+		uintptr(len(mib)),
+		uintptr(unsafe.Pointer(old)),
+		uintptr(unsafe.Pointer(oldlen)),
+		uintptr(unsafe.Pointer(new)),
+		uintptr(newlen))
+	if errno != 0 {
+		return errno
+	}
+	return nil
+}
+
+// nametomib is a copy from "unix.nametomib()" in "unix/syscall_darwin.go".
+func nametomib(name string) (mib []_C_int, err error) {
+	const CTL_MAXNAME = 0xc
+	const siz = unsafe.Sizeof(mib[0])
+
+	// NOTE(rsc): It seems strange to set the buffer to have
+	// size CTL_MAXNAME+2 but use only CTL_MAXNAME
+	// as the size. I don't know why the +2 is here, but the
+	// kernel uses +2 for its own implementation of this function.
+	// I am scared that if we don't include the +2 here, the kernel
+	// will silently write 2 words farther than we specify
+	// and we'll get memory corruption.
+	var buf [CTL_MAXNAME + 2]_C_int
+	n := uintptr(CTL_MAXNAME) * siz
+
+	p := (*byte)(unsafe.Pointer(&buf[0]))
+	bytes, err := byteSliceFromString(name)
+	if err != nil {
+		return nil, err
+	}
+
+	// Magic sysctl: "setting" 0.3 to a string name
+	// lets you read back the array of integers form.
+	if err = sysctl([]_C_int{0, 3}, p, &n, &bytes[0], uintptr(len(name))); err != nil {
+		return nil, err
+	}
+	return buf[0 : n/siz], nil
+}
+
+// byteSliceFromString is a simple copy of "unix.ByteSliceFromString()"
+func byteSliceFromString(s string) ([]byte, error) {
+	if strings.IndexByte(s, 0) != -1 {
+		return nil, fmt.Errorf("invalid argument in cpu.byteSliceFromString()")
+	}
+	a := make([]byte, len(s)+1)
+	copy(a, s)
+	return a, nil
+}

cpu/cpu_freebsd_arm64.go

@@ -0,0 +1,12 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build arm64
+// +build freebsd
+
+package cpu
+
+func osInit() {
+	readARM64Registers()
+}

cpu/cpu_linux_arm64.go

@@ -2,6 +2,10 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build arm64
+// +build linux
+// +build !android
+
 package cpu
 
 // HWCAP/HWCAP2 bits. These are exposed by Linux.
@@ -32,7 +36,7 @@ const (
 	hwcap_ASIMDFHM = 1 << 23
 )
 
-func doinit() {
+func osInit() {
 	if err := readHWCAP(); err != nil {
 		// failed to read /proc/self/auxv, try reading registers directly
 		readARM64Registers()

cpu/cpu_other_arm64.go

@@ -2,9 +2,11 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build !linux && !netbsd && arm64
-// +build !linux,!netbsd,arm64
+//go:build !linux && !netbsd && !darwin && arm64
+// +build !linux,!netbsd,!darwin,arm64
 
 package cpu
 
-func doinit() {}
+func osInit() {
+	setMinimalFeatures()
+}

cpu/cpu_test.go

@@ -42,7 +42,7 @@ func TestAVX512HasAVX2AndAVX(t *testing.T) {
 }
 
 func TestARM64minimalFeatures(t *testing.T) {
-	if runtime.GOARCH != "arm64" || (runtime.GOOS == "darwin" || runtime.GOOS == "ios") {
+	if runtime.GOARCH != "arm64" || runtime.GOOS == "ios" {
 		return
 	}
 	if !cpu.ARM64.HasASIMD {