[cheri] Add support for aarch64c ASM dispatch.

The existing aarch64 ASM dispatch mechanism has been adapted for the
Morello ISA, replacing loads and stores with instructions to load and
store capability values.

In a number of cases values are computed as offsets from a sentry value,
which results in an invalid capability. This code has been changed to
recomputed capabilities using the program counter, before (where
necessary) resealing the capability to an sentry.

Save and restore the full set of caller-saved registers, including c9.

src/GLX/glvnd_genentry.c

@@ -95,12 +95,25 @@ extern char glx_entrypoint_end[];
 
 #elif defined(USE_AARCH64_ASM)
 
+#if defined(__CHERI_PURE_CAPABILITY__)
+#define STUB_SIZE 32
+#define STUB_ASM_ARCH(slot) \
+    "adrp c16, entrypointFunctions + " slot "*16\n" \
+    "str c17, [csp, #-16]\n" \
+    "adrp c17, 0\n" \
+    "scvalue c17, c17, x16\n" \
+    "ldr c16, [c17, #:lo12:(entrypointFunctions + " slot "*16)]\n" \
+    "ldr c17, [csp], #16\n" \
+    "br c16\n" \
+    "nop\n"
+#else   // !__CHERI_PURE_CAPABILITY__
 #define STUB_SIZE 16
 #define STUB_ASM_ARCH(slot) \
     "hint #34\n" \
     "adrp x16, entrypointFunctions + " slot "*8\n" \
     "ldr x16, [x16, #:lo12:(entrypointFunctions + " slot "*8)]\n" \
     "br x16\n"
+#endif  // !__CHERI_PURE_CAPABILITY__
 
 #elif defined(USE_PPC64_ASM) && defined(_CALL_ELF) && (_CALL_ELF == 2)
 
@@ -166,7 +179,15 @@ static void *DefaultDispatchFunc(void)
 
 static GLVNDentrypointStub GetEntrypointStub(int index)
 {
+#if defined(__CHERI_PURE_CAPABILITY__)
+    const ptraddr_t sentry_addr = __builtin_cheri_address_get(glx_entrypoint_start);
+    const ptraddr_t stub_addr = sentry_addr + (index * STUB_SIZE);
+    const void* pcc = __builtin_cheri_program_counter_get();
+    uintptr_t result_cap = (uintptr_t) __builtin_cheri_address_set(pcc, stub_addr);
+    return (GLVNDentrypointStub) __builtin_cheri_seal_entry(result_cap | 1);
+#else   // !__CHERI_PURE_CAPABILITY__
     return (GLVNDentrypointStub) (glx_entrypoint_start + (index * STUB_SIZE));
+#endif  // !__CHERI_PURE_CAPABILITY__
 }
 
 GLVNDentrypointStub glvndGenerateEntrypoint(const char *procName)

src/GLdispatch/vnd-glapi/entry_aarch64_tsd.c

@@ -68,6 +68,35 @@
  * The 'found_dispatch' section computes the correct offset in the dispatch
  * table then does a branch without link to the function address.
  */
+#if defined(__CHERI_PURE_CAPABILITY__)
+#define STUB_ASM_CODE(slot)                           \
+    "stp c1, c0, [csp, #-32]!\n\t"                    \
+    "adrp c0, :got:_glapi_Current\n\t"                \
+    "ldr c0, [c0, #:got_lo12:_glapi_Current]\n\t"     \
+    "ldr c0, [c0]\n\t"                                \
+    "gcvalue x1, c0\n\t"                              \
+    "cbz x1, 10f\n\t"                                 \
+    "11:\n\t"        /* found dispatch */             \
+    "ldr x1, 3f\n\t"                                  \
+    "ldr c16, [c0, x1]\n\t"                           \
+    "ldp c1, c0, [csp], #32\n\t"                      \
+    "br c16\n\t"                                      \
+    "10:\n\t"        /* lookup dispatch */            \
+    "stp c30, c9, [csp, #-32]!\n\t"                   \
+    "stp c7, c6, [csp, #-32]!\n\t"                    \
+    "stp c5, c4, [csp, #-32]!\n\t"                    \
+    "stp c3, c2, [csp, #-32]!\n\t"                    \
+    "adrp c0, :got:_glapi_get_current\n\t"            \
+    "ldr c0, [c0, #:got_lo12:_glapi_get_current]\n\t" \
+    "blr c0\n\t"                                      \
+    "ldp c3, c2, [csp], #32\n\t"                      \
+    "ldp c5, c4, [csp], #32\n\t"                      \
+    "ldp c7, c6, [csp], #32\n\t"                      \
+    "ldr c30, c9, [csp], #32\n\t"                     \
+    "b 11b\n\t"                                       \
+    "3:\n\t"                                          \
+    ".xword " slot " * 16\n\t" /* size of (void *) */
+#else   // !__CHERI_PURE_CAPABILITY__
 #define STUB_ASM_CODE(slot)                           \
     "hint #34\n\t"                                    \
     "stp x1, x0, [sp, #-16]!\n\t"                     \
@@ -81,7 +110,7 @@
     "ldp x1, x0, [sp], #16\n\t"                       \
     "br x16\n\t"                                      \
     "10:\n\t"        /* lookup dispatch */            \
-    "str x30, [sp, #-16]!\n\t"                        \
+    "stp x30, [sp, #-16]!\n\t"                        \
     "stp x7, x6, [sp, #-16]!\n\t"                     \
     "stp x5, x4, [sp, #-16]!\n\t"                     \
     "stp x3, x2, [sp, #-16]!\n\t"                     \
@@ -95,6 +124,7 @@
     "b 11b\n\t"                                       \
     "3:\n\t"                                          \
     ".xword " slot " * 8\n\t" /* size of (void *) */
+#endif  // !__CHERI_PURE_CAPABILITY__
 
 __asm__(".section wtext,\"ax\"\n"
         ".balign " U_STRINGIFY(GLDISPATCH_PAGE_SIZE) "\n"

src/GLdispatch/vnd-glapi/entry_common.c

@@ -80,15 +80,30 @@ int entry_patch_finish(void)
 
 void *entry_get_patch_address(int index)
 {
+#if defined(__CHERI_PURE_CAPABILITY__)
+    const ptraddr_t sentry_addr = __builtin_cheri_address_get(public_entry_start);
+    const ptraddr_t stub_addr = sentry_addr + (index * entry_stub_size);
+    const void* pcc = __builtin_cheri_program_counter_get();
+    uintptr_t result_cap = (uintptr_t) __builtin_cheri_address_set(pcc, stub_addr);
+    return (void *) __builtin_cheri_seal_entry(result_cap | 1);
+#else   // !__CHERI_PURE_CAPABILITY__
     return (void *) (public_entry_start + (index * entry_stub_size));
+#endif  // !__CHERI_PURE_CAPABILITY__
 }
 
 void *entry_save_entrypoints(void)
 {
     size_t size = ((uintptr_t) public_entry_end) - ((uintptr_t) public_entry_start);
     void *buf = malloc(size);
     if (buf != NULL) {
+#if defined(__CHERI_PURE_CAPABILITY__)
+        const ptraddr_t sentry_addr = __builtin_cheri_address_get(public_entry_start);
+        const void* pcc = __builtin_cheri_program_counter_get();
+        uintptr_t src_cap = (uintptr_t) __builtin_cheri_address_set(pcc, sentry_addr);
+        memcpy(buf, src_cap, size);
+#else   // !__CHERI_PURE_CAPABILITY__
         memcpy(buf, public_entry_start, size);
+#endif  // !__CHERI_PURE_CAPABILITY__
     }
     return buf;
 }
@@ -139,7 +154,14 @@ static void InvalidateCache(void)
 void entry_restore_entrypoints(void *saved)
 {
     size_t size = ((uintptr_t) public_entry_end) - ((uintptr_t) public_entry_start);
+#if defined(__CHERI_PURE_CAPABILITY__)
+    const ptraddr_t sentry_addr = __builtin_cheri_address_get(public_entry_start);
+    const void* pcc = __builtin_cheri_program_counter_get();
+    uintptr_t dst_cap = (uintptr_t) __builtin_cheri_address_set(pcc, sentry_addr);
+    memcpy(dst_cap, saved, size);
+#else   // !__CHERI_PURE_CAPABILITY__
     memcpy(public_entry_start, saved, size);
+#endif  // !__CHERI_PURE_CAPABILITY__
     InvalidateCache();
 }
 

src/GLdispatch/vnd-glapi/entry_simple_asm.c

@@ -53,5 +53,13 @@
 
 mapi_func entry_get_public(int index)
 {
+#if defined(__CHERI_PURE_CAPABILITY__)
+    const ptraddr_t sentry_addr = __builtin_cheri_address_get(public_entry_start);
+    const ptraddr_t stub_addr = sentry_addr + (index * entry_stub_size);
+    const void* pcc = __builtin_cheri_program_counter_get();
+    uintptr_t result_cap = (uintptr_t) __builtin_cheri_address_set(pcc, stub_addr);
+    return (mapi_func) __builtin_cheri_seal_entry(result_cap | 1);
+#else   // !__CHERI_ PURE_CAPABILITY__
     return (mapi_func)(public_entry_start + (index * entry_stub_size));
+#endif  // !__CHERI_PURE_CAPABILITY__
 }