#i2016 slowpath-aarch64-port inital

CMakeLists.txt

@@ -99,6 +99,8 @@ endif ()
 # The target arch:
 if (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm")
   set(ARM 1)
+elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64")
+  set(AARCH64 1)
 else ()
   set(X86 1)
 endif ()
@@ -329,13 +331,18 @@ if (UNIX)
   endif ()
   if (ARM)
     set(EXTRA_FLAGS "${EXTRA_FLAGS} -mthumb -march=armv7-a")
+  endif ()
+  if (AARCH64)
+    set(EXTRA_FLAGS "${EXTRA_FLAGS} -march=armv8-a")
+  endif ()
+  if (ARM OR AARCH64)
     if (ANDROID OR CMAKE_C_LIBRARY_ARCHITECTURE MATCHES "gnueabi$")
       set(EXTRA_FLAGS "${EXTRA_FLAGS} -mfloat-abi=softfp")
       # Android requires PIE.  We export symbols to match our test assumptions.
       set(CMAKE_EXE_LINKER_FLAGS
         "${CMAKE_EXE_LINKER_FLAGS} -fPIE -pie -Wl,--export-dynamic")
-    endif ()
   endif ()
+endif ()
   # We use C++11.
   set(EXTRA_CXXFLAGS "-std=c++11")
   set(CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UPPER}
@@ -1029,6 +1036,8 @@ set(asm_deps "${DynamoRIO_DIR}/cpp2asm_defines.h")
 
 if (ARM)
   set(asm_file "asm_utils_arm.asm")
+elseif (AARCH64)
+  set(asm_file "asm_utils_aarch64.asm")
 else ()
   set(asm_file "asm_utils_x86.asm")
 endif ()

common/alloc.c

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2010-2021 Google, Inc.  All rights reserved.
+ * Copyright (c) 2010-2024 Google, Inc.  All rights reserved.
  * Copyright (c) 2008-2010 VMware, Inc.  All rights reserved.
  * **********************************************************/
 
@@ -138,7 +138,7 @@ set_brk(byte *new_val)
 #endif
 
 static void
-alloc_hook(void *wrapcxt, INOUT void **user_data);
+alloc_hook(void *wrapcxt, DR_PARAM_INOUT void **user_data);
 
 static void
 handle_alloc_post(void *wrapcxt, void *user_data);
@@ -967,7 +967,7 @@ replace_realloc_size_app(void *p)
 }
 
 static void
-replace_realloc_size_pre(void *wrapcxt, OUT void **user_data)
+replace_realloc_size_pre(void *wrapcxt, DR_PARAM_OUT void **user_data)
 {
     cls_alloc_t *pt = (cls_alloc_t *)
         drmgr_get_cls_field(dr_get_current_drcontext(), cls_idx_alloc);
@@ -995,12 +995,17 @@ generate_jmp_ind_stub(void *drcontext, app_pc tgt_pc, byte *epc)
     instr_t *instr;
     /* assuming %rax is dead, mov pc => %rax; jmp %rax */
     ASSERT(tgt_pc != NULL, "wrong target pc for call stub");
+#ifdef AARCH64
+    /* XXX i#2016: This will fail for far-away targets. */
+    instr = INSTR_CREATE_b(drcontext, opnd_create_pc(tgt_pc));
+#else
     instr = INSTR_CREATE_mov_imm(drcontext,
                                  opnd_create_reg(DR_REG_XAX),
                                  OPND_CREATE_INTPTR(tgt_pc));
     epc = instr_encode(drcontext, instr, epc);
     instr_destroy(drcontext, instr);
     instr = INSTR_CREATE_jmp_ind(drcontext, opnd_create_reg(DR_REG_XAX));
+#endif
     epc = instr_encode(drcontext, instr, epc);
     instr_destroy(drcontext, instr);
     return epc;
@@ -1594,7 +1599,7 @@ modname_is_libc_or_libcpp(const char *modname)
 
 static bool
 distinguish_operator_by_decoding(routine_type_t generic_type,
-                                 routine_type_t *specific_type OUT,
+                                 routine_type_t *specific_type DR_PARAM_OUT,
                                  const char *name, const module_data_t *mod,
                                  size_t modoffs)
 {
@@ -1691,7 +1696,7 @@ distinguish_operator_by_decoding(routine_type_t generic_type,
  */
 static bool
 distinguish_operator_no_argtypes(routine_type_t generic_type,
-                                 routine_type_t *specific_type OUT,
+                                 routine_type_t *specific_type DR_PARAM_OUT,
                                  const char *name, const module_data_t *mod,
                                  size_t modoffs)
 {
@@ -2482,7 +2487,8 @@ find_alloc_routines(const module_data_t *mod, const possible_alloc_routine_t *po
                 instr_init(drcontext, &inst);
                 decode(drcontext, pc, &inst);
                 if (!instr_valid(&inst) || instr_get_opcode(&inst) ==
-                    IF_X86_ELSE(OP_jmp_ind, OP_bx))
+                    IF_AARCH64_ELSE(OP_br || instr_get_opcode(&inst) == OP_blr,
+                    IF_X86_ELSE(OP_jmp_ind, OP_bx)))
                     pc = NULL;
                 instr_free(drcontext, &inst);
             } else
@@ -2588,10 +2594,10 @@ malloc_wrap__unintercept(app_pc pc, routine_type_t type, alloc_routine_entry_t *
  */
 
 #ifdef WINDOWS
-typedef size_t (__stdcall *rtl_size_func_t)(IN reg_t /*really HANDLE*/ Heap,
-                                            IN ULONG flags,
-                                            IN PVOID ptr);
-typedef size_t (*dbg_size_func_t)(IN byte *pc, int blocktype);
+typedef size_t (__stdcall *rtl_size_func_t)(DR_PARAM_IN reg_t /*really HANDLE*/ Heap,
+                                            DR_PARAM_IN ULONG flags,
+                                            DR_PARAM_IN PVOID ptr);
+typedef size_t (*dbg_size_func_t)(DR_PARAM_IN byte *pc, int blocktype);
 #else
 /* points at libc's version, used in initial heap walk */
 alloc_size_func_t libc_malloc_usable_size;
@@ -2944,7 +2950,7 @@ malloc_entry_redzone_size(malloc_entry_t *e)
 }
 
 static void
-malloc_entry_to_info(malloc_entry_t *e, malloc_info_t *info OUT)
+malloc_entry_to_info(malloc_entry_t *e, malloc_info_t *info DR_PARAM_OUT)
 {
     info->struct_size = sizeof(*info);
     info->base = e->start;
@@ -6391,23 +6397,23 @@ handle_userinfo_pre(void *drcontext, cls_alloc_t *pt, void *wrapcxt,
     /* 3 related routines here:
      *   BOOLEAN NTAPI
      *   RtlGetUserInfoHeap(
-     *       IN PVOID HeapHandle,
-     *       IN ULONG Flags,
-     *       IN PVOID BaseAddress,
-     *       OUT PVOID *UserValue,
-     *       OUT PULONG UserFlags);
+     *       DR_PARAM_IN PVOID HeapHandle,
+     *       DR_PARAM_IN ULONG Flags,
+     *       DR_PARAM_IN PVOID BaseAddress,
+     *       DR_PARAM_OUT PVOID *UserValue,
+     *       DR_PARAM_OUT PULONG UserFlags);
      *   BOOLEAN NTAPI
      *   RtlSetUserValueHeap(
-     *       IN PVOID HeapHandle,
-     *       IN ULONG Flags,
-     *       IN PVOID BaseAddress,
-     *       IN PVOID UserValue);
+     *       DR_PARAM_IN PVOID HeapHandle,
+     *       DR_PARAM_IN ULONG Flags,
+     *       DR_PARAM_IN PVOID BaseAddress,
+     *       DR_PARAM_IN PVOID UserValue);
      *   BOOLEAN NTAPI
      *   RtlSetUserFlagsHeap(
-     *       IN PVOID HeapHandle,
-     *       IN ULONG Flags,
-     *       IN PVOID BaseAddress,
-     *       IN ULONG UserFlags);
+     *       DR_PARAM_IN PVOID HeapHandle,
+     *       DR_PARAM_IN ULONG Flags,
+     *       DR_PARAM_IN PVOID BaseAddress,
+     *       DR_PARAM_IN ULONG UserFlags);
      */
     app_pc base = (app_pc) drwrap_get_arg(wrapcxt, 2);
     if (malloc_is_native(base, pt, true))
@@ -6526,7 +6532,7 @@ handle_alloc_pre_ex(void *drcontext, cls_alloc_t *pt, void *wrapcxt,
                     alloc_routine_entry_t *routine);
 
 static void
-alloc_hook(void *wrapcxt, INOUT void **user_data)
+alloc_hook(void *wrapcxt, DR_PARAM_INOUT void **user_data)
 {
     app_pc pc = drwrap_get_func(wrapcxt);
     /* XXX: for -conservative we should do a lookup and not trust *user_data
@@ -6957,7 +6963,7 @@ malloc_large_remove(byte *start)
 }
 
 bool
-malloc_large_lookup(byte *addr, byte **start OUT, size_t *size OUT)
+malloc_large_lookup(byte *addr, byte **start DR_PARAM_OUT, size_t *size DR_PARAM_OUT)
 {
     bool res = false;
     rb_node_t *node;

common/alloc.h

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2010-2020 Google, Inc.  All rights reserved.
+ * Copyright (c) 2010-2024 Google, Inc.  All rights reserved.
  * Copyright (c) 2008-2010 VMware, Inc.  All rights reserved.
  * **********************************************************/
 
@@ -194,7 +194,7 @@ malloc_add(app_pc start, app_pc end, app_pc real_end,
 
 /* Looks up mallocs in the "large malloc table" (for mallocs used as stacks) */
 bool
-malloc_large_lookup(byte *addr, byte **start OUT, size_t *size OUT);
+malloc_large_lookup(byte *addr, byte **start DR_PARAM_OUT, size_t *size DR_PARAM_OUT);
 
 bool
 malloc_is_pre_us_ex(app_pc start, bool ok_if_invalid);
@@ -275,17 +275,17 @@ alloc_replace_in_cur_arena(byte *addr);
 /* overlap check includes redzone */
 bool
 alloc_replace_overlaps_delayed_free(byte *start, byte *end,
-                                    malloc_info_t *info INOUT);
+                                    malloc_info_t *info DR_PARAM_INOUT);
 
 /* overlap check includes redzone */
 bool
 alloc_replace_overlaps_any_free(byte *start, byte *end,
-                                malloc_info_t *info INOUT);
+                                malloc_info_t *info DR_PARAM_INOUT);
 
 /* overlap check includes redzone */
 bool
 alloc_replace_overlaps_malloc(byte *start, byte *end,
-                              malloc_info_t *info INOUT);
+                              malloc_info_t *info DR_PARAM_INOUT);
 
 /* Allocate application memory for clients.
  * This function can only be used with -replace_malloc and
@@ -379,7 +379,7 @@ client_handle_realloc_null(app_pc pc, dr_mcontext_t *mc);
  * For wrapping:
  *   Up to the caller to delay, via its return value.
  *   Returns the value to pass to free().  Return "tofree" for no change.
- *   The Windows heap param is INOUT so it can be changed as well.
+ *   The Windows heap param is DR_PARAM_INOUT so it can be changed as well.
  *   client_data is from client_add_malloc_routine().
  * For replacing:
  *   The return value is ignored.  Frees are always delayed, unless
@@ -394,7 +394,7 @@ client_handle_realloc_null(app_pc pc, dr_mcontext_t *mc);
 app_pc
 client_handle_free(malloc_info_t *info, byte *tofree, dr_mcontext_t *mc,
                    app_pc free_routine, void *routine_set_data, bool for_reuse
-                   _IF_WINDOWS(ptr_int_t *auxarg INOUT));
+                   _IF_WINDOWS(ptr_int_t *auxarg DR_PARAM_INOUT));
 
 /* For wrapping:
  *   Never called.

common/alloc_replace.c

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2012-2021 Google, Inc.  All rights reserved.
+ * Copyright (c) 2012-2024 Google, Inc.  All rights reserved.
  * **********************************************************/
 
 /* Dr. Memory: the memory debugger
@@ -1139,7 +1139,7 @@ arena_delayed_list_full(arena_header_t *arena)
 
 static inline chunk_header_t *
 next_chunk_forward(arena_header_t *arena, chunk_header_t *head,
-                   arena_header_t **container_out OUT)
+                   arena_header_t **container_out DR_PARAM_OUT)
 {
     arena_header_t *container;
     byte *start = ptr_from_header(head);
@@ -2396,11 +2396,11 @@ alloc_iterate(malloc_iter_cb_t cb, void *iter_data, bool only_live)
 
 static bool
 overlap_helper(chunk_header_t *head,
-               malloc_info_t *info INOUT,
+               malloc_info_t *info DR_PARAM_INOUT,
                uint positive_flags,
                uint negative_flags)
 {
-    /* XXX: this is the one INOUT case of this structure.  Once we extend it,
+    /* XXX: this is the one DR_PARAM_INOUT case of this structure.  Once we extend it,
      * we need to handle back-compat struct size here.  For now, header_to_info()
      * is used here and by above internal code that doesn't set struct-size.
      */
@@ -2421,7 +2421,7 @@ overlap_helper(chunk_header_t *head,
 /* Considers alloc_size to overlap, but returns request size in *found_end */
 static bool
 alloc_replace_overlaps_region(byte *start, byte *end,
-                              malloc_info_t *info INOUT,
+                              malloc_info_t *info DR_PARAM_INOUT,
                               uint positive_flags,
                               uint negative_flags)
 {
@@ -2514,21 +2514,21 @@ alloc_replace_overlaps_region(byte *start, byte *end,
 
 bool
 alloc_replace_overlaps_delayed_free(byte *start, byte *end,
-                                    malloc_info_t *info OUT)
+                                    malloc_info_t *info DR_PARAM_OUT)
 {
     return alloc_replace_overlaps_region(start, end, info, CHUNK_DELAY_FREE, 0);
 }
 
 bool
 alloc_replace_overlaps_any_free(byte *start, byte *end,
-                                malloc_info_t *info OUT)
+                                malloc_info_t *info DR_PARAM_OUT)
 {
     return alloc_replace_overlaps_region(start, end, info, CHUNK_FREED, 0);
 }
 
 bool
 alloc_replace_overlaps_malloc(byte *start, byte *end,
-                              malloc_info_t *info OUT)
+                              malloc_info_t *info DR_PARAM_OUT)
 {
     return alloc_replace_overlaps_region(start, end, info, 0, CHUNK_FREED);
 }
@@ -3574,7 +3574,7 @@ replace_context_exit(void *drcontext, bool thread_exit)
 }
 
 static void
-replace_start_nosy_sequence(void *wrapcxt, OUT void **user_data)
+replace_start_nosy_sequence(void *wrapcxt, DR_PARAM_OUT void **user_data)
 {
     cls_replace_t *data = (cls_replace_t *)
         drmgr_get_cls_field(dr_get_current_drcontext(), cls_idx_replace);
@@ -3588,7 +3588,7 @@ replace_start_nosy_sequence(void *wrapcxt, OUT void **user_data)
 }
 
 static void
-replace_stop_nosy_sequence(void *wrapcxt, OUT void **user_data)
+replace_stop_nosy_sequence(void *wrapcxt, DR_PARAM_OUT void **user_data)
 {
     cls_replace_t *data = (cls_replace_t *)
         drmgr_get_cls_field(dr_get_current_drcontext(), cls_idx_replace);
@@ -4201,7 +4201,8 @@ replace_ignore_arg5(void *arg1, void *arg2, void *arg3, void *arg4, void *arg5)
  * RtlHeap iteration replacement routines
  */
 
-typedef NTSTATUS (*PHEAP_ENUMERATION_ROUTINE)(IN PVOID HeapHandle, IN PVOID UserParam);
+typedef NTSTATUS (*PHEAP_ENUMERATION_ROUTINE)(DR_PARAM_IN PVOID HeapHandle,
+                                              DR_PARAM_IN PVOID UserParam);
 
 typedef struct _getheaps_data_t {
     ULONG actual_len;
@@ -4700,7 +4701,8 @@ alloc_entering_replace_routine(app_pc pc)
 
 static bool
 func_interceptor(routine_type_t type, bool check_mismatch, bool check_winapi_match,
-                 void **routine OUT, bool *at_entry OUT, uint *stack OUT)
+                 void **routine DR_PARAM_OUT, bool *at_entry DR_PARAM_OUT,
+                 uint *stack DR_PARAM_OUT)
 {
     /* almost everything is at the callee entry */
     *at_entry = true;

common/asm_utils.h

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2010-2021 Google, Inc.  All rights reserved.
+ * Copyright (c) 2010-2024 Google, Inc.  All rights reserved.
  * Copyright (c) 2007-2010 VMware, Inc.  All rights reserved.
  * **********************************************************/
 
@@ -25,11 +25,12 @@
 
 /* Returns the current values of xsp and xbp */
 void
-get_stack_registers(reg_t *xsp OUT, reg_t *xbp OUT);
+get_stack_registers(reg_t *xsp DR_PARAM_OUT, reg_t *xbp DR_PARAM_OUT);
 
 /* Returns the current values of xsp and xbp */
 void
-get_unwind_registers(reg_t *xsp OUT, reg_t *xbp OUT, app_pc *xip OUT);
+get_unwind_registers(reg_t *xsp DR_PARAM_OUT, reg_t *xbp DR_PARAM_OUT,
+                     app_pc *xip DR_PARAM_OUT);
 
 #ifdef UNIX
 ptr_int_t