Finish implementation of OS X i386 backtrace. The code hasn't been tested as part of SM yet.

stepmania/src/archutils/Unix/Backtrace.cpp

@@ -412,6 +412,8 @@ struct Frame
 	const Frame *link;
 	const void *return_address;
 };
+#define PROT_RW (VM_PROT_READ|VM_PROT_WRITE)
+#define PROT_EXE (VM_PROT_READ|VM_PROT_EXECUTE)
 
 /* Returns the starting address and the protection. Pass in mach_task_self() and the starting address. */
 static bool GetRegionInfo( mach_port_t self, const void *address, vm_address_t &startOut, vm_prot_t &protectionOut )
@@ -420,12 +422,17 @@ static bool GetRegionInfo( mach_port_t self, const void *address, vm_address_t &
 	mach_msg_type_number_t infoCnt = VM_REGION_BASIC_INFO_COUNT_64;
 	mach_port_t unused;
 	vm_size_t size = 0;
-	startOut = vm_address_t( address );
+	vm_address_t start = vm_address_t( address );
-	kern_return_t ret = vm_region( self, &startOut, &size, VM_REGION_BASIC_INFO_64,
+	kern_return_t ret = vm_region( self, &start, &size, VM_REGION_BASIC_INFO_64,
 				       (vm_region_info_t)&info, &infoCnt, &unused );
 	
-	if( ret != KERN_SUCCESS )
+	if( ret != KERN_SUCCESS ||
+		start >= (vm_address_t)address ||
+		(vm_address_t)address >= start + size )
+	{
 		return false;
+	}
+	startOut = start;
 	protectionOut = info.protection;
 	return true;
 }
@@ -438,15 +445,11 @@ void InitializeBacktrace()
 		return;
 	vm_prot_t protection;
 	if( !GetRegionInfo(mach_task_self(), __builtin_frame_address(0), g_StackPointer, protection) ||
-	    protection != (VM_PROT_READ|VM_PROT_WRITE) )
+	    protection != PROT_RW )
 	{
 		g_StackPointer = 0;
 	}
 	bInitialized = true;
-	// XXX: I can't test this so disable it all.
-#if 1
-	g_StackPointer = 0;
-#endif
 }
 
 void GetSignalBacktraceContext( BacktraceContext *ctx, const ucontext_t *uc )
@@ -527,9 +530,9 @@ void GetBacktrace( const void **buf, size_t size, const BacktraceContext *ctx )
 	}
 	
 	mach_port_t self = mach_task_self();
-	vm_address_t stackPointer;
+	vm_address_t stackPointer = 0;
-	vm_address_t start;
+	vm_prot_t protection = 0;
-	vm_prot_t protection;
+	vm_address_t start = 0;
 	
 	size_t i = 0;
 	if( i < size-1 && ctx->ip )
@@ -546,33 +549,61 @@ void GetBacktrace( const void **buf, size_t size, const BacktraceContext *ctx )
 		}
 	}
 	
+	GetRegionInfo( self, ctx->sp, stackPointer, protection );
+	if( protection != PROT_RW )
+	{
+		/* There isn't much we can do if this is the case. The stack should be read/write
+		 * and since it isn't, give up. */
+		buf[i] = NULL;
+		return;
+	}
 	const Frame *frame = (Frame *)ctx->sp;
 	
 	while( i < size-1 )
 	{
 		// Make sure this is on the stack
-		if( !GetRegionInfo(self, frame, start, protection) || protection != (VM_PROT_READ|VM_PROT_WRITE) )
+		if( !GetRegionInfo(self, frame, start, protection) || protection != PROT_RW )
 			break;
 		if( (start != g_StackPointer && start != stackPointer) || uintptr_t(frame)-uintptr_t(start) < sizeof(Frame) )
 			break;
 		
-		// Valid frame?
+		/* The stack pointer is always 16 byte aligned _before_ the call. Thus a valid frame
-		if( frame->link <= frame || // The frame link goes up.
+		 * should look like the follwoing.
-		    !GetRegionInfo(self, frame->link, start, protection) || // The link is on the stack.
+		 * |                  |
-		    protection != (VM_PROT_READ|VM_PROT_WRITE) ||
+		 * |  Caller's frame  |
-		    (start != g_StackPointer && start != stackPointer) ||
+		 * -------------------- 16 byte boundary
-		    !GetRegionInfo(self, frame->return_address, start, protection) || // readable, executable
+		 * |     Linkage      | This is return_address
-		    (protection & (VM_PROT_READ|VM_PROT_EXECUTE))!=(VM_PROT_READ|VM_PROT_EXECUTE) ||
+		 * - - - - - - - - - - 
-		    !PointsToValidCall(start, frame->return_address) )// Follows a CALL.
+		 * |   Saved %ebp     | This is link
+		 * - - - - - - - - - - 
+		 * |    Rest of       |
+		 * |  Callee's frame  |
+		 *
+		 * Therefore, frame + 8 should be on a 16 byte boundary, the frame link should be
+		 * at a higher address, the link should be on the stack and it should be RW. The
+		 * return address should be EXE and point to a valid call (well, just after). */
+		if( (((uintptr_t)frame+8) & 0xF) != 0 ||// boundary
+		    frame->link <= frame || // the frame link goes up
+		    !GetRegionInfo(self, frame->link, start, protection) ||
+		    (start != g_StackPointer && start != stackPointer) || // the link is on the stack
+		    protection != PROT_RW || // RW
+		    !GetRegionInfo(self, frame->return_address, start, protection) ||
+		    protection != PROT_EXE || // EXE
+		    !PointsToValidCall(start, frame->return_address) )// follows a CALL
+		{
+			/* This is not a valid frame but we might be in code compiled with
+			 * -fomit-frame-pointer so look at each address on the stack that is
+			 * 4 bytes below a 16 byte boundary. */
+			if( (((uintptr_t)frame+4) & 0xF) == 0 )
 			{
-			    // Invalid.
 				void *p = *(void **)frame;
 				if( GetRegionInfo(self, p, start, protection) &&
-				(protection & (VM_PROT_READ|VM_PROT_EXECUTE))==(VM_PROT_READ|VM_PROT_EXECUTE) &&
+				    protection == PROT_EXE &&
 				    PointsToValidCall(start, p) )
 				{
 					buf[i++] = p;
 				}
+			}
 			frame = (Frame *)(intptr_t(frame)+4);
 			continue;
 		}
@@ -583,6 +614,8 @@ void GetBacktrace( const void **buf, size_t size, const BacktraceContext *ctx )
 
 	buf[i] = NULL;	
 }
+#undef PROT_RW
+#undef PROT_EXE
 
 #elif defined(BACKTRACE_METHOD_POWERPC_DARWIN)
 struct Frame