[PATCH] Fix setting function return values for x86 targets

[Mark Kettenis <kettenis at wins dot uva dot nl>]

FYI, this patch fixes some problems uncovered by Michaels new
gdb.base/return2.exp tests for the x86.  STORE_RETURN_VALUE didn't
handle long long's and floating point values correctly.  There are
still some problems with floating point values (similar to the
problems with extracting return values, such that some of the new
tests still fail.  I'll try to come up with a fix, which should be
possible for native GDB (or x86 x x86 cross).

Tested on i586-pc-linux-gnu & checked in.

Mark


2000-12-21  Mark Kettenis  <kettenis@gnu.org>

	* i386-tdep.c: Add missing ')' in comment.
	(i386_extract_return_value): Return directly after issuing the
	warning and filling *VALBUF with zeroes if we cannot get at the
	floating-point registers.
	(i386_store_return_value): New function.
	* config/i386/tm-i386.h (STORE_RETURN_VALUE): Simply call
	i386_store_return_value.
	Add prototype for i386_store_return_value.


Index: i386-tdep.c
===================================================================
RCS file: /cvs/src/src/gdb/i386-tdep.c,v
retrieving revision 1.15
diff -u -p -r1.15 i386-tdep.c
--- i386-tdep.c	2000/07/30 01:48:25	1.15
+++ i386-tdep.c	2000/12/21 20:35:08
@@ -713,7 +713,7 @@ get_longjmp_target (CORE_ADDR *pc)
 
 /* These registers are used for returning integers (and on some
    targets also for returning `struct' and `union' values when their
-   size and alignment match an integer type.  */
+   size and alignment match an integer type).  */
 #define LOW_RETURN_REGNUM 0	/* %eax */
 #define HIGH_RETURN_REGNUM 2	/* %edx */
 
@@ -732,6 +732,7 @@ i386_extract_return_value (struct type *
 	{
 	  warning ("Cannot find floating-point return value.");
 	  memset (valbuf, 0, len);
+	  return;
 	}
 
       /* Floating-point return values can be found in %st(0).  */
@@ -771,6 +772,64 @@ i386_extract_return_value (struct type *
 	}
       else
 	internal_error ("Cannot extract return value of %d bytes long.", len);
+    }
+}
+
+/* Write into the appropriate registers a function return value stored
+   in VALBUF of type TYPE, given in virtual format.  */
+
+void
+i386_store_return_value (struct type *type, char *valbuf)
+{
+  int len = TYPE_LENGTH (type);
+
+  if (TYPE_CODE_FLT == TYPE_CODE (type))
+    {
+      if (NUM_FREGS == 0)
+	{
+	  warning ("Cannot set floating-point return value.");
+	  return;
+	}
+
+      /* Floating-point return values can be found in %st(0).  */
+      if (len == TARGET_LONG_DOUBLE_BIT / TARGET_CHAR_BIT
+	  && TARGET_LONG_DOUBLE_FORMAT == &floatformat_i387_ext)
+	{
+	  /* Copy straight over.  */
+	  write_register_bytes (REGISTER_BYTE (FP0_REGNUM), valbuf,
+				FPU_REG_RAW_SIZE);
+	}
+      else
+	{
+	  char buf[FPU_REG_RAW_SIZE];
+	  DOUBLEST val;
+
+	  /* Convert the value found in VALBUF to the extended
+             floating point format used by the FPU.  This is probably
+             not exactly how it would happen on the target itself, but
+             it is the best we can do.  */
+	  val = extract_floating (valbuf, TYPE_LENGTH (type));
+	  floatformat_from_doublest (&floatformat_i387_ext, &val, buf);
+	  write_register_bytes (REGISTER_BYTE (FP0_REGNUM), buf,
+				FPU_REG_RAW_SIZE);
+	}
+    }
+  else
+    {
+      int low_size = REGISTER_RAW_SIZE (LOW_RETURN_REGNUM);
+      int high_size = REGISTER_RAW_SIZE (HIGH_RETURN_REGNUM);
+
+      if (len <= low_size)
+	write_register_bytes (REGISTER_BYTE (LOW_RETURN_REGNUM), valbuf, len);
+      else if (len <= (low_size + high_size))
+	{
+	  write_register_bytes (REGISTER_BYTE (LOW_RETURN_REGNUM),
+				valbuf, low_size);
+	  write_register_bytes (REGISTER_BYTE (HIGH_RETURN_REGNUM),
+				valbuf + low_size, len - low_size);
+	}
+      else
+	internal_error ("Cannot store return value of %d bytes long.", len);
     }
 }
 
Index: config/i386/tm-i386.h
===================================================================
RCS file: /cvs/src/src/gdb/config/i386/tm-i386.h,v
retrieving revision 1.8
diff -u -p -r1.8 tm-i386.h
--- config/i386/tm-i386.h	2000/07/21 20:52:27	1.8
+++ config/i386/tm-i386.h	2000/12/21 20:35:10
@@ -296,17 +296,11 @@ extern void i387_float_info (void);
 extern void i386_extract_return_value (struct type *type, char *regbuf,
 				       char *valbuf);
 
-/* Write into appropriate registers a function return value of type TYPE, given
-   in virtual format.  */
-
-#define STORE_RETURN_VALUE(TYPE,VALBUF) \
-  {    	       	       	       	       	       	       	       	       	     \
-    if (TYPE_CODE (TYPE) == TYPE_CODE_FLT)				     \
-      write_register_bytes (REGISTER_BYTE (FP0_REGNUM), (VALBUF),	     \
-			    TYPE_LENGTH (TYPE));			     \
-    else								     \
-      write_register_bytes (0, (VALBUF), TYPE_LENGTH (TYPE));  		     \
-  }
+/* Write into the appropriate registers a function return value stored
+   in VALBUF of type TYPE, given in virtual format.  */
+#define STORE_RETURN_VALUE(type, valbuf) \
+  i386_store_return_value ((type), (valbuf))
+extern void i386_store_return_value (struct type *type, char *valbuf);
 
 /* Extract from an array REGBUF containing the (raw) register state the address
    in which a function should return its structure value, as a CORE_ADDR (or an