[PATCH v2] Compute proper length for dynamic types of TYPE_CODE_TYPEDEF

[Keith Seitz <keiths at redhat dot com>]

This bug has recently appeared again and someone reminded me that this patch
is still outstanding (thanks, Jan!). Pedro reviewed this in June.

Revisions in v2:
- Added some comments to the test case
- Renamed types/variables in the test case
- Added initializer for previously uninitialized structure member

Keith
-----

This patch fixes gdb/21356 in which we hit an assertion in
value_contents_bits_eq:

(gdb) p container_object2
(gdb) p container_object2
$1 = {_container_member2 = 15, _vla_struct_object2 = {_some_member = 0,
    _vla_field = {
../../src/gdb/value.c:829: internal-error: int value_contents_bits_eq(const value*, int, const value*, int, int): Assertion `offset1 + length <= TYPE_LENGTH (val1->enclosing_type) * TARGET_CHAR_BIT' failed.
A problem internal to GDB has been detected,
further debugging may prove unreliable.
Quit this debugging session? (y or n)

This is happening because TYPE_LENGTH (val1->enclosing_type) is erroneously
based on enclosing_type, which is a typedef, instead of the actual underlying
type.

This can be traced back to resolve_dynamic_struct, where the size of the
type is computed:

  2093        TYPE_FIELD_TYPE (resolved_type, i)
  2094          = resolve_dynamic_type_internal (TYPE_FIELD_TYPE (resolved_type, i),
  2095                                           &pinfo, 0);
  2096        gdb_assert (TYPE_FIELD_LOC_KIND (resolved_type, i)
  2097                    == FIELD_LOC_KIND_BITPOS);
  2098
  2099        new_bit_length = TYPE_FIELD_BITPOS (resolved_type, i);
  2100        if (TYPE_FIELD_BITSIZE (resolved_type, i) != 0)
  2101          new_bit_length += TYPE_FIELD_BITSIZE (resolved_type, i);
  2102        else
  2103          new_bit_length += (TYPE_LENGTH (TYPE_FIELD_TYPE (resolved_type, i))
  2104                             * TARGET_CHAR_BIT);

In this function, resolved_type is TYPE_CODE_TYPEDEF which is not what we
want to use to calculate the size of the actual field.

This patch fixes this and the similar problem in resolve_dynamic_union.

gdb/ChangeLog:

	PR gdb/21356
	* gdbtypes.c (resolve_dynamic_union, resolve_dynamic_struct):
	Resolve typedefs for type length calculations.

gdb/testsuite/ChangeLog:

	PR gdb/21356
	* gdb.base/vla-datatypes.c (vla_factory): Add typedef for struct
	vla_struct.
	Add new struct vla_struct_with_vla_typedef and union
	vla_union_with_vla_typedef and corresponding instantiation objects.
	Initialize new objects.
	* gdb.base/vla-datatypes.exp: Add tests for
	vla_struct_with_vla_typedef_object and
        vla_union_with_vla_typedef_object.
	Fixup type for vla_struct_object.
---
 gdb/gdbtypes.c                           | 14 ++++++++++----
 gdb/testsuite/gdb.base/vla-datatypes.c   | 28 ++++++++++++++++++++++++++--
 gdb/testsuite/gdb.base/vla-datatypes.exp |  6 +++++-
 3 files changed, 41 insertions(+), 7 deletions(-)

diff --git a/gdb/gdbtypes.c b/gdb/gdbtypes.c
index 01ab6fa8c7..0df5ee1286 100644
--- a/gdb/gdbtypes.c
+++ b/gdb/gdbtypes.c
@@ -2036,8 +2036,10 @@ resolve_dynamic_union (struct type *type,
       t = resolve_dynamic_type_internal (TYPE_FIELD_TYPE (resolved_type, i),
 					 addr_stack, 0);
       TYPE_FIELD_TYPE (resolved_type, i) = t;
-      if (TYPE_LENGTH (t) > max_len)
-	max_len = TYPE_LENGTH (t);
+
+      struct type *real_type = check_typedef (t);
+      if (TYPE_LENGTH (real_type) > max_len)
+	max_len = TYPE_LENGTH (real_type);
     }
 
   TYPE_LENGTH (resolved_type) = max_len;
@@ -2103,8 +2105,12 @@ resolve_dynamic_struct (struct type *type,
       if (TYPE_FIELD_BITSIZE (resolved_type, i) != 0)
 	new_bit_length += TYPE_FIELD_BITSIZE (resolved_type, i);
       else
-	new_bit_length += (TYPE_LENGTH (TYPE_FIELD_TYPE (resolved_type, i))
-			   * TARGET_CHAR_BIT);
+	{
+	  struct type *real_type
+	    = check_typedef (TYPE_FIELD_TYPE (resolved_type, i));
+
+	  new_bit_length += (TYPE_LENGTH (real_type) * TARGET_CHAR_BIT);
+	}
 
       /* Normally, we would use the position and size of the last field
 	 to determine the size of the enclosing structure.  But GCC seems
diff --git a/gdb/testsuite/gdb.base/vla-datatypes.c b/gdb/testsuite/gdb.base/vla-datatypes.c
index 4902282e16..b98fe0da90 100644
--- a/gdb/testsuite/gdb.base/vla-datatypes.c
+++ b/gdb/testsuite/gdb.base/vla-datatypes.c
@@ -46,11 +46,14 @@ vla_factory (int n)
   BAR             bar_vla[n];
   int i;
 
-  struct vla_struct
+  /* Define a typedef for a VLA structure.  */
+  typedef struct vla_struct
   {
     int something;
     int vla_field[n];
-  } vla_struct_object;
+  } vla_struct_t;
+
+  vla_struct_t vla_struct_object;
 
   struct inner_vla_struct
   {
@@ -59,14 +62,33 @@ vla_factory (int n)
     int after;
   } inner_vla_struct_object;
 
+  /* Define a structure which uses a typedef for the VLA field
+     to make sure that GDB creates the proper type for this field,
+     preventing a possible assertion failure (see gdb/21356).  */
+  struct vla_struct_with_vla_typedef
+  {
+    int something;
+    vla_struct_t vla_object;
+  } vla_struct_with_vla_typedef_object;
+
   union vla_union
   {
     int vla_field[n];
   } vla_union_object;
 
+  /* Like vla_struct_with_vla_typedef but a union type.  */
+  union vla_union_with_vla_typedef
+  {
+    int something;
+    vla_struct_t vla_object;
+  } vla_union_with_vla_typedef_object;
+
   vla_struct_object.something = n;
   inner_vla_struct_object.something = n;
   inner_vla_struct_object.after = n;
+  vla_struct_with_vla_typedef_object.something = n * 2;
+  vla_struct_with_vla_typedef_object.vla_object.something = n * 3;
+  vla_union_with_vla_typedef_object.vla_object.something = n * 3 + 1;
   for (i = 0; i < n; i++)
     {
       int_vla[i] = i*2;
@@ -85,6 +107,8 @@ vla_factory (int n)
       vla_struct_object.vla_field[i] = i*2;
       vla_union_object.vla_field[i] = i*2;
       inner_vla_struct_object.vla_field[i] = i*2;
+      vla_struct_with_vla_typedef_object.vla_object.vla_field[i] = i * 3;
+      vla_union_with_vla_typedef_object.vla_object.vla_field[i] = i * 3 - 1;
     }
 
   size_t int_size        = sizeof(int_vla);     /* vlas_filled */
diff --git a/gdb/testsuite/gdb.base/vla-datatypes.exp b/gdb/testsuite/gdb.base/vla-datatypes.exp
index d32ed5afce..e943c4aac9 100644
--- a/gdb/testsuite/gdb.base/vla-datatypes.exp
+++ b/gdb/testsuite/gdb.base/vla-datatypes.exp
@@ -57,6 +57,10 @@ gdb_test "print vla_struct_object" \
     "\\\{something = 5, vla_field = \\\{0, 2, 4, 6, 8\\\}\\\}"
 gdb_test "print vla_union_object" \
     "\\\{vla_field = \\\{0, 2, 4, 6, 8\\\}\\\}"
+gdb_test "print vla_struct_with_vla_typedef_object" \
+    "\\\{something = 10, vla_object = \\\{something = 15, vla_field = \\\{0, 3, 6, 9, 12\\\}\\\}\\\}"
+gdb_test "print vla_union_with_vla_typedef_object" \
+    "\\\{something = 16, vla_object = \\\{something = 16, vla_field = \\\{-1, 2, 5, 8, 11\\\}\\\}\\\}"
 
 # Check whatis of VLA's.
 gdb_test "whatis int_vla" "type = int \\\[5\\\]" "whatis int_vla"
@@ -78,7 +82,7 @@ gdb_test "whatis unsigned_char_vla" "type = unsigned char \\\[5\\\]" \
          "whatis unsigned_char_vla"
 gdb_test "whatis foo_vla" "type = struct foo \\\[5\\\]" "whatis foo_vla"
 gdb_test "whatis bar_vla" "type = BAR \\\[5\\\]" "whatis bar_vla"
-gdb_test "whatis vla_struct_object" "type = struct vla_struct"
+gdb_test "whatis vla_struct_object" "type = vla_struct_t"
 gdb_test "whatis vla_union_object" "type = union vla_union"
 
 # Check ptype of VLA's.
-- 
2.13.6