From 644c3b8f01f2249dd552cda9e2429b479ebe0af8 Mon Sep 17 00:00:00 2001
From: xiezhiheng <xiezhiheng@huawei.com>
Date: Thu, 15 Jul 2021 21:44:03 -0400
Subject: [PATCH 15/22] [CompleteStructReorg] Complete Structure Reorganization

Introduce complete structure reorganization based on original
structure reorganization optimization, which change array of
structure to structure of array in order to better utilize
spatial locality.

diff --git a/gcc/ipa-struct-reorg/escapes.def b/gcc/ipa-struct-reorg/escapes.def
index 929279c8faf..9020cc48952 100644
--- a/gcc/ipa-struct-reorg/escapes.def
+++ b/gcc/ipa-struct-reorg/escapes.def
@@ -56,5 +56,7 @@ DEF_ESCAPE (escape_non_optimize, "Type used by a function which turns off struct
 DEF_ESCAPE (escape_array, "Type is used in an array [not handled yet]")
 DEF_ESCAPE (escape_ptr_ptr, "Type is used in a pointer to a pointer [not handled yet]")
 DEF_ESCAPE (escape_return, "Type escapes via a return [not handled yet]")
+DEF_ESCAPE (escape_separate_instance, "Type escapes via a separate instance")
+DEF_ESCAPE (escape_unhandled_rewrite, "Type escapes via a unhandled rewrite stmt")
 
 #undef DEF_ESCAPE
diff --git a/gcc/ipa-struct-reorg/ipa-struct-reorg.c b/gcc/ipa-struct-reorg/ipa-struct-reorg.c
index 8d1ddc82877..5a19ea0bb40 100644
--- a/gcc/ipa-struct-reorg/ipa-struct-reorg.c
+++ b/gcc/ipa-struct-reorg/ipa-struct-reorg.c
@@ -96,6 +96,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "ipa-struct-reorg.h"
 #include "tree-eh.h"
 #include "bitmap.h"
+#include "cfgloop.h"
 #include "ipa-param-manipulation.h"
 #include "tree-ssa-live.h"  /* For remove_unused_locals.  */
 
@@ -104,6 +105,7 @@ along with GCC; see the file COPYING3.  If not see
 namespace {
 
 using namespace struct_reorg;
+using namespace struct_relayout;
 
 /* Return true iff TYPE is stdarg va_list type.  */
 
@@ -159,6 +161,14 @@ handled_type (tree type)
   return false;
 }
 
+enum srmode
+{
+  NORMAL = 0,
+  COMPLETE_STRUCT_RELAYOUT
+};
+
+static bool is_result_of_mult (tree arg, tree *num, tree struct_size);
+
 } // anon namespace
 
 namespace struct_reorg {
@@ -248,7 +258,8 @@ srtype::srtype (tree type)
   : type (type),
     chain_type (false),
     escapes (does_not_escape),
-    visited (false)
+    visited (false),
+    has_alloc_array (0)
 {
   for (int i = 0; i < max_split; i++)
     newtype[i] = NULL_TREE;
@@ -448,13 +459,6 @@ srtype::dump (FILE *f)
       fn->simple_dump (f);
     }
   fprintf (f, "\n }\n");
-  fprintf (f, "\n field_sites = {");
-  FOR_EACH_VEC_ELT (field_sites, i, field)
-    {
-      fprintf (f, "  \n");
-      field->simple_dump (f);
-    }
-  fprintf (f, "\n }\n");
   fprintf (f, "}\n");
 }
 
@@ -598,15 +602,7 @@ srtype::create_new_type (void)
 
   maxclusters++;
 
-  const char *tname = NULL;
-
-  if (TYPE_NAME (type) != NULL)
-    {
-      if (TREE_CODE (TYPE_NAME (type)) == IDENTIFIER_NODE)
-        tname = IDENTIFIER_POINTER (TYPE_NAME (type));
-      else if (DECL_NAME (TYPE_NAME (type)) != NULL)
-        tname = IDENTIFIER_POINTER (DECL_NAME (TYPE_NAME (type)));
-    }
+  const char *tname = get_type_name (type);
 
   for (unsigned i = 0; i < maxclusters; i++)
     {
@@ -620,7 +616,8 @@ srtype::create_new_type (void)
       if (tname) 
 	{
           name = concat (tname, ".reorg.", id, NULL);
-          TYPE_NAME (newtype[i]) = get_identifier (name);
+	  TYPE_NAME (newtype[i]) = build_decl (UNKNOWN_LOCATION, TYPE_DECL,
+				   get_identifier (name), newtype[i]);
           free (name);
         }
     }
@@ -641,6 +638,10 @@ srtype::create_new_type (void)
     {
       TYPE_FIELDS (newtype[i]) = newfields[i];
       layout_type (newtype[i]);
+      if (TYPE_NAME (newtype[i]) != NULL)
+	{
+	  layout_decl (TYPE_NAME (newtype[i]), 0);
+	}
     }
 
   warn_padded = save_warn_padded;
@@ -805,12 +806,6 @@ srfield::dump (FILE *f)
   fprintf (f, ", offset = " HOST_WIDE_INT_PRINT_DEC, offset);
   fprintf (f, ", type = ");
   print_generic_expr (f, fieldtype);
-  if (type)
-    {
-      fprintf (f, "( srtype = ");
-      type->simple_dump (f);
-      fprintf (f, ")");
-    }
   fprintf (f, "\n}\n");
 }
 
@@ -820,7 +815,10 @@ srfield::dump (FILE *f)
 void
 srfield::simple_dump (FILE *f)
 {
-  fprintf (f, "field (%d)", DECL_UID (fielddecl));
+  if (fielddecl)
+    {
+      fprintf (f, "field (%d)", DECL_UID (fielddecl));
+    }
 }
 
 /* Dump out the access structure to FILE. */
@@ -864,21 +862,123 @@ srdecl::dump (FILE *file)
 
 } // namespace struct_reorg
 
+namespace struct_relayout {
+
+/* Complete Structure Relayout Optimization.
+   It reorganizes all structure members, and puts same member together.
+   struct s {
+     long a;
+     int b;
+     struct s* c;
+   };
+   Array looks like
+     abcabcabcabc...
+   will be transformed to
+     aaaa...bbbb...cccc...
+*/
+
+#define GPTR_SIZE(i) \
+  TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (gptr[i])))
+
+unsigned transformed = 0;
+
+unsigned
+csrtype::calculate_field_num (tree field_offset)
+{
+  if (field_offset == NULL)
+    {
+      return 0;
+    }
+
+  HOST_WIDE_INT off = int_byte_position (field_offset);
+  unsigned i = 1;
+  for (tree field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
+    {
+      if (off == int_byte_position (field))
+	{
+	  return i;
+	}
+      i++;
+    }
+  return 0;
+}
+
+void
+csrtype::init_type_info (void)
+{
+  if (!type)
+    {
+      return;
+    }
+  new_size = old_size = tree_to_uhwi (TYPE_SIZE_UNIT (type));
+
+  /* Close enough to pad to improve performance.
+     33~63 should pad to 64 but 33~48 (first half) are too far away, and
+     65~127 should pad to 128 but 65~96 (first half) are too far away.  */
+  if (old_size > 48 && old_size < 64)
+    {
+      new_size = 64;
+    }
+  if (old_size > 96 && old_size < 128)
+    {
+      new_size = 128;
+    }
+
+  /* For performance reasons, only allow structure size
+     that is a power of 2 and not too big.  */
+  if (new_size != 1 && new_size != 2
+      && new_size != 4 && new_size != 8
+      && new_size != 16 && new_size != 32
+      && new_size != 64 && new_size != 128)
+    {
+      new_size = 0;
+      field_count = 0;
+      return;
+    }
+
+  unsigned i = 0;
+  for (tree field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
+    {
+      if (TREE_CODE (field) == FIELD_DECL)
+	{
+	  i++;
+	}
+    }
+  field_count = i;
+
+  struct_size = build_int_cstu (TREE_TYPE (TYPE_SIZE_UNIT (type)),
+				new_size);
+
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    {
+      fprintf (dump_file, "Type: ");
+      print_generic_expr (dump_file, type);
+      fprintf (dump_file, " has %d members.\n", field_count);
+      fprintf (dump_file, "Modify struct size from %ld to %ld.\n",
+			  old_size, new_size);
+    }
+}
+
+} // namespace struct_relayout
+
 namespace {
 
+/* Structure definition for ipa_struct_reorg and ipa_struct_relayout.  */
+
 struct ipa_struct_reorg
 {
+public:
   // Constructors
   ipa_struct_reorg(void)
     : current_function (NULL),
-      done_recording(false)
+      done_recording (false),
+      current_mode (NORMAL)
   {
   }
 
-  // public methods
-  unsigned execute(void);
+  unsigned execute (enum srmode mode);
   void mark_type_as_escape (tree type, escape_type, gimple *stmt = NULL);
-private:
+
   // fields
   auto_vec_del<srtype> types;
   auto_vec_del<srfunction> functions;
@@ -886,8 +986,8 @@ private:
   srfunction *current_function;
 
   bool done_recording;
+  srmode current_mode;
 
-  // private methods
   void dump_types (FILE *f);
   void dump_types_escaped (FILE *f);
   void dump_functions (FILE *f);
@@ -917,6 +1017,7 @@ private:
   void maybe_record_allocation_site (cgraph_node *, gimple *);
   void record_stmt_expr (tree expr, cgraph_node *node, gimple *stmt);
   void mark_expr_escape(tree, escape_type, gimple *stmt);
+  bool handled_allocation_stmt (gimple *stmt);
   tree allocate_size (srtype *t, gimple *stmt);
 
   void mark_decls_in_as_not_needed (tree fn);
@@ -932,6 +1033,7 @@ private:
   bool get_type_field (tree expr, tree &base, bool &indirect, srtype *&type, srfield *&field, bool &realpart, bool &imagpart, bool &address, bool should_create = false, bool can_escape = false);
   bool wholeaccess (tree expr, tree base, tree accesstype, srtype *t);
 
+  void check_alloc_num (gimple *stmt, srtype *type);
   void check_definition (srdecl *decl, vec<srdecl*>&);
   void check_uses (srdecl *decl, vec<srdecl*>&);
   void check_use (srdecl *decl, gimple *stmt, vec<srdecl*>&);
@@ -944,8 +1046,644 @@ private:
 
   bool has_rewritten_type (srfunction*);
   void maybe_mark_or_record_other_side (tree side, tree other, gimple *stmt);
+  unsigned execute_struct_relayout (void);
+};
+
+struct ipa_struct_relayout
+{
+public:
+  // fields
+  tree gptr[max_relayout_split + 1];
+  csrtype ctype;
+  ipa_struct_reorg *sr;
+  cgraph_node *current_node;
+
+  // Constructors
+  ipa_struct_relayout (tree type, ipa_struct_reorg *sr_)
+  {
+    ctype.type = type;
+    sr = sr_;
+    current_node = NULL;
+    for (int i = 0; i < max_relayout_split + 1; i++)
+      {
+	gptr[i] = NULL;
+      }
+  }
+
+  // Methods
+  tree create_new_vars (tree type, const char *name);
+  void create_global_ptrs (void);
+  unsigned int rewrite (void);
+  void rewrite_stmt_in_function (void);
+  bool rewrite_debug (gimple *stmt, gimple_stmt_iterator *gsi);
+  bool rewrite_stmt (gimple *stmt, gimple_stmt_iterator *gsi);
+  bool handled_allocation_stmt (gcall *stmt);
+  void init_global_ptrs (gcall *stmt, gimple_stmt_iterator *gsi);
+  bool check_call_uses (gcall *stmt);
+  bool rewrite_call (gcall *stmt, gimple_stmt_iterator *gsi);
+  tree create_ssa (tree node, gimple_stmt_iterator *gsi);
+  bool is_candidate (tree xhs);
+  tree rewrite_address (tree xhs, gimple_stmt_iterator *gsi);
+  tree rewrite_offset (tree offset, HOST_WIDE_INT num);
+  bool rewrite_assign (gassign *stmt, gimple_stmt_iterator *gsi);
+  bool maybe_rewrite_cst (tree cst, gimple_stmt_iterator *gsi,
+				    HOST_WIDE_INT &times);
+  unsigned int execute (void);
 };
 
+} // anon namespace
+
+namespace {
+
+/* Methods for ipa_struct_relayout.  */
+
+static void
+set_var_attributes (tree var)
+{
+  if (!var)
+    {
+      return;
+    }
+  gcc_assert (TREE_CODE (var) == VAR_DECL);
+
+  DECL_ARTIFICIAL (var) = 1;
+  DECL_EXTERNAL (var) = 0;
+  TREE_STATIC (var) = 1;
+  TREE_PUBLIC (var) = 0;
+  TREE_USED (var) = 1;
+  DECL_CONTEXT (var) = NULL;
+  TREE_THIS_VOLATILE (var) = 0;
+  TREE_ADDRESSABLE (var) = 0;
+  TREE_READONLY (var) = 0;
+  if (is_global_var (var))
+    {
+      set_decl_tls_model (var, TLS_MODEL_NONE);
+    }
+}
+
+tree
+ipa_struct_relayout::create_new_vars (tree type, const char *name)
+{
+  gcc_assert (type);
+  tree new_type = build_pointer_type (type);
+
+  tree new_name = NULL;
+  if (name)
+    {
+      new_name = get_identifier (name);
+    }
+
+  tree new_var = build_decl (UNKNOWN_LOCATION, VAR_DECL, new_name, new_type);
+
+  /* set new_var's attributes.  */
+  set_var_attributes (new_var);
+
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    {
+      fprintf (dump_file, "Created new var: ");
+      print_generic_expr (dump_file, new_var);
+      fprintf (dump_file, "\n");
+    }
+  return new_var;
+}
+
+void
+ipa_struct_relayout::create_global_ptrs (void)
+{
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    {
+      fprintf (dump_file, "Create global gptrs: {\n");
+    }
+
+  char *gptr0_name = NULL;
+  const char *type_name = get_type_name (ctype.type);
+
+  if (type_name)
+    {
+      gptr0_name = concat (type_name, "_gptr0", NULL);
+    }
+  tree var_gptr0 = create_new_vars (ctype.type, gptr0_name);
+  gptr[0] = var_gptr0;
+  varpool_node::add (var_gptr0);
+
+  unsigned i = 1;
+  for (tree field = TYPE_FIELDS (ctype.type); field;
+	    field = DECL_CHAIN (field))
+    {
+      if (TREE_CODE (field) == FIELD_DECL)
+	{
+	  tree type = TREE_TYPE (field);
+
+	  char *name = NULL;
+	  char id[10] = {0};
+	  sprintf (id, "%d", i);
+	  const char *decl_name = IDENTIFIER_POINTER (DECL_NAME (field));
+
+	  if (type_name && decl_name)
+	    {
+	      name = concat (type_name, "_", decl_name, "_gptr", id, NULL);
+	    }
+	  tree var = create_new_vars (type, name);
+
+	  gptr[i] = var;
+	  varpool_node::add (var);
+	  i++;
+	}
+    }
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    {
+      fprintf (dump_file, "\nTotally create %d gptrs. }\n\n", i);
+    }
+  gcc_assert (ctype.field_count == i - 1);
+}
+
+void
+ipa_struct_relayout::rewrite_stmt_in_function (void)
+{
+  gcc_assert (cfun);
+
+  basic_block bb = NULL;
+  gimple_stmt_iterator si;
+  FOR_EACH_BB_FN (bb, cfun)
+    {
+      for (si = gsi_start_bb (bb); !gsi_end_p (si);)
+	{
+	  gimple *stmt = gsi_stmt (si);
+	  if (rewrite_stmt (stmt, &si))
+	    {
+	      gsi_remove (&si, true);
+	    }
+	  else
+	    {
+	      gsi_next (&si);
+	    }
+	}
+    }
+
+  /* Debug statements need to happen after all other statements
+     have changed.  */
+  FOR_EACH_BB_FN (bb, cfun)
+    {
+      for (si = gsi_start_bb (bb); !gsi_end_p (si);)
+	{
+	  gimple *stmt = gsi_stmt (si);
+	  if (gimple_code (stmt) == GIMPLE_DEBUG
+	      && rewrite_debug (stmt, &si))
+	    {
+	      gsi_remove (&si, true);
+	    }
+	  else
+	    {
+	      gsi_next (&si);
+	    }
+	}
+    }
+}
+
+unsigned int
+ipa_struct_relayout::rewrite (void)
+{
+  cgraph_node *cnode = NULL;
+  function *fn = NULL;
+  FOR_EACH_FUNCTION (cnode)
+    {
+      if (!cnode->real_symbol_p () || !cnode->has_gimple_body_p ())
+	{
+	  continue;
+	}
+      if (cnode->definition)
+	{
+	  fn = DECL_STRUCT_FUNCTION (cnode->decl);
+	  if (fn == NULL)
+	    {
+	      continue;
+	    }
+
+	  current_node = cnode;
+	  push_cfun (fn);
+
+	  rewrite_stmt_in_function ();
+
+	  update_ssa (TODO_update_ssa_only_virtuals);
+
+	  if (flag_tree_pta)
+	    {
+	      compute_may_aliases ();
+	    }
+
+	  remove_unused_locals ();
+
+	  cgraph_edge::rebuild_edges ();
+
+	  free_dominance_info (CDI_DOMINATORS);
+
+	  pop_cfun ();
+	  current_node = NULL;
+	}
+    }
+  return TODO_verify_all;
+}
+
+bool
+ipa_struct_relayout::rewrite_debug (gimple *stmt, gimple_stmt_iterator *gsi)
+{
+  /* Delete debug gimple now.  */
+  return true;
+}
+
+bool
+ipa_struct_relayout::rewrite_stmt (gimple *stmt, gimple_stmt_iterator *gsi)
+{
+  switch (gimple_code (stmt))
+    {
+      case GIMPLE_ASSIGN:
+	return rewrite_assign (as_a <gassign *> (stmt), gsi);
+      case GIMPLE_CALL:
+	return rewrite_call (as_a <gcall *> (stmt), gsi);
+      default:
+	break;
+    }
+  return false;
+}
+
+bool
+ipa_struct_relayout::handled_allocation_stmt (gcall *stmt)
+{
+  if (gimple_call_builtin_p (stmt, BUILT_IN_CALLOC))
+    {
+      return true;
+    }
+  return false;
+}
+
+void
+ipa_struct_relayout::init_global_ptrs (gcall *stmt, gimple_stmt_iterator *gsi)
+{
+  gcc_assert (handled_allocation_stmt (stmt));
+
+  tree lhs = gimple_call_lhs (stmt);
+
+  /* Case that gimple is at the end of bb.  */
+  if (gsi_one_before_end_p (*gsi))
+    {
+      gassign *gptr0 = gimple_build_assign (gptr[0], lhs);
+      gsi_insert_after (gsi, gptr0, GSI_SAME_STMT);
+    }
+  gsi_next (gsi);
+
+  /* Emit gimple gptr0 = _X and gptr1 = _X.  */
+  gassign *gptr0 = gimple_build_assign (gptr[0], lhs);
+  gsi_insert_before (gsi, gptr0, GSI_SAME_STMT);
+  gassign *gptr1 = gimple_build_assign (gptr[1], lhs);
+  gsi_insert_before (gsi, gptr1, GSI_SAME_STMT);
+
+  /* Emit gimple gptr_[i] = gptr_[i-1] + _Y[gap].  */
+  for (unsigned i = 2; i <= ctype.field_count; i++)
+    {
+      gimple *new_stmt = NULL;
+      tree gptr_i_prev_ssa = create_ssa (gptr[i-1], gsi);
+      tree gptr_i_ssa = make_ssa_name (TREE_TYPE (gptr[i-1]));
+
+      /* Emit gimple _Y[gap] = N * sizeof (member).  */
+      tree member_gap = gimplify_build2 (gsi, MULT_EXPR,
+					 long_unsigned_type_node,
+					 gimple_call_arg (stmt, 0),
+					 GPTR_SIZE (i-1));
+
+      new_stmt = gimple_build_assign (gptr_i_ssa, POINTER_PLUS_EXPR,
+				      gptr_i_prev_ssa, member_gap);
+      gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
+
+      gassign *gptr_i = gimple_build_assign (gptr[i], gptr_i_ssa);
+      gsi_insert_before (gsi, gptr_i, GSI_SAME_STMT);
+    }
+  gsi_prev (gsi);
+}
+
+bool
+ipa_struct_relayout::check_call_uses (gcall *stmt)
+{
+  gcc_assert (current_node);
+  srfunction *fn = sr->find_function (current_node);
+  tree lhs = gimple_call_lhs (stmt);
+
+  if (fn == NULL)
+    {
+      return false;
+    }
+
+  srdecl *d = fn->find_decl (lhs);
+  if (d == NULL)
+    {
+      return false;
+    }
+  if (types_compatible_p (d->type->type, ctype.type))
+    {
+      return true;
+    }
+
+  return false;
+}
+
+bool
+ipa_struct_relayout::rewrite_call (gcall *stmt, gimple_stmt_iterator *gsi)
+{
+  if (handled_allocation_stmt (stmt))
+    {
+      /* Rewrite stmt _X = calloc (N, sizeof (struct)).  */
+      tree size = gimple_call_arg (stmt, 1);
+      if (TREE_CODE (size) != INTEGER_CST)
+	{
+	  return false;
+	}
+      if (tree_to_uhwi (size) != ctype.old_size)
+	{
+	  return false;
+	}
+      if (!check_call_uses (stmt))
+	{
+	  return false;
+	}
+
+      if (dump_file && (dump_flags & TDF_DETAILS))
+	{
+	  fprintf (dump_file, "Rewrite allocation call:\n");
+	  print_gimple_stmt (dump_file, stmt, 0);
+	  fprintf (dump_file, "to\n");
+	}
+
+      /* Modify sizeof (struct).  */
+      gimple_call_set_arg (stmt, 1, ctype.struct_size);
+      update_stmt (stmt);
+
+      if (dump_file && (dump_flags & TDF_DETAILS))
+	{
+	  print_gimple_stmt (dump_file, stmt, 0);
+	  fprintf (dump_file, "\n");
+	}
+
+      init_global_ptrs (stmt, gsi);
+    }
+  return false;
+}
+
+tree
+ipa_struct_relayout::create_ssa (tree node, gimple_stmt_iterator *gsi)
+{
+  gcc_assert (TREE_CODE (node) == VAR_DECL);
+  tree node_ssa = make_ssa_name (TREE_TYPE (node));
+  gassign *stmt = gimple_build_assign (node_ssa, node);
+  gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
+  return node_ssa;
+}
+
+bool
+ipa_struct_relayout::is_candidate (tree xhs)
+{
+  if (TREE_CODE (xhs) != COMPONENT_REF)
+    {
+      return false;
+    }
+  tree mem = TREE_OPERAND (xhs, 0);
+  if (TREE_CODE (mem) == MEM_REF)
+    {
+      tree type = TREE_TYPE (mem);
+      if (types_compatible_p (type, ctype.type))
+	{
+	  return true;
+	}
+    }
+  return false;
+}
+
+tree
+ipa_struct_relayout::rewrite_address (tree xhs, gimple_stmt_iterator *gsi)
+{
+  tree mem_ref = TREE_OPERAND (xhs, 0);
+  tree pointer = TREE_OPERAND (mem_ref, 0);
+  tree pointer_offset = TREE_OPERAND (mem_ref, 1);
+  tree field = TREE_OPERAND (xhs, 1);
+
+  tree pointer_ssa = fold_convert (long_unsigned_type_node, pointer);
+  tree gptr0_ssa = fold_convert (long_unsigned_type_node, gptr[0]);
+
+  /* Emit gimple _X1 = ptr - gptr0.  */
+  tree step1 = gimplify_build2 (gsi, MINUS_EXPR, long_unsigned_type_node,
+				pointer_ssa, gptr0_ssa);
+
+  /* Emit gimple _X2 = _X1 / sizeof (struct).  */
+  tree step2 = gimplify_build2 (gsi, TRUNC_DIV_EXPR, long_unsigned_type_node,
+				step1, ctype.struct_size);
+
+  unsigned field_num = ctype.calculate_field_num (field);
+  gcc_assert (field_num > 0 && field_num <= ctype.field_count);
+
+  /* Emit gimple _X3 = _X2 * sizeof (member).  */
+  tree step3 = gimplify_build2 (gsi, MULT_EXPR, long_unsigned_type_node,
+				step2, GPTR_SIZE (field_num));
+
+  /* Emit gimple _X4 = gptr[I].  */
+  tree gptr_field_ssa = create_ssa (gptr[field_num], gsi);
+  tree new_address = make_ssa_name (TREE_TYPE (gptr[field_num]));
+  gassign *new_stmt = gimple_build_assign (new_address, POINTER_PLUS_EXPR,
+					   gptr_field_ssa, step3);
+  gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
+
+  /* MEM_REF with nonzero offset like
+       MEM[ptr + sizeof (struct)] = 0B
+     should be transformed to
+       MEM[gptr + sizeof (member)] = 0B
+  */
+  HOST_WIDE_INT size
+    = tree_to_shwi (TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_address))));
+  tree new_size = rewrite_offset (pointer_offset, size);
+  if (new_size)
+    {
+      TREE_OPERAND (mem_ref, 1) = new_size;
+    }
+
+  /* Update mem_ref pointer.  */
+  TREE_OPERAND (mem_ref, 0) = new_address;
+
+  /* Update mem_ref TREE_TYPE.  */
+  TREE_TYPE (mem_ref) = TREE_TYPE (TREE_TYPE (new_address));
+
+  return mem_ref;
+}
+
+tree
+ipa_struct_relayout::rewrite_offset (tree offset, HOST_WIDE_INT num)
+{
+  if (TREE_CODE (offset) == INTEGER_CST)
+    {
+      bool sign = false;
+      HOST_WIDE_INT off = TREE_INT_CST_LOW (offset);
+      if (off == 0)
+	{
+	  return NULL;
+	}
+      if (off < 0)
+	{
+	  off = -off;
+	  sign = true;
+	}
+      if (off % ctype.old_size == 0)
+	{
+	  HOST_WIDE_INT times = off / ctype.old_size;
+	  times = sign ? -times : times;
+	  return build_int_cst (TREE_TYPE (offset), num * times);
+	}
+    }
+  return NULL;
+}
+
+#define REWRITE_ASSIGN_TREE_IN_STMT(node)		\
+do							\
+{							\
+  tree node = gimple_assign_##node (stmt);		\
+  if (node && is_candidate (node))			\
+    {							\
+      tree mem_ref = rewrite_address (node, gsi);	\
+      gimple_assign_set_##node (stmt, mem_ref);		\
+      update_stmt (stmt);				\
+    }							\
+} while (0)
+
+/*       COMPONENT_REF  = exp  =>     MEM_REF = exp
+	  /       \		      /     \
+       MEM_REF   field		    gptr   offset
+       /    \
+   pointer offset
+*/
+bool
+ipa_struct_relayout::rewrite_assign (gassign *stmt, gimple_stmt_iterator *gsi)
+{
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    {
+      fprintf (dump_file, "Maybe rewrite assign:\n");
+      print_gimple_stmt (dump_file, stmt, 0);
+      fprintf (dump_file, "to\n");
+    }
+
+  switch (gimple_num_ops (stmt))
+    {
+      case 4: REWRITE_ASSIGN_TREE_IN_STMT (rhs3);  // FALLTHRU
+      case 3:
+	{
+	  REWRITE_ASSIGN_TREE_IN_STMT (rhs2);
+	  tree rhs2 = gimple_assign_rhs2 (stmt);
+	  if (rhs2 && TREE_CODE (rhs2) == INTEGER_CST)
+	    {
+	      /* Handle pointer++ and pointer-- or
+		 factor is euqal to struct size.  */
+	      HOST_WIDE_INT times = 1;
+	      if (maybe_rewrite_cst (rhs2, gsi, times))
+		{
+		  tree tmp = build_int_cst (
+				TREE_TYPE (TYPE_SIZE_UNIT (ctype.type)),
+				ctype.new_size * times);
+		  gimple_assign_set_rhs2 (stmt, tmp);
+		  update_stmt (stmt);
+		}
+	    }
+	}  // FALLTHRU
+      case 2: REWRITE_ASSIGN_TREE_IN_STMT (rhs1);  // FALLTHRU
+      case 1: REWRITE_ASSIGN_TREE_IN_STMT (lhs);   // FALLTHRU
+      case 0: break;
+      default: gcc_unreachable ();
+    }
+
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    {
+      print_gimple_stmt (dump_file, stmt, 0);
+      fprintf (dump_file, "\n");
+    }
+  return false;
+}
+
+bool
+ipa_struct_relayout::maybe_rewrite_cst (tree cst, gimple_stmt_iterator *gsi,
+					HOST_WIDE_INT &times)
+{
+  bool ret = false;
+  gcc_assert (TREE_CODE (cst) == INTEGER_CST);
+
+  gimple *stmt = gsi_stmt (*gsi);
+  if (gimple_assign_rhs_code (stmt) == POINTER_PLUS_EXPR)
+    {
+      tree lhs = gimple_assign_lhs (stmt);
+      tree rhs1 = gimple_assign_rhs1 (stmt);
+      if (types_compatible_p (inner_type (TREE_TYPE (rhs1)), ctype.type)
+	  || types_compatible_p (inner_type (TREE_TYPE (lhs)), ctype.type))
+	{
+	  tree num = NULL;
+	  if (is_result_of_mult (cst, &num, TYPE_SIZE_UNIT (ctype.type)))
+	    {
+	      times = TREE_INT_CST_LOW (num);
+	      return true;
+	    }
+	}
+    }
+
+  if (gimple_assign_rhs_code (stmt) == MULT_EXPR)
+    {
+      if (gsi_one_before_end_p (*gsi))
+	{
+	  return false;
+	}
+      gsi_next (gsi);
+      gimple *stmt2 = gsi_stmt (*gsi);
+
+      if (gimple_code (stmt2) == GIMPLE_ASSIGN
+	  && gimple_assign_rhs_code (stmt2) == POINTER_PLUS_EXPR)
+	{
+	  tree lhs = gimple_assign_lhs (stmt2);
+	  tree rhs1 = gimple_assign_rhs1 (stmt2);
+	  if (types_compatible_p (inner_type (TREE_TYPE (rhs1)), ctype.type)
+	      || types_compatible_p (inner_type (TREE_TYPE (lhs)), ctype.type))
+	    {
+	      tree num = NULL;
+	      if (is_result_of_mult (cst, &num, TYPE_SIZE_UNIT (ctype.type)))
+		{
+		  times = TREE_INT_CST_LOW (num);
+		  ret = true;
+		}
+	    }
+	}
+      gsi_prev (gsi);
+      return ret;
+    }
+  return false;
+}
+
+unsigned int
+ipa_struct_relayout::execute (void)
+{
+  ctype.init_type_info ();
+  if (ctype.field_count < min_relayout_split
+      || ctype.field_count > max_relayout_split)
+    {
+      return 0;
+    }
+
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    {
+      fprintf (dump_file, "Complete Struct Relayout Type: ");
+      print_generic_expr (dump_file, ctype.type);
+      fprintf (dump_file, "\n");
+    }
+  transformed++;
+
+  create_global_ptrs ();
+  return rewrite ();
+}
+
+} // anon namespace
+
+namespace {
+
+/* Methods for ipa_struct_reorg.  */
+
 /* Dump all of the recorded types to file F. */
 
 void
@@ -1141,8 +1879,10 @@ ipa_struct_reorg::record_type (tree type)
 		  f->type = t1;
 		  t1->add_field_site (f);
 		}
-	      if (t1 == type1)
-		type1->mark_escape (escape_rescusive_type, NULL);
+	      if (t1 == type1 && current_mode != COMPLETE_STRUCT_RELAYOUT)
+		{
+		  type1->mark_escape (escape_rescusive_type, NULL);
+		}
 	    }
         }
     }
@@ -1279,6 +2019,14 @@ ipa_struct_reorg::record_var (tree decl, escape_type escapes, int arg)
       else
 	e = escape_type_volatile_array_or_ptrptr (TREE_TYPE (decl));
 
+      /* Separate instance is hard to trace in complete struct
+	 relayout optimization.  */
+      if (current_mode == COMPLETE_STRUCT_RELAYOUT
+	  && TREE_CODE (TREE_TYPE (decl)) == RECORD_TYPE)
+	{
+	  e = escape_separate_instance;
+	}
+
       if (e != does_not_escape)
 	type->mark_escape (e, NULL);
     }
@@ -1315,6 +2063,7 @@ ipa_struct_reorg::find_var (tree expr, gimple *stmt)
       || TREE_CODE (expr) == VIEW_CONVERT_EXPR)
     {
       tree r = TREE_OPERAND (expr, 0);
+      tree orig_type = TREE_TYPE (expr);
       if (handled_component_p (r)
           || TREE_CODE (r) == MEM_REF)
         {
@@ -1328,8 +2077,18 @@ ipa_struct_reorg::find_var (tree expr, gimple *stmt)
 				       escape_vce, stmt);
 		}
 	      if (TREE_CODE (r) == MEM_REF)
-		mark_type_as_escape (TREE_TYPE (TREE_OPERAND (r, 1)),
-				     escape_addr, stmt);
+		{
+		  mark_type_as_escape (TREE_TYPE (TREE_OPERAND (r, 1)),
+				       escape_addr, stmt);
+		  tree inner_type = TREE_TYPE (TREE_OPERAND (r, 0));
+		  if (orig_type != inner_type)
+		    {
+		      mark_type_as_escape (orig_type,
+					   escape_cast_another_ptr, stmt);
+		      mark_type_as_escape (inner_type,
+					   escape_cast_another_ptr, stmt);
+		    }
+		}
               r = TREE_OPERAND (r, 0);
 	    }
           mark_expr_escape (r, escape_addr, stmt);
@@ -1354,7 +2113,8 @@ ipa_struct_reorg::find_vars (gimple *stmt)
     {
     case GIMPLE_ASSIGN:
       if (gimple_assign_rhs_class (stmt) == GIMPLE_SINGLE_RHS
-          || gimple_assign_rhs_code (stmt) == POINTER_PLUS_EXPR)
+	  || gimple_assign_rhs_code (stmt) == POINTER_PLUS_EXPR
+	  || gimple_assign_rhs_code (stmt) == NOP_EXPR)
 	{
 	  tree lhs = gimple_assign_lhs (stmt);
 	  tree rhs = gimple_assign_rhs1 (stmt);
@@ -1379,6 +2139,32 @@ ipa_struct_reorg::find_vars (gimple *stmt)
 		current_function->record_decl (t, rhs, -1);
 	    }
 	}
+      else
+	{
+	  /* Because we won't handle these stmts in rewrite phase,
+	     just mark these types as escaped.  */
+	  switch (gimple_num_ops (stmt))
+	    {
+	      case 4: mark_type_as_escape (
+				TREE_TYPE (gimple_assign_rhs3 (stmt)),
+				escape_unhandled_rewrite, stmt);
+				// FALLTHRU
+	      case 3: mark_type_as_escape (
+				TREE_TYPE (gimple_assign_rhs2 (stmt)),
+				escape_unhandled_rewrite, stmt);
+				// FALLTHRU
+	      case 2: mark_type_as_escape (
+				TREE_TYPE (gimple_assign_rhs1 (stmt)),
+				escape_unhandled_rewrite, stmt);
+				// FALLTHRU
+	      case 1: mark_type_as_escape (
+				TREE_TYPE (gimple_assign_lhs (stmt)),
+				escape_unhandled_rewrite, stmt);
+				// FALLTHRU
+	      case 0: break;
+	      default: gcc_unreachable ();
+	    }
+	}
       break;
 
     case GIMPLE_CALL:
@@ -1460,9 +2246,23 @@ is_result_of_mult (tree arg, tree *num, tree struct_size)
   /* If we have a integer, just check if it is a multiply of STRUCT_SIZE. */
   if (TREE_CODE (arg) == INTEGER_CST)
     {
-      if (integer_zerop (size_binop (FLOOR_MOD_EXPR, arg, struct_size)))
+      bool sign = false;
+      HOST_WIDE_INT size = TREE_INT_CST_LOW (arg);
+      if (size < 0)
+	{
+	  size = -size;
+	  sign = true;
+	}
+      tree arg2 = build_int_cst (TREE_TYPE (arg), size);
+      if (integer_zerop (size_binop (FLOOR_MOD_EXPR, arg2, struct_size)))
 	{
-	  *num = size_binop (FLOOR_DIV_EXPR, arg, struct_size);
+	  tree number = size_binop (FLOOR_DIV_EXPR, arg2, struct_size);
+	  if (sign)
+	    {
+	      number = build_int_cst (TREE_TYPE (number),
+				      -tree_to_shwi (number));
+	    }
+	  *num = number;
 	  return true;
 	}
       return false;
@@ -1532,15 +2332,19 @@ is_result_of_mult (tree arg, tree *num, tree struct_size)
 
 /* Return TRUE if STMT is an allocation statement that is handled. */
 
-static bool
-handled_allocation_stmt (gimple *stmt)
+bool
+ipa_struct_reorg::handled_allocation_stmt (gimple *stmt)
 {
-  if (gimple_call_builtin_p (stmt, BUILT_IN_REALLOC)
-      || gimple_call_builtin_p (stmt, BUILT_IN_MALLOC)
-      || gimple_call_builtin_p (stmt, BUILT_IN_CALLOC)
-      || gimple_call_builtin_p (stmt, BUILT_IN_ALIGNED_ALLOC)
-      || gimple_call_builtin_p (stmt, BUILT_IN_ALLOCA)
-      || gimple_call_builtin_p (stmt, BUILT_IN_ALLOCA_WITH_ALIGN))
+  if (current_mode == COMPLETE_STRUCT_RELAYOUT
+      && gimple_call_builtin_p (stmt, BUILT_IN_CALLOC))
+    return true;
+  if (current_mode != COMPLETE_STRUCT_RELAYOUT
+      && (gimple_call_builtin_p (stmt, BUILT_IN_REALLOC)
+	  || gimple_call_builtin_p (stmt, BUILT_IN_MALLOC)
+	  || gimple_call_builtin_p (stmt, BUILT_IN_CALLOC)
+	  || gimple_call_builtin_p (stmt, BUILT_IN_ALIGNED_ALLOC)
+	  || gimple_call_builtin_p (stmt, BUILT_IN_ALLOCA)
+	  || gimple_call_builtin_p (stmt, BUILT_IN_ALLOCA_WITH_ALIGN)))
     return true;
   return false;
 }
@@ -1582,7 +2386,8 @@ ipa_struct_reorg::allocate_size (srtype *type, gimple *stmt)
       /* Check that second argument is a constant equal to the size of structure.  */
       if (operand_equal_p (arg1, struct_size, 0))
 	return size;
-      /* Check that first argument is a constant equal to the size of structure.  */
+      /* ??? Check that first argument is a constant
+	 equal to the size of structure.  */
       if (operand_equal_p (size, struct_size, 0))
 	return arg1;
       if (dump_file && (dump_flags & TDF_DETAILS))
@@ -1699,6 +2504,29 @@ ipa_struct_reorg::maybe_record_assign (cgraph_node *node, gassign *stmt)
     }
 }
 
+bool
+check_mem_ref_offset (tree expr)
+{
+  tree num = NULL;
+  bool ret = false;
+
+  if (TREE_CODE (expr) != MEM_REF)
+    {
+      return false;
+    }
+
+  /* Try to find the structure size.  */
+  tree field_off = TREE_OPERAND (expr, 1);
+  tree tmp = TREE_OPERAND (expr, 0);
+  if (TREE_CODE (tmp) == ADDR_EXPR)
+    {
+      tmp = TREE_OPERAND (tmp, 0);
+    }
+  tree size = TYPE_SIZE_UNIT (inner_type (TREE_TYPE (tmp)));
+  ret = is_result_of_mult (field_off, &num, size);
+  return ret;
+}
+
 tree
 get_ref_base_and_offset (tree &e, HOST_WIDE_INT &offset, bool &realpart, bool &imagpart, tree &accesstype)
 {
@@ -1738,7 +2566,10 @@ get_ref_base_and_offset (tree &e, HOST_WIDE_INT &offset, bool &realpart, bool &i
 	    gcc_assert (TREE_CODE (field_off) == INTEGER_CST);
 	    /* So we can mark the types as escaping if different. */
 	    accesstype = TREE_TYPE (field_off);
-	    offset += tree_to_uhwi (field_off);
+	    if (!check_mem_ref_offset (expr))
+	      {
+		offset += tree_to_uhwi (field_off);
+	      }
 	    return TREE_OPERAND (expr, 0);
 	  }
 	  default:
@@ -2115,6 +2946,39 @@ ipa_struct_reorg::check_type_and_push (tree newdecl, srtype *type, vec<srdecl*>
 
 }
 
+void
+ipa_struct_reorg::check_alloc_num (gimple *stmt, srtype *type)
+{
+  if (current_mode == COMPLETE_STRUCT_RELAYOUT
+      && handled_allocation_stmt (stmt))
+    {
+      tree arg0 = gimple_call_arg (stmt, 0);
+      basic_block bb = gimple_bb (stmt);
+      cgraph_node *node = current_function->node;
+      if (integer_onep (arg0))
+	{
+	  /* Actually NOT an array, but may ruin other array.  */
+	  type->has_alloc_array = -1;
+	}
+      else if (bb->loop_father != NULL
+	       && loop_outer (bb->loop_father) != NULL)
+	{
+	  /* The allocation is in a loop.  */
+	  type->has_alloc_array = -2;
+	}
+      else if (node->callers != NULL)
+	{
+	  type->has_alloc_array = -3;
+	}
+      else
+	{
+	  type->has_alloc_array = type->has_alloc_array < 0
+				  ? type->has_alloc_array
+				  : type->has_alloc_array + 1;
+	}
+    }
+}
+
 /*
   2) Check SSA_NAMEs for non type usages (source or use) (worlist of srdecl)
      a) if the SSA_NAME is sourced from a pointer plus, record the pointer and
@@ -2158,6 +3022,7 @@ ipa_struct_reorg::check_definition (srdecl *decl, vec<srdecl*> &worklist)
       if (!handled_allocation_stmt (stmt)
           || !allocate_size (type, stmt))
         type->mark_escape (escape_return, stmt);
+      check_alloc_num (stmt, type);
       return;
     }
   /* If the SSA_NAME is sourced from an inline-asm, just mark the type as escaping.  */
@@ -2196,6 +3061,24 @@ ipa_struct_reorg::check_definition (srdecl *decl, vec<srdecl*> &worklist)
       return;
     }
 
+  if (gimple_assign_rhs_code (stmt) == MAX_EXPR
+      || gimple_assign_rhs_code (stmt) == MIN_EXPR
+      || gimple_assign_rhs_code (stmt) == BIT_IOR_EXPR
+      || gimple_assign_rhs_code (stmt) == BIT_XOR_EXPR
+      || gimple_assign_rhs_code (stmt) == BIT_AND_EXPR)
+    {
+      tree rhs2 = gimple_assign_rhs2 (stmt);
+      if (TREE_CODE (rhs) == SSA_NAME)
+	{
+	  check_type_and_push (rhs, type, worklist, stmt);
+	}
+      if (TREE_CODE (rhs2) == SSA_NAME)
+	{
+	  check_type_and_push (rhs2, type, worklist, stmt);
+	}
+      return;
+    }
+
   /* Casts between pointers and integer are escaping.  */
   if (gimple_assign_cast_p (stmt))
     {
@@ -2258,6 +3141,13 @@ ipa_struct_reorg::check_other_side (srdecl *decl, tree other, gimple *stmt, vec<
   srtype *t1 = find_type (inner_type (t));
   if (t1 == type)
     {
+      /* In Complete Struct Relayout opti, if lhs type is the same
+	 as rhs type, we could return without any harm.  */
+      if (current_mode == COMPLETE_STRUCT_RELAYOUT)
+	{
+	  return;
+	}
+
       tree base;
       bool indirect;
       srtype *type1;
@@ -2305,8 +3195,11 @@ ipa_struct_reorg::check_use (srdecl *decl, gimple *stmt, vec<srdecl*> &worklist)
       tree rhs1 = gimple_cond_lhs (stmt);
       tree rhs2 = gimple_cond_rhs (stmt);
       tree orhs = rhs1;
-      if (gimple_cond_code (stmt) != EQ_EXPR
-          && gimple_cond_code (stmt) != NE_EXPR)
+      enum tree_code code = gimple_cond_code (stmt);
+      if (code != EQ_EXPR && code != NE_EXPR
+	  && (current_mode != COMPLETE_STRUCT_RELAYOUT
+	      || (code != LT_EXPR && code != LE_EXPR
+		  && code != GT_EXPR && code != GE_EXPR)))
 	{
 	  mark_expr_escape (rhs1, escape_non_eq, stmt);
 	  mark_expr_escape (rhs2, escape_non_eq, stmt);
@@ -2336,8 +3229,11 @@ ipa_struct_reorg::check_use (srdecl *decl, gimple *stmt, vec<srdecl*> &worklist)
       tree rhs1 = gimple_assign_rhs1 (stmt);
       tree rhs2 = gimple_assign_rhs2 (stmt);
       tree orhs = rhs1;
-      if (gimple_assign_rhs_code (stmt) != EQ_EXPR
-          && gimple_assign_rhs_code (stmt) != NE_EXPR)
+      enum tree_code code = gimple_assign_rhs_code (stmt);
+      if (code != EQ_EXPR && code != NE_EXPR
+	  && (current_mode != COMPLETE_STRUCT_RELAYOUT
+	      || (code != LT_EXPR && code != LE_EXPR
+		  && code != GT_EXPR && code != GE_EXPR)))
 	{
 	  mark_expr_escape (rhs1, escape_non_eq, stmt);
 	  mark_expr_escape (rhs2, escape_non_eq, stmt);
@@ -2618,6 +3514,12 @@ ipa_struct_reorg::record_accesses (void)
       /* Record accesses inside a function. */
       if(cnode->definition)
 	record_function (cnode);
+      else
+	{
+	  tree return_type = TREE_TYPE (TREE_TYPE (cnode->decl));
+	  mark_type_as_escape (return_type, escape_return, NULL);
+	}
+
     }
 
   if (dump_file && (dump_flags & TDF_DETAILS))
@@ -2734,8 +3636,11 @@ ipa_struct_reorg::propagate_escape (void)
 void
 ipa_struct_reorg::prune_escaped_types (void)
 {
-  detect_cycles ();
-  propagate_escape ();
+  if (current_mode != COMPLETE_STRUCT_RELAYOUT)
+    {
+      detect_cycles ();
+      propagate_escape ();
+    }
 
   if (dump_file && (dump_flags & TDF_DETAILS))
     {
@@ -3862,16 +4767,82 @@ ipa_struct_reorg::rewrite_functions (void)
 }
 
 unsigned int
-ipa_struct_reorg::execute (void)
+ipa_struct_reorg::execute_struct_relayout (void)
 {
-  /* FIXME: If there is a top-level inline-asm, the pass immediately returns. */
-  if (symtab->first_asm_symbol ())
-    return 0;
-  record_accesses ();
-  prune_escaped_types ();
-  analyze_types ();
+  unsigned retval = 0;
+  for (unsigned i = 0; i < types.length (); i++)
+    {
+      tree type = types[i]->type;
+      if (TYPE_FIELDS (type) == NULL)
+	{
+	  continue;
+	}
+      if (types[i]->has_alloc_array != 1)
+	{
+	  continue;
+	}
+      if (types[i]->chain_type)
+	{
+	  continue;
+	}
+      retval |= ipa_struct_relayout (type, this).execute ();
+    }
 
-  return rewrite_functions ();
+  if (dump_file)
+    {
+      if (transformed)
+	{
+	  fprintf (dump_file, "\nNumber of structures to transform in "
+		   "Complete Structure Relayout is %d\n", transformed);
+	}
+      else
+	{
+	  fprintf (dump_file, "\nNo structures to transform in "
+		   "Complete Structure Relayout.\n");
+	}
+    }
+
+  return retval;
+}
+
+unsigned int
+ipa_struct_reorg::execute (enum srmode mode)
+{
+  unsigned int ret = 0;
+
+  if (mode == NORMAL)
+    {
+      current_mode = NORMAL;
+      /* If there is a top-level inline-asm,
+	 the pass immediately returns.  */
+      if (symtab->first_asm_symbol ())
+	{
+	  return 0;
+	}
+      record_accesses ();
+      prune_escaped_types ();
+      analyze_types ();
+
+      ret = rewrite_functions ();
+    }
+  else if (mode == COMPLETE_STRUCT_RELAYOUT)
+    {
+      if (dump_file)
+	{
+	  fprintf (dump_file, "\n\nTry Complete Struct Relayout:\n");
+	}
+      current_mode = COMPLETE_STRUCT_RELAYOUT;
+      if (symtab->first_asm_symbol ())
+	{
+	  return 0;
+	}
+      record_accesses ();
+      prune_escaped_types ();
+
+      ret = execute_struct_relayout ();
+    }
+
+  return ret;
 }
 
 const pass_data pass_data_ipa_struct_reorg =
@@ -3896,17 +4867,27 @@ public:
 
   /* opt_pass methods: */
   virtual bool gate (function *);
-  virtual unsigned int execute (function *) { return ipa_struct_reorg ().execute(); }
+  virtual unsigned int execute (function *)
+  {
+    unsigned int ret = 0;
+    ret = ipa_struct_reorg ().execute (NORMAL);
+    if (!ret)
+      {
+	ret = ipa_struct_reorg ().execute (COMPLETE_STRUCT_RELAYOUT);
+      }
+    return ret;
+  }
 
 }; // class pass_ipa_struct_reorg
 
 bool
 pass_ipa_struct_reorg::gate (function *)
 {
-  return (optimize
+  return (optimize >= 3
 	  && flag_ipa_struct_reorg
 	  /* Don't bother doing anything if the program has errors.  */
-	  && !seen_error ());
+	  && !seen_error ()
+	  && flag_lto_partition == LTO_PARTITION_ONE);
 }
 
 } // anon namespace
diff --git a/gcc/ipa-struct-reorg/ipa-struct-reorg.h b/gcc/ipa-struct-reorg/ipa-struct-reorg.h
index f2163910896..d8fe399bdf8 100644
--- a/gcc/ipa-struct-reorg/ipa-struct-reorg.h
+++ b/gcc/ipa-struct-reorg/ipa-struct-reorg.h
@@ -121,6 +121,7 @@ public:
 
   tree newtype[max_split];
   bool visited;
+  int has_alloc_array;
 
   // Constructors
   srtype(tree type);
@@ -232,4 +233,34 @@ struct srdecl
 
 } // namespace struct_reorg
 
+
+namespace struct_relayout {
+
+const int min_relayout_split = 8;
+const int max_relayout_split = 16;
+
+struct csrtype
+{
+  tree type;
+  unsigned HOST_WIDE_INT old_size;
+  unsigned HOST_WIDE_INT new_size;
+  unsigned field_count;
+  tree struct_size;
+
+  // Constructors
+  csrtype ()
+    : type (NULL),
+      old_size (0),
+      new_size (0),
+      field_count (0),
+      struct_size (NULL)
+  {}
+
+  // Methods
+  unsigned calculate_field_num (tree field_offset);
+  void init_type_info (void);
+};
+
+} // namespace struct_relayout
+
 #endif
diff --git a/gcc/testsuite/g++.dg/struct/no-body-function.cpp b/gcc/testsuite/g++.dg/struct/no-body-function.cpp
new file mode 100644
index 00000000000..4e56e73fcae
--- /dev/null
+++ b/gcc/testsuite/g++.dg/struct/no-body-function.cpp
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-std=gnu++17 -Wno-builtin-declaration-mismatch -O3 -fwhole-program -flto-partition=one -fipa-struct-reorg -S" } */
+
+struct S {
+    int x;
+    double y;
+};
+S f();
+
+const auto [x0, y0] = f();
+const auto [x1, y1] = f();
+
+static union {
+int a;
+double b;
+};
+
+const auto [x2, y2] = f();
diff --git a/gcc/testsuite/g++.dg/struct/struct-reorg-1.cpp b/gcc/testsuite/g++.dg/struct/struct-reorg-1.cpp
new file mode 100644
index 00000000000..6ab71abe140
--- /dev/null
+++ b/gcc/testsuite/g++.dg/struct/struct-reorg-1.cpp
@@ -0,0 +1,13 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -fwhole-program -flto-partition=one -fipa-struct-reorg -fdump-ipa-struct_reorg-details -S" } */
+
+struct Foo { int foo; int a; };
+Foo& ignoreSetMutex = *(new Foo);
+
+struct Goo { int goo; int a; };
+
+int main ()
+{
+  Goo* a;
+  return a->goo = 90;
+}
diff --git a/gcc/testsuite/g++.dg/struct/struct-reorg-2.cpp b/gcc/testsuite/g++.dg/struct/struct-reorg-2.cpp
new file mode 100644
index 00000000000..72b7db8a9ce
--- /dev/null
+++ b/gcc/testsuite/g++.dg/struct/struct-reorg-2.cpp
@@ -0,0 +1,17 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -fwhole-program -flto-partition=one -fipa-struct-reorg -fdump-ipa-struct_reorg-details" } */
+
+#include <stdlib.h>
+
+struct testg {
+  int b;
+  float c;
+};
+
+testg *testgvar;
+int main ()
+{
+  testgvar = (testg*) calloc(10, sizeof(testg));
+  int b = testgvar->b;
+  return b;
+}
diff --git a/gcc/testsuite/g++.dg/struct/struct-reorg-3.cpp b/gcc/testsuite/g++.dg/struct/struct-reorg-3.cpp
new file mode 100644
index 00000000000..771164a96e7
--- /dev/null
+++ b/gcc/testsuite/g++.dg/struct/struct-reorg-3.cpp
@@ -0,0 +1,24 @@
+/* { dg-do run } */
+/* { dg-options "-O3 -fwhole-program -flto-partition=one -fipa-struct-reorg -fdump-ipa-struct_reorg-details" } */
+
+#include <stdlib.h>
+
+struct testg {
+  int b;
+  float c;
+  double d;
+  double e;
+  double f;
+  double h;
+  double i;
+  double j;
+  int k;
+};
+
+testg *testgvar;
+int main ()
+{
+  testgvar = (testg*) calloc(10, sizeof(testg));
+  int b = testgvar->b;
+  return b;
+}
diff --git a/gcc/testsuite/g++.dg/struct/struct-reorg.exp b/gcc/testsuite/g++.dg/struct/struct-reorg.exp
new file mode 100644
index 00000000000..4307f69e2cd
--- /dev/null
+++ b/gcc/testsuite/g++.dg/struct/struct-reorg.exp
@@ -0,0 +1,26 @@
+#   Copyright (C) 2021-2021 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GCC; see the file COPYING3.  If not see
+# <http://www.gnu.org/licenses/>.
+
+load_lib g++-dg.exp
+
+# Initialize `dg'.
+dg-init
+
+g++-dg-runtest [lsort [glob -nocomplain $srcdir/$subdir/*.cpp]] \
+	"" ""
+
+# All done.
+dg-finish
diff --git a/gcc/testsuite/gcc.dg/struct/complete_struct_relayout.c b/gcc/testsuite/gcc.dg/struct/complete_struct_relayout.c
new file mode 100644
index 00000000000..811030bf167
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/struct/complete_struct_relayout.c
@@ -0,0 +1,60 @@
+// { dg-do run }
+
+#include <stdlib.h>
+#include <stdio.h>
+
+typedef struct node node_t;
+typedef struct node* node_p;
+
+struct node {
+  unsigned long a;
+  unsigned long b;
+  node_p c;
+  node_p d;
+  long e;
+  long f;
+  long g;
+  long h;
+  long i;
+  long j;
+  long k;
+  long l;
+  int m;
+  int n;
+};
+
+const int MAX = 10000;
+node_p n;
+
+int
+main ()
+{
+  n = (node_p) calloc (MAX, sizeof (node_t));
+
+  for (int i = 0; i < MAX; i++)
+    {
+      n[i].a = 100;
+    }
+  for (int i = 0; i < MAX; i++)
+    {
+      if (n[i].a != 100)
+	{
+	  abort ();
+	}
+    }
+
+  for (int i = 0; i < MAX; i++)
+    {
+      n[i].l = n[i].a;
+    }
+  for (int i = 0; i < MAX; i++)
+    {
+      if (n[i].l != 100)
+	{
+	  abort ();
+	}
+    }
+  return 0;
+}
+
+/* { dg-final { scan-ipa-dump "Number of structures to transform in Complete Structure Relayout is 1" "struct_reorg" } } */
diff --git a/gcc/testsuite/gcc.dg/struct/csr_allocation-1.c b/gcc/testsuite/gcc.dg/struct/csr_allocation-1.c
new file mode 100644
index 00000000000..63bb695ae14
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/struct/csr_allocation-1.c
@@ -0,0 +1,46 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+typedef struct node node_t;
+typedef struct node* node_p;
+
+struct node {
+  unsigned long a;
+  unsigned long b;
+  node_p c;
+  node_p d;
+  long e;
+  long f;
+  long g;
+  long h;
+  long i;
+  long j;
+  long k;
+  long l;
+  int m;
+  int n;
+};
+
+const int MAX = 1;
+node_p n;
+
+int
+main ()
+{
+  n = (node_p) calloc (MAX, sizeof (node_t));
+
+  for (int i = 0; i < MAX; i++)
+    {
+      n[i].a = 100;
+    }
+  for (int i = 0; i < MAX; i++)
+    {
+      if (n[i].a != 100)
+	{
+	  abort ();
+	}
+    }
+  return 0;
+}
+
+/* { dg-final { scan-ipa-dump "No structures to transform in Complete Structure Relayout." "struct_reorg" } } */
diff --git a/gcc/testsuite/gcc.dg/struct/csr_allocation-2.c b/gcc/testsuite/gcc.dg/struct/csr_allocation-2.c
new file mode 100644
index 00000000000..0f75d5d121c
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/struct/csr_allocation-2.c
@@ -0,0 +1,59 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+typedef struct node node_t;
+typedef struct node* node_p;
+
+struct node {
+  unsigned long a;
+  unsigned long b;
+  node_p c;
+  node_p d;
+  long e;
+  long f;
+  long g;
+  long h;
+  long i;
+  long j;
+  long k;
+  long l;
+  int m;
+  int n;
+};
+
+const int MAX = 10;
+node_p n;
+node_p m;
+
+int main()
+{
+  int i;
+  for (i = 0; i < MAX / 5; i++)
+    {
+      n = (node_p) calloc(MAX, sizeof(node_t));
+      if (i == 0)
+	{
+	  m = n;
+	}
+    }
+
+  for (int i = 0; i < MAX; i++)
+    {
+      n[i].a = 100;
+    }
+  for (int i = 0; i < MAX; i++)
+    {
+      m[i].a = 50;
+    }
+
+  for (int i = 0; i < MAX; i++)
+    {
+      if (n[i].a != 100)
+	{
+	  abort ();
+	}
+    }
+  return 0;
+}
+
+/* { dg-final { scan-ipa-dump "No structures to transform in Complete Structure Relayout." "struct_reorg" } } */
diff --git a/gcc/testsuite/gcc.dg/struct/csr_allocation-3.c b/gcc/testsuite/gcc.dg/struct/csr_allocation-3.c
new file mode 100644
index 00000000000..3dcb674c6e9
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/struct/csr_allocation-3.c
@@ -0,0 +1,77 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+typedef struct node node_t;
+typedef struct node* node_p;
+
+struct node {
+  unsigned long a;
+  unsigned long b;
+  node_p c;
+  node_p d;
+  long e;
+  long f;
+  long g;
+  long h;
+  long i;
+  long j;
+  long k;
+  long l;
+  int m;
+  int n;
+};
+
+const int MAX = 10;
+node_p n;
+node_p m;
+
+void test (int, int) __attribute__((noinline));
+
+void
+test (int num, int flag)
+{
+  if (num <= 0)
+    {
+      return;
+    }
+  n = (node_p) calloc (num, sizeof (node_t));
+  if (flag)
+    {
+      m = n;
+    }
+  return;
+}
+
+int
+main ()
+{
+  test (MAX, 1);
+  test (MAX, 0);
+
+  for (int i = 0; i < MAX; i++)
+    {
+      n[i].a = 100;
+    }
+  for (int i = 0; i < MAX; i++)
+    {
+      m[i].a = 50;
+    }
+
+  for (int i = 0; i < MAX; i++)
+    {
+      if (n[i].a != 100)
+	{
+	  abort ();
+	}
+    }
+  for (int i = 0; i < MAX; i++)
+    {
+      if (m[i].a != 50)
+	{
+	  abort ();
+	}
+    }
+  return 0;
+}
+
+/* { dg-final { scan-ipa-dump "No structures to transform in Complete Structure Relayout." "struct_reorg" } } */
diff --git a/gcc/testsuite/gcc.dg/struct/csr_cast_int.c b/gcc/testsuite/gcc.dg/struct/csr_cast_int.c
new file mode 100644
index 00000000000..6907158c9b0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/struct/csr_cast_int.c
@@ -0,0 +1,52 @@
+// { dg-do run }
+
+#include <stdlib.h>
+#include <stdio.h>
+
+typedef struct node node_t;
+typedef struct node* node_p;
+
+struct node {
+  unsigned long a;
+  unsigned long b;
+  node_p c;
+  node_p d;
+  long e;
+  long f;
+  long g;
+  long h;
+  long i;
+  long j;
+  long k;
+  long l;
+  int m;
+  int n;
+};
+
+const int MAX = 100;
+node_p n;
+unsigned long y;
+
+int
+main ()
+{
+  n = (node_p) calloc (MAX, sizeof (node_t));
+
+  for (int i = 0; i < MAX; i++)
+    {
+      n[i].b = 50;
+    }
+
+  node_p x = &n[5];
+  y = (unsigned long) x;
+  y += 8;
+
+  if (*((unsigned long*) y) != 50)
+    {
+      abort ();
+    }
+
+  return 0;
+}
+
+/* { dg-final { scan-ipa-dump "struct node has escaped: \"Type escapes a cast from/to intergral type\"" "struct_reorg" } } */
diff --git a/gcc/testsuite/gcc.dg/struct/csr_separate_instance.c b/gcc/testsuite/gcc.dg/struct/csr_separate_instance.c
new file mode 100644
index 00000000000..9e5e05838e6
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/struct/csr_separate_instance.c
@@ -0,0 +1,48 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+typedef struct node node_t;
+typedef struct node* node_p;
+
+struct node {
+  unsigned long a;
+  unsigned long b;
+  node_p c;
+  node_p d;
+  long e;
+  long f;
+  long g;
+  long h;
+  long i;
+  long j;
+  long k;
+  long l;
+  int m;
+  int n;
+};
+
+const int MAX = 10000;
+node_p n;
+node_t t;
+
+int
+main ()
+{
+  n = (node_p) calloc (MAX, sizeof (node_t));
+  t.a = 100;
+
+  for (int i = 0; i < MAX; i++)
+    {
+      n[i].a = t.a;
+    }
+  for (int i = 0; i < MAX; i++)
+    {
+      if (n[i].a != 100)
+	{
+	  abort ();
+	}
+    }
+  return 0;
+}
+
+/* { dg-final { scan-ipa-dump "struct node has escaped: \"Type escapes via a separate instance\"" "struct_reorg" } } */
diff --git a/gcc/testsuite/gcc.dg/struct/sr_address_of_field.c b/gcc/testsuite/gcc.dg/struct/sr_address_of_field.c
new file mode 100644
index 00000000000..9d58edab80a
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/struct/sr_address_of_field.c
@@ -0,0 +1,37 @@
+/* { dg-do run } */
+
+static struct S {
+  int *p1;
+  int *p2;
+} s;
+
+typedef __UINTPTR_TYPE__ uintptr_t;
+
+int
+foo ()
+{
+  int i = 1;
+  int j = 2;
+  struct S s;
+  int **p;
+  s.p1 = &i;
+  s.p2 = &j;
+  p = &s.p1;
+  uintptr_t pi = (uintptr_t) p;
+  pi = pi + sizeof (int *);
+  p = (int **)pi;
+  **p = 3;
+  return j;
+}
+
+int
+main ()
+{
+  if (foo () != 3)
+    {
+      __builtin_abort ();
+    }
+  return 0;
+}
+
+/* { dg-final { scan-ipa-dump "struct S has escaped: \"Type escapes via taking the address of field\"" "struct_reorg" } } */
diff --git a/gcc/testsuite/gcc.dg/struct/sr_convert_mem.c b/gcc/testsuite/gcc.dg/struct/sr_convert_mem.c
new file mode 100644
index 00000000000..a99ee0de484
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/struct/sr_convert_mem.c
@@ -0,0 +1,23 @@
+/* { dg-do compile } */
+
+struct T1 {
+  long var1;
+  int  var2;
+};
+
+struct T2 {
+  long var1;
+  int  var2;
+};
+
+void test (void*);
+
+__attribute__((used)) void
+foo (struct T2 *t2)
+{
+  struct T1* t1 = (void *)(&t2[1]);
+  void*  data   = (void *)(&t1[1]);
+
+  test(data);
+  return;
+}
diff --git a/gcc/testsuite/gcc.dg/struct/sr_maxmin_expr.c b/gcc/testsuite/gcc.dg/struct/sr_maxmin_expr.c
new file mode 100644
index 00000000000..fb135ef0bb5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/struct/sr_maxmin_expr.c
@@ -0,0 +1,25 @@
+// { dg-do compile }
+
+#include <stdlib.h>
+
+struct S {
+  unsigned long a;
+  unsigned long b;
+};
+
+struct S* s;
+struct S* t = (struct S*) 1000;
+
+int
+main ()
+{
+  s = (struct S*) calloc (1000, sizeof (struct S));
+  s = s > t ? s : t;
+  if (s == 0)
+    {
+      abort ();
+    }
+  return 0;
+}
+
+/* { dg-final { scan-ipa-dump "No structures to transform." "struct_reorg" } } */
diff --git a/gcc/testsuite/gcc.dg/struct/sr_pointer_and.c b/gcc/testsuite/gcc.dg/struct/sr_pointer_and.c
new file mode 100644
index 00000000000..9a4b10d9aef
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/struct/sr_pointer_and.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+
+struct test {long val; struct test* next; };
+
+unsigned long P_DATA;
+
+void func (struct test*);
+
+__attribute__((used)) static void
+foo (struct test* pt)
+{
+  struct test t;
+
+  t.next = (void *)((unsigned long)pt->next & P_DATA);
+  func(&t);
+  return;
+}
diff --git a/gcc/testsuite/gcc.dg/struct/sr_pointer_minus.c b/gcc/testsuite/gcc.dg/struct/sr_pointer_minus.c
new file mode 100644
index 00000000000..9a82da0d6e1
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/struct/sr_pointer_minus.c
@@ -0,0 +1,33 @@
+// { dg-do compile }
+
+#include <stdlib.h>
+
+typedef struct node node_t;
+typedef struct node* node_p;
+
+struct node {
+  unsigned long a;
+  unsigned long b;
+};
+
+int max;
+int x;
+
+node_p n;
+node_p z;
+
+int
+main ()
+{
+  n = (node_p) calloc (max, sizeof (node_t));
+
+  node_p xp = &n[x];
+
+  if (xp - z == 10)
+    {
+      abort ();
+    }
+  return 0;
+}
+
+/* { dg-final { scan-ipa-dump "struct node has escaped: \"Type escapes via a unhandled rewrite stmt\"" "struct_reorg" } } */
-- 
2.21.0.windows.1