/* Resource constrained software pipeliner.  This file contains
   implementation of core algorithm of the resource constrained
   software pipeliner.  Copyright (C) 2003 Free Software Foundation,
   Inc.

   Written by Vladimir Makarov <vmakarov@redhat.com>
   
   This file is part of GCC.

   GCC is free software; you can redistribute it and/or modify it under
   the terms of the GNU General Public License as published by the Free
   Software Foundation; either version 2, or (at your option) any later
   version.

   GCC is distributed in the hope that it will be useful, but WITHOUT ANY
   WARRANTY; without even the implied warranty of MERCHANTABILITY or
   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   for more details.

   You should have received a copy of the GNU General Public License
   along with GCC; see the file COPYING.  If not, write to the Free
   Software Foundation, 59 Temple Place - Suite 330, Boston, MA
   02111-1307, USA.  */

/* Idea of this software pipeline algorithm is taken from RCSP
   (resource-constrained software pipelining).  You can find RCSP
   description e.g. in

      Resource-constrained Software Pipelining by A. Aiken,
      A. Nicolau, and S. Novack IEEE Transactions on Parallel and
      Distributed Systems, 6(12), pp. 1248--1270, December 1995.

   The algorithm is trying to schedule loop insns (operations)
   by searching repeating patterns of loop insns (operations).

   This implementation assumes that all current and future
   reservations of processor functional units are coded by data block
   of fixed length.  This is good assumption if we use pipeline
   hazards recognizer implemented as deterministic finite automaton.

   Actually, usage of backtracking could find optimally software
   pipelined loop.  The core of algorithm also could be used for
   inter-block scheduling with possibility of copying insns too.  So it
   is very powerful method for possible future improvements of the
   scheduler.

   What could we make the next for improving RCSP?

   1. Better aliasing memory locations.
   
   2. Register renaming for registers which are temporary on each
      iteration.

   3. Loop unrolling with epilogue loop for loops whose number of
      iterations is known before starting loop.  This could permit to
      move risky insns from one iteration to another.

   4. Better recognition of risky insns and/or transformation of them
      into corresponding non exceptional insns.

   5. Modification of algorithm for making more compact pipelined
      loops and /or more effective ones (see comments for merging
      frontier states).  Additional heuristics in choosing insn for
      scheduling (critical path length, register pressure, basic block
      movement, probability for speculative moving).  Implementation
      of more accurate merging cpu states.

   6. Adding jump insns during scheduling and conversion of conditional
      branches.  More one conditional branches in packet.

   7. Speedup the pipeliner.  We could add new iteration and update
      DDG only when it is necessary.  We could specially process
      simple cases for moving operation in basic block.

   1-4 are relative to the pipeliner interface.  5-7 are for the
   pipeliner itself.  */

#include "config.h"
#include "system.h"
#include "rcsp-int.h"
#include "hashtab.h"

/* The following macro value defines unrolling the loop.  There will
   be (RCSP_UNROLL_FACTOR * maximal iteration look ahead + 1) loop bodies.
   The value should be more 1.  */
#define RCSP_UNROLL_FACTOR 2

/* The following macro value defines maximal size of successfuly
   pipelined loop in original loop size.  The value should be more
   1.  */
#define RCSP_EXPANSION_LOOP_FACTOR 8

typedef struct locref * locref_ptr;
typedef htab_t          locrefset_t;
typedef htab_t          packet_set_t;

/* The following describes a location reference in interior packet.
   There may be several usages of one location (but they are deleted
   when the location definition is occurred).  */
struct locref
{
  /* Flag of that this is reference for the location usage.  */
  char usage_p;
  /* The following is defined only for location definition.  Value
     true means that only ops defining (but using) the loc will be
     depended on this op.  */
  char write_barrier_p;
  loc_ptr loc;
  /* Operation in which the loc is mentioned.  */
  op_ptr op;
  /* The next locref with the loc defined or used in OP->PACKET.  */
  locref_ptr next_locref;
};

/* The following variable value is used maximal look ahead (window) in
   iterations.  */
int iteration_look_ahead;

/* The following variable value is current time.  */
static int curr_time;

#ifdef RCSP_DEBUG
int rcsp_debug_level = 3;
#endif

/* Define this if you want fast evaluation of achievability of one
   packet from another one.  The cache size is (sizeof (int) + 1) * (2
   ** round_up (log2 (packets_number))) ** 2.  So the cache may be
   huge in many cases (e.g. 5MB for 1K packets).  */
#define USE_ACHIEVABLE_CACHE 1


#ifdef USE_ACHIEVABLE_CACHE

/* CFG_shift is more than the following value we don't use the cache.  */
#define MAX_CFG_SHIFT 10

/* The following variables are used to faster evaluation of packets
   achievability.  */
static int    CFG_tick;       /* Increased every time when CFG is changed.    */
static int *  CFG_validation; /* Used for validation of the next array value.  */
static char * CFG_achievable; /* Cache of achievable values.                  */
static int    CFG_shift;      /* Used for fast access to the previous arrays.  */

/* This macro is used to evaluate achievability of the packet TO from
   packet FROM when the cache may be used.  */
#define ACHIEVABLE(from, to)						      \
  (CFG_shift <= MAX_CFG_SHIFT						      \
   &&									      \
   CFG_validation [((from)->number << CFG_shift) + (to)->number] == CFG_tick  \
   ? (achievable_mode_t) CFG_achievable [((from)->number << CFG_shift)	      \
					+ (to)->number]			      \
   : achievable (from, to))

#else

#define ACHIEVABLE(from, to) achievable (from, to)

#endif /* #ifdef USE_ACHIEVABLE_CACHE */

/* This is dummy packet which has no predecessors and contains one
   dummy operation.  It is used to store dummy definitions of
   locations used somewhere in the CFG and not defined somewhere upper
   in the CFG.  */
static packet_ptr top_packet;

/* The following variable contains current number of CFG traversing
   for different goals:

   1. Original building DDG (including sorting input dependencies),
      evaluating achievability of packets in CFG and post-dominator
      relation in CFG, removing unachievable packets, and output of
      pipelined insns.

   2. Traversing graph for searching for set `covered'.

   3. Evaluating cost of making CFG move consistent.

   4. Adding elements to available list of new packets and outputting
      additional jumps on back edges.

   5. Checking CFG and DDG, printing packets, and outputting
      additional jumps.  */

int last_packet_visit_number1;
static int last_packet_visit_number2;
static int last_packet_visit_number3;
static int last_packet_visit_number4;
static int last_packet_visit_number5;

/* The following variable value is length (in bytes) of structure
   which represents cpu functional units state.  */
static int cpu_state_length;

/* The following variable value is a reference to temporary structure
   which represents cpu functional units state.  */
static cpu_state_ptr temp_cpu_state;



/* This page contains functions for work with operations (insns).  */

/* The following variable value is reference for array of ready ops.
   Each array element with index N starts list of ready ops with
   iteration number N.  It should only be read from outside the
   page.  */
static op_ptr *ready_ops;

/* The following variable value is size of array ready_ops (number of
   loop body iterations after unrolling).  It should only be read from
   outside the page.  */
static int ready_ops_size;

/* The following variavle value is number of ops which are already
   scheduled.  It should only be read from outside the page.  */
static int scheduled_ops_number;

/* The following variable refers for unused ops.  */
static op_ptr free_ops;

/* The following variable refers for all allocated ops.  */
static op_ptr all_ops;

/* The following variable value is maximal index of ready_ops for
   which ready_ops [i] == NULL for all i < MIN_READY_OP_IT_NUM.  */
static int min_ready_op_it_num;

/* The following function frees OP.  */

static void
free_op (op_ptr op)
{
  op->next_op = free_ops;
  free_ops = op;
}

/* The following function returns free op.  */

static op_ptr
get_free_op (void)
{
  op_ptr result;

  if (free_ops)
    {
      result = free_ops;
      free_ops = free_ops->next_op;
    }
  else
    {
      result = rcsp_malloc (sizeof (struct op));
      result->all_ops_chain = all_ops;
      all_ops = result;
    }
  return result;
}

/* The following function creates and returns new operation with given
   characteristics.  */

op_ptr
op_create (insn_ptr insn, int iter)
{
  op_ptr result;

  result = get_free_op ();
  result->insn = insn;
  result->it_num = iter;
  result->packet = NULL;
  result->true_target = result->false_target = NULL;
  result->in_deps = result->out_deps = NULL;
  result->next_op = NULL;
  result->next_added_in_deps_op = NULL;
  result->initiator_op = NULL;
  result->scheduled_p = FALSE;
  result->ready_p = FALSE;
  result->next_ready_op = result->prev_ready_op = NULL;
  result->can_be_converted_p = result->processed_4_consistent_jumps_p
    = result->use_def_p = FALSE;
  result->true_jump_p = result->target_fixed_p = TRUE;
  return result;
}

/* The following function deletes operation.  If OP is ready then it
   is removed from ready list.  */

static void
op_delete (op_ptr op)
{
  if (op->ready_p)
    {
      /* Remove from ready list.  */
      if (op->prev_ready_op == NULL)
	ready_ops [op->it_num] = op->next_ready_op;
      else
	op->prev_ready_op->next_ready_op = op->next_ready_op;
      if (op->next_ready_op != NULL)
	op->next_ready_op->prev_ready_op = op->prev_ready_op;
    }
  if (op->scheduled_p)
    scheduled_ops_number--;
  free_op (op);
}

/* The following function makes copy of OP (op should be not
   scheduled).  If OP is ready, then the copy will be also ready.  The
   copy does not belong to a packet.  */

static op_ptr
op_copy (op_ptr op)
{
  op_ptr result;

  if (op->scheduled_p)
    abort ();
  result = get_free_op ();
  result->insn = insn_copy (op->insn);
  result->it_num = op->it_num;
  result->true_target = op->true_target;
  result->false_target = op->false_target;
  result->packet = NULL;
  result->in_deps = result->out_deps = NULL;
  result->next_op = NULL;
  result->next_added_in_deps_op = NULL;
  result->initiator_op = op->initiator_op;
  result->scheduled_p = FALSE;
  result->ready_p = op->ready_p;
  result->prev_ready_op = NULL;
  if (!op->ready_p)
    result->next_ready_op = NULL;
  else
    {
      result->next_ready_op = ready_ops [op->it_num];
      if (result->next_ready_op != NULL)
	result->next_ready_op->prev_ready_op = result;
      ready_ops [op->it_num] = result;
    }
  result->can_be_converted_p = op->can_be_converted_p;
  result->target_fixed_p = op->target_fixed_p;
  result->use_def_p = op->use_def_p;
  result->true_jump_p = op->true_jump_p;
  result->processed_4_consistent_jumps_p = op->processed_4_consistent_jumps_p;
  return result;
}

/* The following function fixes that OP is ready.  */

static void
op_become_ready (op_ptr op)
{
  if (op->scheduled_p || op->ready_p)
    abort ();
  op->ready_p = TRUE;
  /* Add OP to ready list.  */
  op->prev_ready_op = NULL;
  op->next_ready_op = ready_ops [op->it_num];
  if (op->next_ready_op != NULL)
    op->next_ready_op->prev_ready_op = op;
  ready_ops [op->it_num] = op;
  if (min_ready_op_it_num > op->it_num)
    min_ready_op_it_num = op->it_num;
}

/* The following function fixes that OP is not ready (it should be
   ready before the function call).  */

static void
op_become_not_ready (op_ptr op)
{
  if (op->scheduled_p || !op->ready_p)
    abort ();
  op->ready_p = FALSE;
  /* Remove from ready list.  */
  if (op->prev_ready_op == NULL)
    ready_ops [op->it_num] = op->next_ready_op;
  else
    op->prev_ready_op->next_ready_op = op->next_ready_op;
  if (op->next_ready_op != NULL)
    op->next_ready_op->prev_ready_op = op->prev_ready_op;
  if (min_ready_op_it_num == op->it_num)
    for (min_ready_op_it_num = op->it_num;
	 min_ready_op_it_num < ready_ops_size - 1
	   && ready_ops [min_ready_op_it_num] == NULL;
	 min_ready_op_it_num++)
      ;
}

/* The following function fixes that ready OP has been scheduled.  */

static void
op_become_scheduled (op_ptr op)
{
  op_become_not_ready (op);
  op->scheduled_p = TRUE;
  scheduled_ops_number++;
}

/* The function initiates work with ops.  */

static void
start_ops (void)
{
  scheduled_ops_number = 0;
  min_ready_op_it_num = 0;
  ready_ops_size = iteration_look_ahead * RCSP_UNROLL_FACTOR + 1;
  ready_ops = rcsp_malloc (ready_ops_size * sizeof (op_ptr));
  memset ((char *) ready_ops, 0, ready_ops_size * sizeof (op_ptr));
  all_ops = free_ops = NULL;
}

/* The function finishes all work with ops.  */

static void
finish_ops (void)
{
  op_ptr curr_op, next_op;

  for (curr_op = all_ops; curr_op != NULL; curr_op = next_op)
    {
      next_op = curr_op->all_ops_chain;
      rcsp_free (curr_op);
    }
  if (ready_ops != NULL)
    rcsp_free (ready_ops);
}



/* The page contains functions for work with avail lists.  Frontier
   and formed packets have such lists.  The list refers for operations
   whose all op sources of input dependencies have started (but may be
   input data are not ready).  */

/* The following variable refers for unused elements of avail
   lists.  */
static avail_list_ptr free_avail_list_els;

/* The following variable refers for all allocated elements of avail
   lists.  */
static avail_list_ptr all_avail_list_els;

/* The following variables refer for array of pointers to available
   list elements and its length in the pointers.  This array is used to
   sort available lists.  */
static avail_list_ptr *avail_list_array;
static int avail_list_array_length;

/* The function frees avail_list element EL.  */

static void
avail_list_el_free (avail_list_ptr el)
{
  el->next_avail = free_avail_list_els;
  free_avail_list_els = el;
}

/* The following function returns free avail_list element with member
   values PACKET, OP, and AVAIL_P.  */

static avail_list_ptr
avail_list_el_create (packet_ptr packet, op_ptr op, int avail_p)
{
  avail_list_ptr result;

  if (free_avail_list_els)
    {
      result = free_avail_list_els;
      free_avail_list_els = free_avail_list_els->next_avail;
    }
  else
    {
      result = rcsp_malloc (sizeof (struct avail_list));
      result->all_avail_list_els_chain = all_avail_list_els;
      all_avail_list_els = result;
    }
  result->packet = packet;
  result->op = op;
  result->avail_p = avail_p;
  result->prev_avail = result->next_avail = NULL;
  return result;
}

/* The following function frees all elements of AVAIL_LIST.  */

static void
avail_list_free (avail_list_ptr avail_list)
{
  avail_list_ptr curr_avail_list_el, next_avail_list_el;

  for (curr_avail_list_el = avail_list;
       curr_avail_list_el != NULL;
       curr_avail_list_el = next_avail_list_el)
    {
      next_avail_list_el = curr_avail_list_el->next_avail;
      avail_list_el_free (curr_avail_list_el);
    }
}

/* The function returns length of AVAIL_LIST.  */

static int
avail_list_length (avail_list_ptr avail_list)
{
  avail_list_ptr curr;
  int length = 0;

  for (curr = avail_list; curr != NULL; curr = curr->next_avail)
    length++;
  return length;
}

/* The following function sorts AVAIL_LIST and returns the result.
   Comparison function (-1, 0, 1) is CMP_FUNC.  */

static avail_list_ptr
avail_list_sort (avail_list_ptr avail_list, int (* cmp_func) (avail_list_ptr *, avail_list_ptr *))
{
  int length = avail_list_length (avail_list);
  int i;
  avail_list_ptr curr;
  int (*compar)(const void *, const void *)
    = (int (*)(const void *, const void *)) cmp_func;

  if (length <= 1)
    return avail_list;
  if (avail_list_array == NULL)
    {
      avail_list_array = rcsp_malloc (sizeof (avail_list_ptr) * length);
      avail_list_array_length = length;
    }
  else if (avail_list_array_length < length)
    {
      avail_list_array
	= rcsp_realloc (avail_list_array,
			sizeof (avail_list_ptr) * avail_list_array_length,
			sizeof (avail_list_ptr) * length);
      avail_list_array_length = length;
    }
  for (i = 0, curr = avail_list; curr != NULL; i++, curr = curr->next_avail)
    avail_list_array [i] = curr;
  if (i != length)
    abort ();
  qsort (avail_list_array, length, sizeof (avail_list_ptr), compar);
  for (i = 0; i < length - 1; i++)
    {
      avail_list_array [i]->next_avail = avail_list_array [i + 1];
      avail_list_array [i + 1]->prev_avail = avail_list_array [i];
    }
  avail_list_array [0]->prev_avail = NULL;
  avail_list_array [length - 1]->next_avail = NULL;
  return avail_list_array [0];
}

/* The function initiates work with avail lists.  */

static void
start_avail_lists (void)
{
  all_avail_list_els = free_avail_list_els = NULL;
  avail_list_array = NULL;
}

/* The function finishes all work with avail lists.  */

static void
finish_avail_lists (void)
{
  avail_list_ptr curr_el, next_el;

  for (curr_el = all_avail_list_els; curr_el != NULL; curr_el = next_el)
    {
      next_el = curr_el->all_avail_list_els_chain;
      rcsp_free (curr_el);
    }
  if (avail_list_array != NULL)
    rcsp_free (avail_list_array);
}



/* The page contains functions for work with packets.  */

/* The following variable refers for unused packets.  */
static packet_ptr free_packets;

/* The following variable refers for all allocated packets.  */
static packet_ptr all_packets;

/* The following variable is number of packets created and copied.  */
static int packets_number;

/* The following function frees PACKET.  */
static void
free_packet (packet_ptr packet)
{
  packet->next_packet = free_packets;
  free_packets = packet;
}

/* The macro value is initial length of arrays `succ' and `pred' of
   packet.  */
#define INIT_SUCC_PRED_LENGTH 5

/* The following function returns free packet.  */
static packet_ptr
get_free_packet (void)
{
  packet_ptr result;

  if (free_packets != NULL)
    {
      result = free_packets;
      free_packets = free_packets->next_packet;
    }
  else
    {
      result = rcsp_malloc (sizeof (struct packet));
      result->all_packets_chain = all_packets;
      all_packets = result;
      result->succ_length = INIT_SUCC_PRED_LENGTH;
      result->succ = rcsp_malloc (sizeof (packet_ptr) * INIT_SUCC_PRED_LENGTH);
      result->pred_length = INIT_SUCC_PRED_LENGTH;
      result->pred = rcsp_malloc (sizeof (packet_ptr) * INIT_SUCC_PRED_LENGTH);
      result->cpu_state = rcsp_malloc (cpu_state_length);
    }
  return result;
}

/* The following function checks that length of array `succ' (if
   SUCC_P) or `pred' of PACKET not less than LENGTH.  If it is not
   true, the array is expanded.  */

static void
expand_packet_succ_pred (packet_ptr packet, int succ_p, int length)
{
  int old_length;

  if (succ_p)
    {
      if (length > packet->succ_length)
	{
	  old_length = packet->succ_length;
	  packet->succ_length += packet->succ_length / 2 + 1;
          if (packet->succ_length < length)
            packet->succ_length = length;
          packet->succ
	    = rcsp_realloc (packet->succ, sizeof (packet_ptr) * old_length,
			    sizeof (packet_ptr) * packet->succ_length);
	}
    }
  else
    {
      if (length > packet->pred_length)
	{
	  old_length = packet->pred_length;
	  packet->pred_length += packet->pred_length / 2 + 1;
          if (packet->pred_length < length)
            packet->pred_length = length;
	  packet->pred
	    = rcsp_realloc (packet->pred, sizeof (packet_ptr) * old_length,
			    sizeof (packet_ptr) * packet->pred_length);
	}
    }
}

#ifdef USE_ACHIEVABLE_CACHE

/* The following function checks ACHIEVABLE_CACHE size and expands it
   if it is necessary.  */

static void
expand_achievable_cache (void)
{
  int i, old_cache_size, cache_size;

  if (packets_number >= (1 << CFG_shift))
    {
      old_cache_size = (1 << CFG_shift) * (1 << CFG_shift);
      while (packets_number >= (1 << CFG_shift))
	CFG_shift++;
      if (CFG_shift <= MAX_CFG_SHIFT)
	{
	  cache_size = (1 << CFG_shift) * (1 << CFG_shift);
	  CFG_validation
	    = rcsp_realloc (CFG_validation, old_cache_size * sizeof (int),
			    cache_size * sizeof (int));
	  for (i = 0; i < cache_size; i++)
	    CFG_validation [i] = 0;
	  CFG_achievable
	    = rcsp_realloc (CFG_achievable, old_cache_size * sizeof (char),
			    cache_size * sizeof (char));
	}
    }
}

#endif

/* The following function creates packet with ISSUE_TIME and default
   member values.  */
packet_ptr
packet_create (int issue_time)
{
  packet_ptr p = get_free_packet ();
  
  p->number = ++packets_number;
  p->stop_packet_p = FALSE;
  p->exit_packet_p = FALSE;
  p->frontier_packet_p = FALSE;
  p->formed_packet_p = FALSE;
  p->main_exit_p = FALSE;
  *p->succ = NULL;
  *p->pred = NULL;
  p->visit_number1 = p->visit_number2
    = p->visit_number3 = p->visit_number4 = p->visit_number5 = -1;
  p->first_op = p->last_op = NULL;
  p->min_it_n = -1;
  p->copy = NULL;
  p->next_frontier_packet = NULL;
  p->avail_list = NULL;
  p->next_packet = NULL;
  p->issue_time = issue_time;
#ifdef USE_ACHIEVABLE_CACHE
  expand_achievable_cache ();
#endif
  return p;
}

/* The following function deletes packet.  The packet should not refer
   for a CFG packet or should be not referred from a CFG packet.  */
static void
packet_delete (packet_ptr packet)
{
  op_ptr curr_op, next_op;

  if (*packet->succ != NULL || *packet->pred != NULL)
    abort ();
  avail_list_free (packet->avail_list);
  for (curr_op = packet->first_op; curr_op != NULL; curr_op = next_op)
    {
      next_op = curr_op->next_op;
      op_delete (curr_op);
    }
  free_packet (packet);
}

/* The following function appends OP to PACKET at the beginning of the
   operation list (if HEAD_P) or at the end of it.  */

void
packet_append_op (packet_ptr packet, op_ptr op, int head_p)
{
  op->packet = packet;
  if (packet->first_op == NULL)
    {
      packet->first_op = op;
      packet->last_op = op;
      op->next_op = NULL;
    }
  else if (head_p)
    {
      op->next_op = packet->first_op;
      packet->first_op = op;
    }
  else
    {
      op->next_op = NULL;
      packet->last_op->next_op = op;
      packet->last_op = op;
    }
}

/* The following function makes copy of interior packet and all its
   operations and clears all in/out dependencies.  */

packet_ptr
packet_copy (packet_ptr packet)
{
  packet_ptr p = get_free_packet ();
  op_ptr curr_op, copy;
  int min_it_n;

  if (packet->avail_list != NULL
      || packet->frontier_packet_p || packet->formed_packet_p
      || packet->stop_packet_p)
    /* This is not interior packet.  */
    abort ();
  p->number = ++packets_number;
  expand_packet_succ_pred (p, TRUE, packet->succ_length);
  expand_packet_succ_pred (p, FALSE, packet->pred_length);
  memcpy (p->succ, packet->succ, packet->succ_length * sizeof (packet_ptr));
  memcpy (p->pred, packet->pred, packet->pred_length * sizeof (packet_ptr));
#ifdef USE_ACHIEVABLE_CACHE
  CFG_tick++;
#endif
  p->visit_number1 = p->visit_number2 = p->visit_number3 = -1;
  p->first_op = p->last_op = NULL;
  min_it_n = -1;
  for (curr_op = packet->first_op;
       curr_op != NULL;
       curr_op = curr_op->next_op)
    {
      if (min_it_n < 0 || min_it_n > curr_op->it_num)
	min_it_n = curr_op->it_num;
      copy = op_copy (curr_op);
      packet_append_op (p, copy, FALSE);
    }
  p->stop_packet_p = FALSE;
  p->exit_packet_p = packet->exit_packet_p;
  p->main_exit_p = FALSE;
  p->min_it_n = min_it_n;
  p->copy = NULL;
  p->next_frontier_packet = NULL;
  p->frontier_packet_p = FALSE;
  p->formed_packet_p = FALSE;
  p->avail_list = NULL;
  p->next_packet = NULL;
  p->issue_time = packet->issue_time;
#ifdef USE_ACHIEVABLE_CACHE
  expand_achievable_cache ();
#endif
  return p;
}

/* The following function returns place of WHAT in array ARR.  If
   there is no such packet, the function return place of array end
   marker (containing NULL).  */

static packet_ptr *
packet_find_in_arr (packet_ptr * arr, packet_ptr what)
{
  if (arr == NULL)
    abort ();
  for (; *arr != NULL; arr++)
    if (*arr == what)
      break;
  return arr;
}

/* The following function removes WHAT from array ARR (by shifting all
   following array elements).  If there is no such packet, the
   function does nothing.  */

static void
packet_del_from_arr (packet_ptr * arr, packet_ptr what)
{
  packet_ptr *place;

  if (arr == NULL)
    abort ();
  if (what == NULL)
    return;
  place = packet_find_in_arr (arr, what);
  if (*place != NULL)
    {
#ifdef USE_ACHIEVABLE_CACHE
      CFG_tick++;
#endif
      for (; *place != NULL; place++)
	*place = place [1];
    }
}

/* The function adds SUCC_PRED at end of array `succ' (if SUCC_P) or
   `pred' of PACKET.  If there is already such packet in the array,
   the function does nothing.  */

static void
packet_add_to_arr (packet_ptr packet, packet_ptr succ_pred, int succ_p)
{
  packet_ptr *arr, *place;

  if (succ_pred == NULL)
    return;
  arr = succ_p ? packet->succ : packet->pred;
  place = packet_find_in_arr (arr, succ_pred);
  if (*place != NULL)
    return;
#ifdef USE_ACHIEVABLE_CACHE
  CFG_tick++;
#endif
  expand_packet_succ_pred (packet, succ_p, place - arr + 2);
  if (succ_p)
    {
      packet->succ [place - arr] = succ_pred;
      packet->succ [place - arr + 1] = NULL;
    }
  else
    {
      packet->pred [place - arr] = succ_pred;
      packet->pred [place - arr + 1] = NULL;
    }
}

/* The function connects TO and FROM modifying correspondingly their
   array `succ' and `pred'.  When POSSIBLE_BRANCH_P the function tries
   to set up {true,false}_target too.  In this case JUMP_P means that
   we should set up target corresponding to jump and corresponding to
   fall through otherwise.  The function is safe even if there is
   already such link between the packets.  */

void
packet_connect (packet_ptr from, packet_ptr to, int possible_branch_p, int jump_p)
{
  op_ptr curr_op;
  packet_ptr *place;
  int new_p;

  if (from == NULL || to == NULL)
    abort ();
  place = packet_find_in_arr (from->succ, to);
  new_p = FALSE;
  if (*place == NULL)
    {
      packet_add_to_arr (from, to, TRUE);
      new_p = TRUE;
    }
  packet_add_to_arr (to, from, FALSE);
  if (new_p && possible_branch_p)
    {
      /* Set up targets for ops.  */
      for (curr_op = from->first_op;
	   curr_op != NULL;
	   curr_op = curr_op = curr_op->next_op)
	if (uncond_branch_p (curr_op->insn))
	  {
	    /* There is only one unconditional jump in a packet. ???
               What should be here if we permit several branches in
               packet.  */
	    curr_op->true_target = to;
	    break;
	  }
	else if (cond_branch_p (curr_op->insn))
	  {
	    if (curr_op->true_target == NULL
		&& ((curr_op->true_jump_p && jump_p)
		    || (!curr_op->true_jump_p && !jump_p)))
	      {
		curr_op->true_target = to;
		break;
	      }
	    else if (curr_op->false_target == NULL
		     && ((!curr_op->true_jump_p && jump_p)
			 || (curr_op->true_jump_p && !jump_p)))
	      {
		curr_op->false_target = to;
		break;
	      }
	  }
    }
}

/* The following function returns operation in PACKET which is branch
   (with jump if JUMP_P) onto TARGET, otherwise NULL.  */

op_ptr
packet_find_branch (packet_ptr packet, packet_ptr target, int jump_p)
{
  op_ptr curr_op;

  if (target == NULL)
    abort ();
  for (curr_op = packet->first_op; curr_op != NULL; curr_op = curr_op->next_op)
    if ((curr_op->false_target == target && (!jump_p || !curr_op->true_jump_p))
	|| (curr_op->true_target == target
	    && (!jump_p || curr_op->true_jump_p)))
      return curr_op;
  return NULL;
}

/* The function changes possible link FROM->OLD_TO onto FROM->NEW_TO.
   The function also can be used for repair inconsistent values `succ'
   and `pred' for copied packets.  If NEW_TO is NULL link (FROM,
   OLD_TO) is simply removed.  */

void
packet_change_dest (packet_ptr from, packet_ptr old_to, packet_ptr new_to)
{
  packet_ptr *place;
  op_ptr branch;

  if (old_to == new_to || old_to == NULL)
    abort ();
  branch = packet_find_branch (from, old_to, FALSE);
  if (branch != NULL)
    {
      if (branch->true_target == old_to)
	branch->true_target = new_to;
      else if (branch->false_target == old_to)
	branch->false_target = new_to;
      else
	abort ();
    }
  if (new_to == NULL || *packet_find_in_arr (from->succ, new_to) != NULL)
    packet_del_from_arr (from->succ, old_to);
  else
    {
      place = packet_find_in_arr (from->succ, old_to);
      if (*place == NULL)
	packet_add_to_arr (from, new_to, TRUE);
      else
	{
	  *place = new_to;
#ifdef USE_ACHIEVABLE_CACHE
	  CFG_tick++;
#endif
	}
    }
  packet_del_from_arr (old_to->pred, from);
  if (new_to != NULL)
    packet_add_to_arr (new_to, from, FALSE);
}

/* The function changes possible link OLD_FROM->TO onto NEW_FROM->TO.
   The function also can be used for repair inconsistent values `succ'
   and `pred' for copied packets.  If NEW_ROM is NULL link (TO,
   OLD_FROM) is simply removed.  */

void
packet_change_src (packet_ptr to, packet_ptr old_from, packet_ptr new_from)
{
  packet_ptr *place;
  op_ptr branch;

  if (old_from == new_from || old_from == NULL)
    abort ();
  branch = packet_find_branch (old_from, to, FALSE);
  if (branch != NULL)
    {
      if (branch->true_target == to)
	branch->true_target = NULL;
      else if (branch->false_target == to)
	branch->false_target = NULL;
      else
	abort ();
    }
  if (new_from == NULL || *packet_find_in_arr (to->pred, new_from) != NULL)
    packet_del_from_arr (to->pred, old_from);
  else
    {
      place = packet_find_in_arr (to->pred, old_from);
      if (*place == NULL)
	packet_add_to_arr (to, new_from, FALSE);
      else
	{
	  *place = new_from;
#ifdef USE_ACHIEVABLE_CACHE
	  CFG_tick++;
#endif
	}
    }
  packet_del_from_arr (old_from->succ, to);
  if (new_from != NULL)
    packet_add_to_arr (new_from, to, TRUE);
}

/* The function initiates work with packets.  */

static void
start_packets (void)
{
#ifdef USE_ACHIEVABLE_CACHE
  int i, cache_size;
#endif

  all_packets = free_packets = NULL;
  packets_number = 0;
#ifdef USE_ACHIEVABLE_CACHE
  CFG_tick = 0;
  CFG_shift = 8;
  cache_size = (1 << CFG_shift) * (1 << CFG_shift);
  CFG_validation = rcsp_malloc (cache_size * sizeof (int));
  for (i = 0; i < cache_size; i++)
    CFG_validation [i] = 0;
  CFG_achievable = rcsp_malloc (cache_size * sizeof (char));
#endif
}

/* The function finishes all work with packets.  */

static void
finish_packets (void)
{
  packet_ptr curr_packet, next_packet;

  for (curr_packet = all_packets;
       curr_packet != NULL;
       curr_packet = next_packet)
    {
      next_packet = curr_packet->all_packets_chain;
      rcsp_free (curr_packet->succ);
      rcsp_free (curr_packet->pred);
      rcsp_free (curr_packet);
    }
#ifdef USE_ACHIEVABLE_CACHE
  rcsp_free (CFG_achievable);
  rcsp_free (CFG_validation);
#endif
}



/* The page contains functions for work with dependencies.  */

/* The following variable refers for unused dependencies.  */
static dep_ptr unused_deps;

/* The following variable refers for all allocated dependencies.  */
static dep_ptr all_deps;

/* The following table is used to find dependency by pair (from, loc)
   quickly.  */
static htab_t deps_table;

/* The following variable values are dep array and its length.  The
   array is used to sort list of input deps of operation.  */
static dep_ptr *in_deps_array;
static int in_deps_array_length;

/* The following variables forms list of ops to which new dependencies
   have been added.  */
static op_ptr first_added_in_deps_op, last_added_in_deps_op;

/* The following function frees dependency DEP.  */
static void
free_dep (dep_ptr dep)
{
  dep->next_out = unused_deps;
  unused_deps = dep;
}

/* The following function returns free dep.  */

static dep_ptr
get_free_dep (void)
{
  dep_ptr result;

  if (unused_deps != NULL)
    {
      result = unused_deps;
      unused_deps = unused_deps->next_out;
    }
  else
    {
      result = rcsp_malloc (sizeof (struct dep));
      result->all_deps_chain = all_deps;
      all_deps = result;
    }
  return result;
}

/* The following function deletes dependency DEP.  */

static void
dep_delete (dep_ptr dep)
{
  dep_ptr prev = dep->prev_in;
  dep_ptr next = dep->next_in;
  void **slot;

  /* Remove dep from double link chain of input dependencies for
     dep->from.  */
  if (prev == NULL)
    dep->to->in_deps = next;
  else
    prev->next_in = next;
  if (next != NULL)
    next->prev_in = prev;
  /* The same for output dependencies of dep->to.  */
  prev = dep->prev_out;
  next = dep->next_out;
  if (prev == NULL)
    dep->from->out_deps = next;
  else
    prev->next_out = next;
  if (next != NULL)
    next->prev_out = prev;
  /* Remove dep from loc dependencies.  */
  prev = dep->prev_out_loc_dep;
  next = dep->next_out_loc_dep;
  if (prev != NULL)
    prev->next_out_loc_dep = next;
  if (next != NULL)
    next->prev_out_loc_dep = prev;
  slot = htab_find_slot (deps_table, dep, FALSE);
  if (slot == NULL)
    abort ();
  else if (*(dep_ptr *) slot == dep)
    {
      if (prev != NULL)
	abort ();
      if (next == NULL)
	htab_clear_slot (deps_table, slot);
      else
	*(dep_ptr *) slot = next;
    }
  free_dep (dep);
}

/* The following function returns the first output dependency (marked
   with LOC) from OP.  */

static dep_ptr
dep_out_find (op_ptr op, loc_ptr loc)
{
#if 0
  dep_ptr curr_dep;

  for (curr_dep = op->out_deps;
       curr_dep != NULL;
       curr_dep = curr_dep->next_out)
    if (loc_eq_p (curr_dep->to_loc, loc))
      return curr_dep;
  return NULL;
#else
  struct dep dep;

  dep.to_loc = loc;
  dep.from = op;
  return (dep_ptr) htab_find (deps_table, &dep);
#endif
}

/* The following function creates dependency with given
   characteristics.  */

static void
dep_create (op_ptr from, op_ptr to, loc_ptr loc, dep_mode_t dep_mode, int latency)
{
  dep_ptr first_loc_dep;
  void **slot;
  dep_ptr curr_loc_dep;
  dep_ptr dep = get_free_dep ();
  
  dep->dep_mode = dep_mode;
  dep->latency = latency;
  dep->from = from;
  dep->to_loc = loc;
  dep->to = to;
  /* Insert dep in loc_dep list.  */
  slot = htab_find_slot (deps_table, dep, TRUE);
  first_loc_dep = *(dep_ptr *) slot;
  if (first_loc_dep == NULL)
    {
      dep->prev_out_loc_dep = dep->next_out_loc_dep = NULL;
      *(dep_ptr *) slot = dep;
    }
  else
    {
      for (curr_loc_dep = first_loc_dep;
	   curr_loc_dep != NULL;
	   curr_loc_dep = curr_loc_dep->next_out_loc_dep)
	if (curr_loc_dep->to == to && curr_loc_dep->dep_mode == dep_mode
	    && curr_loc_dep->latency == latency)
	  break;
      if (curr_loc_dep != NULL)
	{
	  /* There is already such dep.  */
	  if (!loc_eq_p (curr_loc_dep->to_loc, loc)
	      || curr_loc_dep->from != from)
	    abort ();
	  free_dep (dep);
	  return;
	}
      dep->prev_out_loc_dep = first_loc_dep;
      dep->next_out_loc_dep = first_loc_dep->next_out_loc_dep;
      if (first_loc_dep->next_out_loc_dep != NULL)
        first_loc_dep->next_out_loc_dep->prev_out_loc_dep = dep;
      first_loc_dep->next_out_loc_dep = dep;
    }
  if (to->next_added_in_deps_op == NULL && to != last_added_in_deps_op)
    {
      /* TO is not in the list.  Insert TO into the list of operations
	 to which input dependencies have been added.  */
      if (first_added_in_deps_op == NULL)
	first_added_in_deps_op = to;
      else
	last_added_in_deps_op->next_added_in_deps_op = to;
      last_added_in_deps_op = to;
    }
  /* Insert dep in output dep list.  */
  dep->prev_out = NULL;
  if (from->out_deps == NULL)
    dep->next_out = NULL;
  else
    {
      dep->next_out = from->out_deps;
      from->out_deps->prev_out = dep;
    }
  from->out_deps = dep;
  /* Insert dep in input dep list.  */
  dep->prev_in = NULL;
  if (to->in_deps == NULL)
    dep->next_in = NULL;
  else
    {
      dep->next_in = to->in_deps;
      to->in_deps->prev_in = dep;
    }
  to->in_deps = dep;
}

/* The following function is used as comparison function to sort input
   dependencies list of operation.  */

static int
in_deps_cmp (const void * dep1, const void * dep2)
{
  dep_ptr d1 = *(dep_ptr *) dep1;
  dep_ptr d2 = *(dep_ptr *) dep2;
  
  if (d1->to_loc < d2->to_loc)
    return -1;
  else if (d1->to_loc > d2->to_loc)
    return 1;
  else
    return d1->from->packet->number - d2->from->packet->number;
}

/* The following function sorts input dependencies list of OP.  The
   goal of sorting is to place input dependencies of OP with the same
   to_loc nearby.  */

static void
sort_op_in_deps (op_ptr op)
{
  int length = 0;
  int i;
  dep_ptr curr_dep;

  for (curr_dep = op->in_deps;
       curr_dep != NULL;
       curr_dep = curr_dep->next_in)
    length++;
  if (length <= 1)
    return;
  if (in_deps_array == NULL)
    {
      in_deps_array = rcsp_malloc (sizeof (dep_ptr) * length);
      in_deps_array_length = length;
    }
  else if (length > in_deps_array_length)
    {
      in_deps_array = rcsp_realloc (in_deps_array,
				    sizeof (dep_ptr) * in_deps_array_length,
				    sizeof (dep_ptr) * length);
      in_deps_array_length = length;
    }
  for (i = 0, curr_dep = op->in_deps;
       curr_dep != NULL;
       curr_dep = curr_dep->next_in, i++)
    in_deps_array [i] = curr_dep;
  qsort (in_deps_array, length, sizeof (dep_ptr), in_deps_cmp);
  for (i = 1; i < length; i++)
    {
      in_deps_array [i - 1]->next_in = in_deps_array [i];
      in_deps_array [i]->prev_in = in_deps_array [i - 1];
    }
  in_deps_array [0]->prev_in = in_deps_array [length - 1]->next_in = NULL;
  op->in_deps = in_deps_array [0];
}

/* The following function sorts input dependencies for all OP to which
   new input dependencies added.  After this such ops are forgot.  */

static void
dep_sort_added_in_deps_op (void)
{
  op_ptr curr_op, next_op;

  for (curr_op = first_added_in_deps_op; curr_op != NULL; curr_op = next_op)
    {
      sort_op_in_deps (curr_op);
      next_op = curr_op->next_added_in_deps_op;
      curr_op->next_added_in_deps_op = NULL;
    }
  first_added_in_deps_op = last_added_in_deps_op = NULL;
}

/* Hash function for DEP.  */

static unsigned
dep_hash (const void * dep)
{
  dep_ptr d = (dep_ptr) dep;
  return loc_hash (d->to_loc) + ((unsigned) d->from << 8);
}

/* The following function returns TRUE if DEP1 has the same key as
   DEP2.  */

static int
dep_eq (const void * dep1, const void * dep2)
{
  dep_ptr d1 = (dep_ptr) dep1;
  dep_ptr d2 = (dep_ptr) dep2;

  return d1->from == d2->from && loc_eq_p (d1->to_loc, d2->to_loc);
}

/* The function initiates work with deps.  */

static void
start_deps (void)
{
  all_deps = unused_deps = NULL;
  deps_table = htab_create (1000, dep_hash, dep_eq, NULL);
  in_deps_array = NULL;
  first_added_in_deps_op = last_added_in_deps_op = NULL;
}

/* The function finishes all work with deps.  */

static void
finish_deps (void)
{
  dep_ptr curr_dep, next_dep;

  for (curr_dep = all_deps;
       curr_dep != NULL;
       curr_dep = next_dep)
    {
      next_dep = curr_dep->all_deps_chain;
      rcsp_free (curr_dep);
    }
  htab_delete (deps_table);
  if (in_deps_array != NULL)
    rcsp_free (in_deps_array);
}



/* The page contains functions for work with software pipeliner
   generalized states.  When we found the same state that means that
   we found the same repetitive pattern in software pipelining, and
   instead of packet being formed we use already formed packet
   corresponding to the found software pipeliner generalized state.  */

/* Hash function for SCHEDULE_STATE.  */

static unsigned
schedule_state_hash (const void * packet)
{
  packet_ptr p = (packet_ptr) packet;
  unsigned result = 0;
  avail_list_ptr avail;
  op_ptr curr_op;
  int i;

  for (i = 0; i < cpu_state_length; i++)
    result
      += ((unsigned char *) p->cpu_state) [i] << (i % sizeof (unsigned) * 8);
  for (curr_op = p->first_op; curr_op != NULL; curr_op = curr_op->next_op)
    if (curr_op->initiator_op != NULL)
      result += insn_hash (curr_op->initiator_op->insn);
    else
      {
	result += insn_hash (curr_op->insn);
	if (cond_branch_p (curr_op->insn))
	  result += !curr_op->true_jump_p;
      }
  for (avail = p->avail_list; avail != NULL; avail = avail->next_avail)
    {
      /* To speed it up, Take only available operations into account.  */
      if (!avail->avail_p)
	break;
      /* Ignore iteration number.  */
      if (avail->op->initiator_op != NULL)
	result += (insn_hash (avail->op->initiator_op->insn) + avail->delay);
      else
	result += (insn_hash (avail->op->insn) + avail->delay);
    }
  return result;
}

/* The following function returns TRUE if OP1 is equal to OP2 with the
   point of view of scheduled state equality. *UNDEFINED_DIFF_P and
   *DIFF are in/out parameters used to comparison of the operation
   iteration numbers.  */

static int
schedule_state_op_eq (op_ptr op1, op_ptr op2, int * undefined_diff_p, int * diff)
{
  if (op1->initiator_op != op2->initiator_op)
    return FALSE;
  else if (op1->initiator_op != NULL)
    {
      if (!insn_eq_p (op1->initiator_op->insn, op2->initiator_op->insn))
	return FALSE;
      else if (op1->initiator_op->packet->exit_packet_p)
	{
	  if (!op2->initiator_op->packet->exit_packet_p)
	    abort ();
	  /* We require equality for iteration numbers.  */
	  if (op1->initiator_op->it_num != op2->initiator_op->it_num)
	    return FALSE;
	}
      else if (*undefined_diff_p)
	{
	  *diff = (op1->initiator_op->it_num - op2->initiator_op->it_num);
	  *undefined_diff_p = FALSE;
	}
      else if (op1->initiator_op->it_num - op2->initiator_op->it_num != *diff)
	return FALSE;
    }
  else if (!insn_eq_p (op1->insn, op2->insn)
	   || (!insn_dummy_p (op1->insn) && cond_branch_p (op1->insn)
	       && !op1->true_jump_p != !op2->true_jump_p))
    return FALSE;
  else if (op1->packet->exit_packet_p && op2->packet->exit_packet_p)
    {
      /* We require equality for iteration numbers.  */
      if (op1->it_num != op2->it_num)
	return FALSE;
    }
  else if (op1->packet->exit_packet_p || op2->packet->exit_packet_p)
    /* Insns from loop and exit branch are never equal.  */
    return FALSE;
  else if (*undefined_diff_p)
    {
      *diff = op1->it_num - op2->it_num;
      *undefined_diff_p = FALSE;
    }
  else if (op1->it_num - op2->it_num != *diff)
    return FALSE;
  return TRUE;
}

/* The function returns TRUE if SCHEDULE_STATE1 is equal to
   SCHEDULE_STATE2 with the point of view of the pipeliner.  It means
   that they have the same CPU_STATE and sets "available" with
   possible the same differences in the packet operations.  */

static int
schedule_state_eq (const void * packet1, const void * packet2)
{
  packet_ptr p1 = (packet_ptr) packet1;
  packet_ptr p2 = (packet_ptr) packet2;
  avail_list_ptr avail1, avail2;
  op_ptr curr_op1, curr_op2;
  int diff = 0;
  int undefined_diff_p;

  if (memcmp (p1->cpu_state, p2->cpu_state, cpu_state_length) != 0)
    return FALSE;
  undefined_diff_p = TRUE;
  for (curr_op1 = p1->first_op, curr_op2 = p2->first_op;
       curr_op1 != NULL && curr_op2 != NULL;
       curr_op1 = curr_op1->next_op, curr_op2 = curr_op2->next_op)
    if (!schedule_state_op_eq (curr_op1, curr_op2, &undefined_diff_p, &diff))
      return FALSE;
  if (curr_op1 != curr_op2)
    return FALSE;
  for (avail1 = p1->avail_list, avail2 = p2->avail_list;
       avail1 != NULL && avail2 != NULL;
       avail1 = avail1->next_avail, avail2 = avail2->next_avail)
    {
      curr_op1 = avail1->op;
      curr_op2 = avail2->op;
      if (curr_op1 == NULL || curr_op2 == NULL)
        abort ();
      if (avail1->avail_p != avail2->avail_p
	  || !schedule_state_op_eq (curr_op1, curr_op2,
				    &undefined_diff_p, &diff))
	return FALSE;
      /* Take delays into account for non-exit packets.  */ 
      if ((!curr_op1->packet->exit_packet_p
	   || !curr_op2->packet->exit_packet_p)
	  && avail1->avail_p && avail1->delay != avail2->delay)
	return FALSE;
    }
  if (avail1 != avail2)
    return FALSE;
  if (p1->exit_packet_p && p2->exit_packet_p)
    {
      /* Available lists are not sufficient for exit packets.  We
         could check all achievable packets may be ignoring iteration
         numbers. ??? But now we do not do it.  The question is it
         worth to do it, especially if we implement temporary register
         renaming.  Because in this case we can not ignore iteration
         numbers.  */
      return FALSE;
    }
  return TRUE;
}

/* The following function creates empty scheduler states set.  */

static packet_set_t
packet_set_create (void)
{
  return htab_create (1000, schedule_state_hash, schedule_state_eq, NULL);
}

/* The following function deletes PACKET_SET.  */

static void
packet_set_delete (packet_set_t packet_set)
{
  htab_delete (packet_set);
}

/* The function inserts PACKET into PACKET_SET if it is not there.
   The function returns the packet which is in PACKET_SET.  */

static packet_ptr
packet_set_insert (packet_set_t packet_set, packet_ptr packet)
{
  void **slot;

  slot = htab_find_slot (packet_set, packet, TRUE);
  if (*slot == NULL)
    *slot = (void *) packet;
  return (packet_ptr) *slot;
}


/* This page contains functions for work with frontier packet list.  */

/* The following variable refers for the frontier packet list head.
   It should be only read from outside the page.  */
static packet_ptr first_frontier_packet;

/* The following variables refer for array of pointers to packets and
   its length in the pointers.  This array is used to sort the frontier
   packet list.  */
static packet_ptr *frontier_packet_array;
static int frontier_packet_array_length;

/* The following function returns the frontier packet list head and
   removes it from the list.  */

static packet_ptr
frontier_packet_list_head_remove (void)
{
  packet_ptr result = first_frontier_packet;
  
  if (first_frontier_packet == NULL)
    abort ();
  first_frontier_packet = first_frontier_packet->next_frontier_packet;
  if (!result->frontier_packet_p)
    abort ();
  result->frontier_packet_p = FALSE;
  return result;
}

/* The following function returns the frontier packet list length.  */

static int
frontier_packet_list_length (void)
{
  packet_ptr curr_packet;
  int length;

  for (length = 0, curr_packet = first_frontier_packet;
       curr_packet != NULL;
       curr_packet = curr_packet->next_frontier_packet)
    length++;
  return length;
}

/* The following function adds PACKET to list of frontier packets.  */

static void
add_frontier_packet (packet_ptr packet)
{
  if (packet->frontier_packet_p || packet->formed_packet_p)
    abort ();
  packet->frontier_packet_p = TRUE;
  packet->next_frontier_packet = first_frontier_packet;
  first_frontier_packet = packet;
}

/* The following function is used as comparison function to sort
   frontier packet list to choose the first frontier packet for
   scheduling.  The first we schedule packets whose successors are not
   point of CFG merge.  Between such packets we choose packet whose
   minimal op iteration number is minimal.  Then we choose the first
   created frontier packet.  Packets with the same merge point will be
   adjacent.  Remember that frontier packets has only one
   successor.  */

static int
frontier_packet_sort_cmp (const void * packet1, const void * packet2)
{
  packet_ptr p1 = *(packet_ptr *) packet1;
  packet_ptr p2 = *(packet_ptr *) packet2;
  
  if (p1->succ [0]->pred [1] == NULL)
    {
      if (p2->succ [0]->pred [1] != NULL)
	return -1; /* choose packet whose successor is not CFG merge point.  */
      else if (p1->min_it_n != p2->min_it_n)
	return p1->min_it_n - p2->min_it_n; /* chose the packet with
					       minimal iteration number.  */
      else
	return p1->number - p2->number; /* chose the first created packet */
    }
  else if (p2->succ [0]->pred [1] == NULL)
    return 1; /* choose packet whose successor is not CFG merge point.  */
  else
    /* Choose packet according to the merge point.  */
    return p1->succ [0]->number - p2->succ [0]->number;
}

/* The following function sorts the frontier packet list and merges
   frontier packets with the same successor which is CFG merge point
   (in other words which has more one predecessor).  FILE and PREFIX
   are used for debugging output.  ??? Merging is necessary if we have
   constraint mentioned in get_best_candidate_avail_op.  Otherwise
   merging probably results in more compact and less effective
   pipelined loop.  Investigate this.  */

static void
frontier_packet_list_sort_and_merge (FILE * f, const char * prefix)
{
  int i;
  packet_ptr curr_packet, pred_packet;
  int length = frontier_packet_list_length ();
  int decr_time;

  if (first_frontier_packet == NULL)
    abort ();
  if (frontier_packet_array == NULL)
    {
      frontier_packet_array = rcsp_malloc (sizeof (packet_ptr) * length);
      frontier_packet_array_length = length;
    }
  else if (frontier_packet_array_length < length)
    {
      frontier_packet_array
	= rcsp_realloc (frontier_packet_array,
			sizeof (packet_ptr) * frontier_packet_array_length,
			sizeof (packet_ptr) * length);
      frontier_packet_array_length = length;
    }
  for (i = 0, curr_packet = first_frontier_packet;
       curr_packet != NULL;
       i++, curr_packet = curr_packet->next_frontier_packet)
    {
      if (curr_packet->succ [0] == NULL || curr_packet->succ [1] != NULL)
	abort ();
      frontier_packet_array [i] = curr_packet;
    }
  if (i != length)
    abort ();
  qsort (frontier_packet_array, length, sizeof (packet_ptr),
	 frontier_packet_sort_cmp);
  pred_packet = NULL;
  for (i = 0; i < length; i++)
    {
      curr_packet = frontier_packet_array [i];
      if (pred_packet != NULL)
	pred_packet->next_frontier_packet = curr_packet;
      if (pred_packet != NULL
	  && curr_packet->first_op == NULL && pred_packet->first_op == NULL
	  && curr_packet->succ [0]->pred [1] != NULL
	  && pred_packet->succ[0]->number == curr_packet->succ[0]->number)
	/* Merge two frontier packets (CURR_PACKET and PRED_PACKET)
	   into PRED_PACKET: */
	{
#ifdef RCSP_DEBUG
	  if (rcsp_debug_level >= 2)
	    fprintf (f, "\n%s\n%s merging packets %3d and %3d into %3d\n",
		     prefix, prefix, pred_packet->number, curr_packet->number,
		     pred_packet->number);
#endif
	  /* Changing CFG: Redirect CURR_PACKET predecessors onto
             PRED_PACKET.  */
	  while (curr_packet->pred[0] != NULL)
	    /* Remember: packet_change_dest removes the link from pred
	       by shifting array `pred'.  */
	    packet_change_dest (curr_packet->pred[0], curr_packet,
				pred_packet);
	  /* Remember: packet_change_src removes the link from succ by
	     shifting array `succ'.  */
	  packet_change_src (curr_packet->succ[0], curr_packet, NULL);
	  if (curr_packet->succ[0] != NULL)
	    abort ();
	  /* Set packet issue time, CPU state and delay for available
             ops.  */
	  if (pred_packet->issue_time < curr_packet->issue_time)
	    {
	      decr_time = curr_packet->issue_time - pred_packet->issue_time;
	      while (decr_time-- != 0)
		cpu_state_advance_cycle (pred_packet->cpu_state);
	      pred_packet->issue_time = curr_packet->issue_time;
	    }
	  else if (pred_packet->issue_time > curr_packet->issue_time)
	    {
	      decr_time = pred_packet->issue_time - curr_packet->issue_time;
	      while (decr_time-- != 0)
		cpu_state_advance_cycle (curr_packet->cpu_state);
	    }
	  cpu_state_union (pred_packet->cpu_state, curr_packet->cpu_state);
	  packet_delete (curr_packet);
	}
      else
	pred_packet = curr_packet;
    }
  if (pred_packet == NULL)
    abort ();
  pred_packet->next_frontier_packet = NULL;
  first_frontier_packet = frontier_packet_array [0];
}


/* The page contains functions to start and finish work with all
   abstract data needed for pipelining.  */

/* The following function initiates work with the frontier packet list.  */

static void
start_frontier_packet_list (void)
{
  first_frontier_packet = NULL;
  frontier_packet_array = NULL;
}

/* Start work with abstract data needed for pipelining.  */

void
start_pipeline_data (void)
{
  cpu_state_length = cpu_state_size ();
  temp_cpu_state = rcsp_malloc (cpu_state_length);
  start_ops ();
  start_avail_lists ();
  start_packets ();
  start_deps ();
  start_frontier_packet_list ();
}

/* The following function finishes work with the frontier packet
   list.  */

static void
finish_frontier_packet_list (void)
{
  if (frontier_packet_array != NULL)
    rcsp_free (frontier_packet_array = NULL);
}
/* Finish work with abstract data needed for pipelining.  */

void
finish_pipeline_data (void)
{
  finish_frontier_packet_list ();
  finish_deps ();
  finish_packets ();
  finish_avail_lists ();
  finish_ops ();
  rcsp_free (temp_cpu_state);
}


/* The page contains functions for building original DDG.  These
   functions work with locrefs used/defined by operations.  We keep
   set of the last active locrefs used/defined by operations.  */

/* The following set contains recent locrefs defined or used in ops.
   The set is used for faster building original DDG.  The key is
   location and operation.  */
static locrefset_t last_op_refs;

/* The following variable refers for unused locrefs.  */
static locref_ptr free_locrefs;

/* The following function frees LOCREF.  */
static void
free_locref (locref_ptr locref)
{
  locref->next_locref = free_locrefs;
  free_locrefs = locref;
}

/* The following function returns free locref.  */

static locref_ptr
get_free_locref (void)
{
  locref_ptr result;

  if (free_locrefs)
    {
      result = free_locrefs;
      free_locrefs = free_locrefs->next_locref;
    }
  else
    result = rcsp_malloc (sizeof (struct locref));
  return result;
}

/* The following function returns hash key of LOCREF.  */

static unsigned
locref_hash (const void * locref)
{
  locref_ptr l = (locref_ptr) locref;

  return loc_hash (l->loc) + ((unsigned) l->op << 8);
}

/* The following function returns TRUE if locref1 has the same key as
   LOCRE2.  */

static int
locref_eq (const void * locref1, const void * locref2)
{
  locref_ptr l1 = (locref_ptr) locref1;
  locref_ptr l2 = (locref_ptr) locref2;

  return l1->op == l2->op && loc_eq_p (l1->loc, l2->loc);
}

/* The following function returns list of active locrefs with LOC
   used/defined in OP.  */

static locref_ptr
locref_find (loc_ptr loc, op_ptr op)
{
  struct locref locref;

  locref.loc = loc;
  locref.op = op;
  return (locref_ptr) htab_find (last_op_refs, &locref);
}

/* The following function adds locref with LOC used (if USAGE_P) or
   defined in OP to the list of active locrefs.  */

static void
locref_add (op_ptr op, loc_ptr loc, int usage_p, int write_barrier_p)
{
  void **slot;
  locref_ptr last_locref;
  locref_ptr l = get_free_locref ();
  
  l->op = op;
  l->loc = loc;
  l->usage_p = usage_p;
  l->write_barrier_p = write_barrier_p;
  l->next_locref = NULL;
  slot = htab_find_slot (last_op_refs, l, TRUE);
  if (*slot == NULL)
    *slot = l;
  else
    {
      for (last_locref = (locref_ptr) *slot;
	   last_locref->next_locref != NULL; 
	   last_locref = last_locref->next_locref)
	;
      last_locref->next_locref = l;
    }
}

/* The following function removes all active locrefs with LOC
   used/defined in OP.  */

static void
locref_remove_all (loc_ptr loc, op_ptr op)
{
  struct locref locref;
  locref_ptr curr_locref, next_locref;
  void **slot;
  
  locref.loc = loc;
  locref.op = op;
  slot = htab_find_slot (last_op_refs, &locref, FALSE);
  if (slot != NULL)
    {
      for (curr_locref = (locref_ptr) *slot;
	   curr_locref != NULL; 
	   curr_locref = next_locref)
	{
	  next_locref = curr_locref->next_locref;
	  free_locref (curr_locref);
	}
      htab_clear_slot (last_op_refs, slot);
    }
}

/* The function initiates work with locrefs.  */

static void
start_locrefs (void)
{
  free_locrefs = NULL;
  last_op_refs = htab_create (1000, locref_hash, locref_eq, NULL);
}

/* The following function is used to remove locrefs staring with
   LOCREF which are in the hash table.  */

static int
free_htab_locrefs (void **locref_ref, void *param ATTRIBUTE_UNUSED)
{
  locref_ptr curr_locref, next_locref;

  for (curr_locref = (locref_ptr) *locref_ref;
       curr_locref != NULL; 
       curr_locref = next_locref)
    {
      next_locref = curr_locref->next_locref;
      free_locref (curr_locref);
    }
  return TRUE;
}

/* The function finishes all work with locrefs.  */

static void
finish_locrefs (void)
{
  locref_ptr curr_locref, next_locref;

  htab_traverse (last_op_refs, free_htab_locrefs, NULL);
  for (curr_locref = free_locrefs;
       curr_locref != NULL; 
       curr_locref = next_locref)
    {
      next_locref = curr_locref->next_locref;
      rcsp_free (curr_locref);
    }
  htab_delete (last_op_refs);
}



/* The page contains functions of initial building DDG.  */

/* The following creates dummy packet (see comments for
   `top_packet').  */

packet_ptr
create_dummy_packet (void)
{
  op_ptr dummy_op;
  packet_ptr dummy_packet;
  
  dummy_packet = packet_create (0);
  dummy_op = op_create (NULL, 0);
  packet_append_op (dummy_packet, dummy_op, TRUE);
  return dummy_packet;
}

/* The following function adds reference for location LOC used (if
   USAGE_P) or defined in operation OP.  If LOC is defined then all
   locations in LOC defined or used in OP are removed.  The function
   also sets up flag `use_def_p' if it is necessary.  */

static void
add_locref (op_ptr op, loc_ptr loc, int usage_p, write_kind_t write_kind)
{
  loc_ptr curr_loc;
  int i;

  if (!usage_p)
    {
      if (write_kind != WRITE_BARRIER && locref_find (loc, op) != NULL)
	op->use_def_p = TRUE;
      if (write_kind == USUAL_WRITE)
	locref_remove_all (loc, op);
      for (i = 0; ; i++)
	{
	  curr_loc = loc_sub (loc, i == 0);
	  if (curr_loc == NULL)
	    break;
	  if (write_kind != WRITE_BARRIER
	      && locref_find (curr_loc, op) != NULL)
	    op->use_def_p = TRUE;
	  if (write_kind == USUAL_WRITE)
	    locref_remove_all (curr_loc, op);
	}
    }
  locref_add (op, loc, usage_p, write_kind == WRITE_BARRIER);
}

/* The following function processes location LOC (which is a super or
   sub location of OP_LOC) to find and to add (if it is found) new
   dependency from PRED_OP to OP on OP_LOC used (if USAGE_P) or
   defined in OP.  */

static int
process_locrefs (op_ptr pred_op, loc_ptr loc, int usage_p,
		 int write_barrier_p, op_ptr op, loc_ptr op_loc)
{
  int        cont_p = TRUE;
  locref_ptr locref;
  locref_ptr curr_locref;

  locref = locref_find (loc, pred_op);
  for (curr_locref = locref;
       curr_locref != NULL;
       curr_locref = curr_locref->next_locref)
    if (curr_locref->op != op)
      {
	if (!usage_p)
	  {
	    if (!curr_locref->usage_p)
	      {
		/* Add "def-def" dep.  */
		dep_create (curr_locref->op, op, op_loc, DEF_DEF,
			    insn_dep_time (curr_locref->op->insn, op->insn,
					   DEF_DEF));
		cont_p = FALSE;
	      }
	    else if (!write_barrier_p)
	      /* Add "use-def" dep.  */
	      dep_create (curr_locref->op, op, op_loc, USE_DEF,
			  insn_dep_time (curr_locref->op->insn, op->insn,
					 USE_DEF));
	  }
	else if (!curr_locref->usage_p && !curr_locref->write_barrier_p)
	  {
	    /* Add "def-use" dep.  */
	    dep_create (curr_locref->op, op, op_loc, DEF_USE,
			insn_dep_time (curr_locref->op->insn, op->insn,
				       DEF_USE));
	    cont_p = FALSE;
	  }
      }
  return cont_p;
}

/* This recursive function processes pred packets starting with PACKET
   to add deps on LOC used (if USAGE_P) or defined in OP from another
   op in PACKET.  */

static void
process_packet_op_locs (packet_ptr packet, op_ptr op, loc_ptr loc,
			int usage_p, write_kind_t write_kind)
{
  int pred_number;
  int i;
  int cont_p;
  loc_ptr curr_loc;

  if (packet->visit_number1 == last_packet_visit_number1)
    return;
  packet->visit_number1 = last_packet_visit_number1;
  if (packet == top_packet)
    {
      /* Include dummy definition of loc in the top packet and add
	 dependency arc.  We define dummy definition even for
	 definition too.  It is needed for our algorithm of making
	 interblock operation moving.  */
      add_locref (top_packet->first_op, loc, FALSE, USUAL_WRITE);
      dep_create (top_packet->first_op, op, loc,
		  (usage_p ? DEF_USE : DEF_DEF), 0);
      return;
    }

  cont_p = process_locrefs (packet->first_op, loc,
			    usage_p, write_kind == WRITE_BARRIER, op, loc);
  if (cont_p)
    for (i = 0; ; i++)
      {
	curr_loc = loc_sub (loc, i == 0);
	if (curr_loc == NULL)
	  break;
	process_locrefs (packet->first_op, curr_loc,
			 usage_p, write_kind == WRITE_BARRIER, op, loc);
      }

  if (cont_p)
    for (pred_number = 0;
	 packet->pred [pred_number] != NULL;
	 pred_number++)
      process_packet_op_locs (packet->pred [pred_number], op, loc,
			      usage_p, write_kind);
}

/* The following function processes operation OP for building DDG.  */

static void
process_op_locs (op_ptr op)
{
  int i, usage_p;
  write_kind_t write_kind;
  loc_ptr loc;

  for (i = 0; ; i++)
    {
      loc = getloc (op->insn, i, &usage_p, &write_kind);
      if (loc == NULL)
	break;
      last_packet_visit_number1++;
      process_packet_op_locs (op->packet, op, loc, usage_p, write_kind);
      add_locref (op, loc, usage_p, write_kind);
    }
}

/* This recursive function processes (depth-first CFG traversing)
   PACKET and all its successors to insert them into the packet list
   whose the first element is FIRST_PACKET.  The function returns the
   first element of the list after processing all the packets.  */

static packet_ptr
link_packets (packet_ptr packet, packet_ptr first_packet)
{
  int succ_number;

  if (packet->visit_number1 == last_packet_visit_number1)
    return first_packet;
  packet->visit_number1 = last_packet_visit_number1;
  for (succ_number = 0;
       packet->succ [succ_number] != NULL;
       succ_number++)
    first_packet = link_packets (packet->succ [succ_number], first_packet);
  packet->next_packet = first_packet;
  first_packet = packet;
  return first_packet;
}

/* The following function builds DDG for CFG with root START.  */

static void
build_deps (packet_ptr start)
{
  packet_ptr curr_packet, next_packet;

  start_locrefs ();
  top_packet = create_dummy_packet ();
  packet_connect (top_packet, start, FALSE, FALSE);
  last_packet_visit_number1++;
  link_packets (start, NULL);
  for (curr_packet = start;
       curr_packet != NULL;
       curr_packet = next_packet)
    {
      next_packet = curr_packet->next_packet;
      curr_packet->next_packet = NULL;
      if (curr_packet->first_op != NULL
	  && curr_packet->first_op->next_op != NULL)
	/* At most one insn should be in the packet now.  */
	abort ();
      if (curr_packet->first_op != NULL
	  && !insn_dummy_p (curr_packet->first_op->insn))
	process_op_locs (curr_packet->first_op);
    }
  finish_locrefs ();
  dep_sort_added_in_deps_op ();
}


/* This page contains functions for evaluating CFG achievability,
   dominator and postdominator relation.  */

/* This recursive function returns
     o IGNORE_ACHIEVABILITY if we should ignore this path to calculate
       achievability: there is a cycle in CFG and there are no other
       paths (FROM, TO).
     o UNACHIEVABLE if TO is not achievable from FROM and there are no
       CFG cycles containing path (FROM, TO).
     o BASIC_BLOCK_PATH if TO is achievable from FROM and (TO,FROM) is
       a basic block.
     o OTHER_PATH otherwise.

   Remember the function may be called for formed packets too,
   therefore there may be cycles in CFG.  */

static achievable_mode_t
achievable_1 (packet_ptr from, packet_ptr to)
{
  int succ_number;
  int ignore_p;
  achievable_mode_t result;

  if (from->visit_number1 == last_packet_visit_number1)
    return from->achievable_result;
  from->visit_number1 = last_packet_visit_number1;
  /* This is needed if we walk around a cycle in CFG.  */
  from->achievable_result = IGNORE_ACHIEVABILITY;
  if (from == to)
    result = BASIC_BLOCK_PATH;
  else
    {
      result = UNACHIEVABLE;
      ignore_p = FALSE;
      for (succ_number = 0; from->succ [succ_number] != NULL; succ_number++)
	if ((result = achievable_1 (from->succ [succ_number], to))
	    == IGNORE_ACHIEVABILITY)
	  ignore_p = TRUE;
	else if (result != UNACHIEVABLE)
	  break;
      if (result == UNACHIEVABLE && ignore_p)
	result = IGNORE_ACHIEVABILITY;
      else if (result == BASIC_BLOCK_PATH
	       && (succ_number > 0
		   || from->succ [succ_number + 1] != NULL
		   || from->succ[0]->pred [1] != NULL))
	/* There are side exit from the patch (from, to) or more one
	   enter into the path.  */
	result = OTHER_PATH;
    }
#ifdef USE_ACHIEVABLE_CACHE
  if (CFG_shift <= MAX_CFG_SHIFT && result != IGNORE_ACHIEVABILITY)
    {
      CFG_validation [(from->number << CFG_shift) + to->number] = CFG_tick;
      CFG_achievable [(from->number << CFG_shift) + to->number] = result;
    }
#endif
  from->achievable_result = result;
  return result;
}

/* See comments for function `achievable_1'.  There is one exception
   -- the function never returns IGNORE_ACHIEVABILITY.  */

static achievable_mode_t
achievable (packet_ptr from, packet_ptr to)
{
  achievable_mode_t result;

  last_packet_visit_number1++;
  if (to == NULL || from == NULL)
    abort ();
  result = achievable_1 (from, to);
  if (result == IGNORE_ACHIEVABILITY)
    {
      /* We found cycle in CFG and there are no other paths (FROM,
         TO).  So TO is not accessible from FROM.  Remember that such
         conclusion may be not true for any intermediate CFG node.  */
      result = UNACHIEVABLE;
#ifdef USE_ACHIEVABLE_CACHE
      if (CFG_shift <= MAX_CFG_SHIFT)
	{
	  CFG_validation [(from->number << CFG_shift) + to->number] = CFG_tick;
	  CFG_achievable [(from->number << CFG_shift) + to->number] = result;
	}
#endif
    }
  return result;
}

/* The following recursive function processes PACKET (which may be on
   a path (FROM, TO)) and returns TRUE if FROM is a dominator of
   TO.  */

static int
dom_p_1 (packet_ptr from, packet_ptr packet, packet_ptr to)
{
  int succ_number, pred_number;
  packet_ptr succ, pred;
  achievable_mode_t path;

  path = ACHIEVABLE (packet, to);
  if (path == UNACHIEVABLE)
    return TRUE;
  else if (path == BASIC_BLOCK_PATH)
    return TRUE;
  for (succ_number = 0;
       (succ = packet->succ [succ_number]) != NULL;
       succ_number++)
    {
      if (!dom_p_1 (from, succ, to))
	return FALSE;
      for (pred_number = 0;
	   (pred = succ->pred [pred_number]) != NULL;
	   pred_number++)
	if (pred != succ && ACHIEVABLE (from, pred) == UNACHIEVABLE)
	  return FALSE;
    }
  return TRUE;
}

/* The following function returns TRUE if FROM is a dominator of TO.
   TO and FROM should be not formed packets.  In this case traversed
   CFG part has no cycles.  */

static int
dom_p (packet_ptr from, packet_ptr to)
{
  if (to->formed_packet_p || from->formed_packet_p)
    abort ();
  return dom_p_1 (from, from, to);
}

/* The following recursive function returns TRUE if TO is a
   post-dominator of FROM.  */

static int
postdom_p_1 (packet_ptr to, packet_ptr from)
{
  int succ_number;
  packet_ptr succ;
  achievable_mode_t path;

  path = ACHIEVABLE (from, to);
  if (path == UNACHIEVABLE)
    return FALSE;
  else if (path == BASIC_BLOCK_PATH)
    return TRUE;
  for (succ_number = 0;
       (succ = from->succ [succ_number]) != NULL;
       succ_number++)
    if (!postdom_p_1 (to, succ))
      return FALSE;
  return TRUE;
}

/* The following function returns TRUE if TO is a post-dominator of
   FROM.  TO and FROM should be not formed packets.  In this case
   traversed CFG part has no cycles.  */

static int
postdom_p (packet_ptr to, packet_ptr from)
{
  if (to->formed_packet_p || from->formed_packet_p)
    abort ();
  return postdom_p_1 (to, from);
}


/* This page contains functions for defining does moving operation
   kill a live location.  */

/* When we are processing operation FROM on which OP is depended, the
   following function returns TRUE if live LOC is killed after moving
   OP to PACKET.  */

static int
kill_loc_move_p (op_ptr op ATTRIBUTE_UNUSED, packet_ptr packet, 
		 op_ptr from, loc_ptr loc)
{
  dep_ptr curr_out_dep;

  for (curr_out_dep = dep_out_find (from, loc);
       curr_out_dep != NULL;
       curr_out_dep = curr_out_dep->next_out_loc_dep)
    if (curr_out_dep->dep_mode == DEF_USE
	/* We can place OP after insn which uses LOC in the same package: */
	&& packet != curr_out_dep->to->packet
	/* Ignore OP itself: */
	&& op->packet != curr_out_dep->to->packet
	&& ACHIEVABLE (packet, curr_out_dep->to->packet) != UNACHIEVABLE)
      return TRUE;
  return FALSE;
}

/* The following function returns TRUE if a live location is killed
   after moving OP to PACKET.  */

static int
kill_move_p (op_ptr op, packet_ptr packet)
{
  dep_ptr curr_in_dep;
  loc_ptr curr_loc;
  int i;

  if (ACHIEVABLE (packet, op->packet) == BASIC_BLOCK_PATH)
    /* Moving in basic block.  */
    return FALSE;
  for (curr_in_dep = op->in_deps;
       curr_in_dep != NULL;
       curr_in_dep = curr_in_dep->next_in)
    if (curr_in_dep->dep_mode == DEF_DEF
	&& ACHIEVABLE (curr_in_dep->from->packet, packet) != UNACHIEVABLE)
      {
        if (kill_loc_move_p (op, packet, curr_in_dep->from,
			     curr_in_dep->to_loc))
          return TRUE;
	for (i = 0; ; i++)
          {
            curr_loc = loc_super (curr_in_dep->to_loc, i == 0);
            if (curr_loc == NULL)
              break;
            if (kill_loc_move_p (op, packet, curr_in_dep->from, curr_loc))
              return TRUE;
          }

	for (i = 0; ; i++)
	  {
	    curr_loc = loc_sub (curr_in_dep->to_loc, i == 0);
	    if (curr_loc == NULL)
	      break;
	    if (kill_loc_move_p (op, packet, curr_in_dep->from, curr_loc))
	      return TRUE;
	  }
      }
  return FALSE;
}



/* This page contains functions for evaluation of set "covered".  */

/* The following recursive function marks (transitively) all packets
   (except for PACKET) which are are achievable from PACKET and which
   are sources of dependencies for OP.  */

static void
mark_src_packets (op_ptr op, packet_ptr packet)
{
  packet_ptr from_packet;
  dep_ptr curr_in_dep;

  if (op->packet->visit_number2 == last_packet_visit_number2)
    return;
  for (curr_in_dep = op->in_deps;
       curr_in_dep != NULL;
       curr_in_dep = curr_in_dep->next_in)
    {
      from_packet = curr_in_dep->from->packet;
      if (packet != from_packet
	  && ACHIEVABLE (packet, from_packet) != UNACHIEVABLE)
	{
	  mark_src_packets (curr_in_dep->from, packet);
	  from_packet->visit_number2 = last_packet_visit_number2;
	}
    }
}

/* The following variable is list of packets which are in set
   `covered' (see comments for function `mark_covered').  */
static packet_ptr covered_list;

/* The following recursive function traverses CFG starting with PACKET
   and marks packets which are in set `covered' for OP and increments
   COVERED_SIZE after each marking.  The function returns TRUE if
   PACKET is in set `covered'.  */

static int
mark_covered_1 (op_ptr op, packet_ptr packet, int * covered_size)
{
  int succ_number;
  packet_ptr succ;

  if (packet->visit_number2 == last_packet_visit_number2 - 1)
    /* Op is src of a dependency for OP.  It can not be in set
       `covered'.  It is supposed that before this traversing CFG,
       traversing for marking packets which are sources of
       dependencies for OP has been made.  */
    return FALSE;
  else if (packet->visit_number2 == last_packet_visit_number2)
    /* Op is already in set `covered' */
    return TRUE;
  else if (op->packet == packet)
    {
      /* Mark that packet of OP is in set `covered' because it is end
	 of all paths.  */
      packet->visit_number2 = last_packet_visit_number2;
      packet->next_packet = covered_list;
      covered_list = packet;
      (*covered_size)++;
      return TRUE;
    }
  for (succ_number = 0;
       (succ = packet->succ [succ_number]) != NULL;
       succ_number++)
    if (ACHIEVABLE (succ, op->packet) != UNACHIEVABLE
	&& mark_covered_1 (op, succ, covered_size)
        && packet->visit_number2 != last_packet_visit_number2)
      {
        /* Mark that packet is in set `covered'.  */
        packet->visit_number2 = last_packet_visit_number2;
	packet->next_packet = covered_list;
	covered_list = packet;
        (*covered_size)++;
      }
  return packet->visit_number2 == last_packet_visit_number2;
}

/* The following function marks all packets which are in set `covered'
   for OP and PACKET.  The packet is in the set if there is a path
   (PACKET, OP) in CFG and OP is not data dependent from any packets
   on the path.  The function returns size of the set.  If OP can be
   moved into PACKET (i.e. there is a such path), the set contains
   PACKET and OP too.  The function also forms list of packets in set
   `covered'.  */

static int
mark_covered (op_ptr op, packet_ptr packet)
{
  int covered_size = 0;

  if (op->packet == packet)
    abort ();
  last_packet_visit_number2++;
  mark_src_packets (op, packet);
  last_packet_visit_number2++;
  covered_list = NULL;
  mark_covered_1 (op, packet, &covered_size);
  return covered_size;
}



/* This page contains functions for defining move consistence and cost
   of moving operation.  */

/* The following recursive function traverses CFG starting with PACKET
   and counts number of packets which should be copied in order to
   make moving OP (see function move_cost).  It is supposed that
   marking packets in set `covered' made before this.  */

static int
traverse_CFG_for_consistent_cost (op_ptr op, packet_ptr packet)
{
  int cost = 0;
  int succ_number;
  packet_ptr succ;

  if (packet->visit_number3 == last_packet_visit_number3)
    return 0;
  packet->visit_number3 = last_packet_visit_number3;
  if (packet->visit_number2 == last_packet_visit_number2)
    /* packet is in set `covered'.  */
    cost++;
  if (packet != op->packet)
    for (succ_number = 0;
	 (succ = packet->succ [succ_number]) != NULL;
	 succ_number++)
      if (ACHIEVABLE (succ, op->packet) != UNACHIEVABLE)
        cost += traverse_CFG_for_consistent_cost (op, succ);
  return cost;
}

/* The following function evaluates cost of moving OP into PACKET.
   The cost is number of necessary packet copies for making CGF
   consistent for moving OP and size of covered set when OP is a
   branch.  Negative cost means that OP can not be moved into PACKET.
   Otherwise, the function also sets up parameter INCONSISTENCE_P when
   making CFG move consistence is required.  */

static int
move_cost (op_ptr op, packet_ptr packet, int * inconsistence_p)
{
  packet_ptr curr_packet;
  dep_ptr curr_in_dep;
  int cost, covered_size;
  
  if (op->packet == packet)
    abort ();
  if (ACHIEVABLE (packet, op->packet) == UNACHIEVABLE
      || kill_move_p (op, packet))
    return -1;
  covered_size = mark_covered (op, packet);
  if (covered_size == 0)
    return -1;
  if (covered_size < 2)
    abort ();
  cost = 0;
  /* Add number packet copies for making CFG move consistent.  */
  last_packet_visit_number3++;
  for (curr_in_dep = op->in_deps;
       curr_in_dep != NULL;
       curr_in_dep = curr_in_dep->next_in)
    if (packet != curr_in_dep->from->packet
	&& ACHIEVABLE (packet, curr_in_dep->from->packet) != UNACHIEVABLE)
      cost += traverse_CFG_for_consistent_cost (op, curr_in_dep->from->packet);
  /* If OP is achievable from another frontier packet, take into
     account necessary copies on path from the frontier packet to OP.  */
  if (!dom_p (packet, op->packet))
    for (curr_packet = first_frontier_packet;
	 curr_packet != NULL;
	 curr_packet = curr_packet->next_frontier_packet)
      {
	if (curr_packet == packet)
	  abort ();
	if (ACHIEVABLE (curr_packet, op->packet) != UNACHIEVABLE)
	  cost += traverse_CFG_for_consistent_cost (op, curr_packet);
      }
  if (cost != 0 && op->use_def_p)
    return -1;
  *inconsistence_p = cost != 0;
  return (cond_branch_p (op->insn) ? covered_size - 2 : 0) + cost;
}



/* The following function returns negative value if OP is not ready
   for PACKET (i.e. there is no a path from root to OP through PACKET
   on which all ops whose outputs are OP inputs have been issued).
   Otherwise the function returns absolute maximal time when all the
   inputs will be ready.  */

static int
get_op_ready_time (op_ptr op, packet_ptr packet)
{
  int loc_value_ready_p;
  dep_ptr curr_in_dep;
  loc_ptr prev_loc;
  int issue_time, ready_time;

  prev_loc = NULL;
  loc_value_ready_p = TRUE;
  ready_time = 0;
  for (curr_in_dep = op->in_deps;
       curr_in_dep != NULL;
       curr_in_dep = curr_in_dep->next_in)
    if (curr_in_dep->dep_mode == DEF_USE)
      {
	if (prev_loc == NULL || !loc_eq_p (prev_loc, curr_in_dep->to_loc))
	  {
	    if (!loc_value_ready_p)
	      return -1;
	    loc_value_ready_p = FALSE;
	    prev_loc = curr_in_dep->to_loc;
	  }
	if (curr_in_dep->from->packet == packet
	    || (curr_in_dep->from->packet->formed_packet_p
		&& ACHIEVABLE (curr_in_dep->from->packet, packet)
		   != UNACHIEVABLE))
	  {
	    loc_value_ready_p = TRUE;
	    issue_time = curr_in_dep->from->packet->issue_time;
	    if (issue_time < 0)
	      abort ();
	    if (ready_time < issue_time + curr_in_dep->latency)
	      ready_time = issue_time + curr_in_dep->latency;
	  }
      }
  return (loc_value_ready_p ? ready_time : -1);
}



/* This page contains functions to make CFG move consistent.  */

/* The following variable is list of copied packets in order to make
   CFG is move consistent.  */
static packet_ptr copied_packets_4_move_consistence;

/* The following function sets up in/out dependencies for copied
   packets.  */

static void
copy_dependencies_4_consistent_move (packet_ptr packet)
{
  dep_ptr in_deps, out_deps, curr_dep;

  if (packet->visit_number2 != last_packet_visit_number2
      || packet->visit_number3 != last_packet_visit_number3)
    abort ();
  if (packet->first_op == NULL || packet->first_op->next_op != NULL)
    abort ();
  in_deps = packet->first_op->in_deps;
  out_deps = packet->first_op->out_deps;
  if (packet->copy->first_op->next_op != NULL)
    abort ();
  for (curr_dep = out_deps; curr_dep != NULL; curr_dep = curr_dep->next_out)
    {
      if (curr_dep->to->packet->first_op->next_op != NULL)
	abort ();
      if (curr_dep->to->packet->visit_number2 == last_packet_visit_number2)
	{
	  if (curr_dep->to->packet->visit_number3 != last_packet_visit_number3)
	    abort ();
	  dep_create (packet->copy->first_op,
		      curr_dep->to->packet->copy->first_op,
		      curr_dep->to_loc, curr_dep->dep_mode, curr_dep->latency);
	}
      else
	dep_create (packet->copy->first_op, curr_dep->to->packet->first_op,
		    curr_dep->to_loc, curr_dep->dep_mode, curr_dep->latency);
    }
  for (curr_dep = in_deps; curr_dep != NULL; curr_dep = curr_dep->next_in)
    if (curr_dep->from->packet->visit_number2 == last_packet_visit_number2
	&& curr_dep->from->packet->visit_number3 == last_packet_visit_number3)
      {
	op_ptr curr_op, curr_copy_op;

	/* Input dependency may be from formed packet which may have
           more one operation.  */
	for (curr_op = curr_dep->from->packet->first_op,
	       curr_copy_op = curr_dep->from->packet->copy->first_op;
	     curr_op != curr_dep->from;
	     curr_op = curr_op->next_op, curr_copy_op = curr_copy_op->next_op)
	  ;
	dep_create (curr_copy_op, packet->copy->first_op, curr_dep->to_loc,
		    curr_dep->dep_mode, curr_dep->latency);
      }
    else
      dep_create (curr_dep->from, packet->copy->first_op,
		  curr_dep->to_loc, curr_dep->dep_mode, curr_dep->latency);
}

/* The following function removes obsolete (from unachievable packet
   operations) input dependencies of operation of interior PACKET and
   its copy.  */

static void
remove_obsolete_dependencies_4_move_consistence (packet_ptr packet)
{
  dep_ptr curr_dep, next_dep;

  if (packet->first_op == NULL || packet->first_op->next_op != NULL)
    abort ();
  for (curr_dep = packet->first_op->in_deps;
       curr_dep != NULL;
       curr_dep = next_dep)
    {
      next_dep = curr_dep->next_in;
      if (ACHIEVABLE (curr_dep->from->packet, packet) == UNACHIEVABLE)
	dep_delete (curr_dep);
    }
  for (curr_dep = packet->copy->first_op->in_deps;
       curr_dep != NULL;
       curr_dep = next_dep)
    {
      next_dep = curr_dep->next_in;
      if (ACHIEVABLE (curr_dep->from->packet, packet->copy) == UNACHIEVABLE)
	dep_delete (curr_dep);
    }
}

/* The following function modifies CFG which was not valid after
   making move consistent packets copies (PACKET is a packet for which
   move consistent copy has been made).  Actually the order of
   processing packets for which move consistent copies have been made
   is not important.  Practically each packet is processed twice: as
   pred and as succ.  */

static void
modify_packet_CFG (packet_ptr packet)
{
  packet_ptr pred, succ;
  int pred_number, succ_number;

  if (packet->visit_number2 != last_packet_visit_number2
      || packet->visit_number3 != last_packet_visit_number3)
    abort ();
  for (pred_number = 0;
       (pred = packet->pred [pred_number]) != NULL;
       pred_number++)
    if (pred->visit_number2 == last_packet_visit_number2)
      {
	/* pred is in `covered' */
	if (pred->visit_number3 != last_packet_visit_number3)
	  /* No copy of pred is made: remove arc (pred, packet->copy).  */
	  packet_change_dest (pred, packet->copy, packet);
	else
	  /* Copy is made: remove possible arc (pred->copy, packet)
	     and add arc (pred->copy, packet->copy).  */
	  packet_change_dest (pred->copy, packet, packet->copy);
      }
    else
      {
	/* Pred is not in `covered' (it means there is no copy):
	   remove arc (pred, packet) and add arc (pred, packet->copy).
	   Remember: packet_change_dest removes the link from pred by
	   shifting array `pred'.*/
	packet_change_dest (pred, packet, packet->copy);
	pred_number--;
      }

  for (succ_number = 0;
       (succ = packet->succ [succ_number]) != NULL;
       succ_number++)
    if (succ->visit_number2 == last_packet_visit_number2)
      {
	/* succ is in `covered' */
	if (succ->visit_number3 != last_packet_visit_number3)
	  /* It should be visited.  */
	  abort ();
	packet_change_src (succ->copy, packet, packet->copy);
      }
    else
      /* We do not change true or false target in branch because they
         are the same after copying.  */
      packet_connect (packet->copy, succ, FALSE, FALSE);
}

/* The following function makes GFG consistent for moving OP into
   packet TO.  */

static void
make_move_consistent (op_ptr op, packet_ptr to)
{
  packet_ptr curr_packet, next_packet;
  int inconsistence_p;
  packet_ptr curr_frontier_packet;
  int ready_p, copy_ready_p;

  if (move_cost (op, to, &inconsistence_p) <= 0 || !inconsistence_p)
    abort ();
  copied_packets_4_move_consistence = NULL;
  for (curr_packet = covered_list;
       curr_packet != NULL;
       curr_packet = next_packet)
    {
      next_packet = curr_packet->next_packet;
      if (curr_packet->visit_number3 == last_packet_visit_number3)
	{
	  if (curr_packet->visit_number2 != last_packet_visit_number2)
	    abort ();
	  curr_packet->copy = packet_copy (curr_packet);
	  curr_packet->next_packet = copied_packets_4_move_consistence;
	  copied_packets_4_move_consistence = curr_packet;
	}
    }
  /* Modify CFG.  */
  for (curr_packet = copied_packets_4_move_consistence;
       curr_packet != NULL;
       curr_packet = curr_packet->next_packet)
    modify_packet_CFG (curr_packet);
  /* Modify DDG.  */
  for (curr_packet = copied_packets_4_move_consistence;
       curr_packet != NULL;
       curr_packet = curr_packet->next_packet)
    copy_dependencies_4_consistent_move (curr_packet);
  dep_sort_added_in_deps_op ();
  for (curr_packet = copied_packets_4_move_consistence;
       curr_packet != NULL;
       curr_packet = curr_packet->next_packet)
    remove_obsolete_dependencies_4_move_consistence (curr_packet);
#if 1
  /* ??? Update ready ops.  After copying packets, copied ops are also
     ready ops if their originals are.  Some of ops (copied ones and
     originals) may be not really ready because of CFG changes.  Is it
     worth to remove them from ready ops.  To remain them is in ready
     ops is safe but it requires more time to evaluate available ops
     for packets.  */
  for (curr_packet = copied_packets_4_move_consistence;
       curr_packet != NULL;
       curr_packet = curr_packet->next_packet)
    if (curr_packet->first_op->ready_p)
      {
	if (!curr_packet->copy->first_op->ready_p)
	  abort ();
	copy_ready_p
	  = get_op_ready_time (curr_packet->copy->first_op, to) >= 0;
	ready_p = get_op_ready_time (curr_packet->first_op, to) >= 0;
	for (curr_frontier_packet = first_frontier_packet;
	     curr_frontier_packet != NULL;
	     curr_frontier_packet = curr_frontier_packet->next_frontier_packet)
	  {
	    if (!ready_p && get_op_ready_time (curr_packet->first_op,
					       curr_frontier_packet) >= 0)
	      ready_p = TRUE;
	    if (!copy_ready_p
		&& get_op_ready_time (curr_packet->copy->first_op,
				      curr_frontier_packet) >= 0)
	      copy_ready_p = TRUE;
	    if (copy_ready_p && ready_p)
	      break;
	  }
	if (!ready_p)
	  op_become_not_ready (curr_packet->first_op);
	if (!copy_ready_p)
	  op_become_not_ready (curr_packet->copy->first_op);
      }
#endif
}



/* This page contains the rest of software pipeliner function.  */

/* The following function adds new dependencies after moving OP on LOC
   used (if USAGE_P) or defined in OP when we are processing operation
   FROM on which OP is depended.  */

static void
update_loc_deps (op_ptr op, op_ptr from, loc_ptr loc, int usage_p)
{
  dep_ptr curr_out_dep;
  dep_mode_t dep_mode;

  for (curr_out_dep = dep_out_find (from, loc);
       curr_out_dep != NULL;
       curr_out_dep = curr_out_dep->next_out_loc_dep)
    if ((!usage_p || curr_out_dep->dep_mode != DEF_USE)
	&& op != curr_out_dep->to
	/* Ignore prev ops in OP packet.  */
	&& op->packet != curr_out_dep->to->packet
	&& ACHIEVABLE (op->packet, curr_out_dep->to->packet) != UNACHIEVABLE)
      {
	/* DEF_USE is not possible here because we could kill a live
	   location value.  It is guaranteed by function
	   `kill_move_p'.  */
	dep_mode = (usage_p ? USE_DEF : DEF_DEF);
	dep_create (op, curr_out_dep->to, curr_out_dep->to_loc, dep_mode,
		    insn_dep_time (op->insn, curr_out_dep->to->insn,
				   dep_mode));
      }
}

/* The following function updates DDG after OP has been moved.  */

static void
update_deps (op_ptr op)
{
  dep_ptr curr_in_dep;
  loc_ptr curr_loc;
  int i;

  for (curr_in_dep = op->in_deps;
       curr_in_dep != NULL;
       curr_in_dep = curr_in_dep->next_in)
    {
      update_loc_deps (op, curr_in_dep->from, curr_in_dep->to_loc,
                       curr_in_dep->dep_mode == DEF_USE);
      for (i = 0; ; i++)
        {
          curr_loc = loc_super (curr_in_dep->to_loc, i == 0);
          if (curr_loc == NULL)
            break;
          update_loc_deps (op, curr_in_dep->from, curr_loc,
                           curr_in_dep->dep_mode == DEF_USE);
        }

      for (i = 0; ; i++)
	{
	  curr_loc = loc_sub (curr_in_dep->to_loc, i == 0);
	  if (curr_loc == NULL)
	    break;
	  update_loc_deps (op, curr_in_dep->from,
			   curr_in_dep->to_loc,
			   curr_in_dep->dep_mode == DEF_USE);
	}
    }
}

/* The following function creates new packet with unconditional jump
   between packets FROM and TO.  The function returns created
   operation.  */

static op_ptr
create_simple_jump (packet_ptr from, packet_ptr to, op_ptr initiator_op)
{
  packet_ptr new_packet;
  op_ptr new_op;
  insn_ptr new_insn;

  new_packet = packet_create (-1);
  new_packet->exit_packet_p = from->exit_packet_p;
  new_packet->formed_packet_p = to->formed_packet_p;
  new_insn = create_uncond_branch (to->first_op->insn);
  new_op = op_create (new_insn, to->first_op->it_num);
  new_op->initiator_op = initiator_op;
  packet_append_op (new_packet, new_op, FALSE);
  packet_change_dest (from, to, new_packet);
  packet_connect (new_packet, to, TRUE, TRUE);
  return new_op;
}

/* The following function creates new avail list element of PACKET
   with given characteristics (OP, AVAIL_P, COST, INCONSISTENCE_P,
   READY_TIME).  */

static void
create_avail_list_el (packet_ptr packet, op_ptr op, int avail_p,
		      int cost, int inconsistence_p, int ready_time)
{
  avail_list_ptr new_el;

  if (op->packet->visit_number4 == last_packet_visit_number4)
    /* It is already in available ops list.  */
    return;
  op->packet->visit_number4 = last_packet_visit_number4;
  new_el = avail_list_el_create (packet, op, avail_p);
  new_el->cost = cost;
  new_el->inconsistence_p = inconsistence_p;
  new_el->delay = ready_time - curr_time;
  if (new_el->delay < 0)
    new_el->delay = 0;
  new_el->next_avail = packet->avail_list;
  if (packet->avail_list != NULL)
    packet->avail_list->prev_avail = new_el;
  packet->avail_list = new_el;
  if (!op->packet->exit_packet_p && op->it_num < packet->min_it_n && avail_p)
    /* It is not possible situation for available operations.  */
    abort ();
}

/* The following function returns TRUE if we need a unconditional jump
   insn between PACKET and its successor SUCC.  */

static int
needs_jump_p (packet_ptr packet, packet_ptr succ)
{
  packet_ptr pred;
  int pred_number;
  op_ptr branch;

  if (packet_find_branch (packet, succ, TRUE) != NULL)
    return dest_packet_requires_jump_p (succ);
  for (pred_number = 0;
       (pred = succ->pred [pred_number]) != NULL;
       pred_number++)
    if (pred != packet)
      {
	branch = packet_find_branch (pred, succ, FALSE);
	if (branch == NULL
	    || (cond_branch_p (branch->insn)
		&& ((branch->true_jump_p && branch->false_target == succ)
		    || (!branch->true_jump_p && branch->true_target == succ))
		&& branch->target_fixed_p))
	  break;
      }
  return pred != NULL || dest_packet_requires_jump_p (succ);
}

/* The following function processes all new conditional branches in
   PACKET and adds necessary jumps or converts them for CFG
   consistence.  */

static void
add_consistent_jumps (packet_ptr packet)
{
  op_ptr curr_op, branch, new_op;
  packet_ptr next_packet, jump_packet, pred_packet, pred_packet1;
  packet_ptr pred;
  int pred_number;
  int require_jump_p, can_be_converted_p;
  avail_list_ptr new_el ATTRIBUTE_UNUSED;

  for (curr_op = packet->first_op; curr_op != NULL; curr_op = curr_op->next_op)
    if (!insn_dummy_p (curr_op->insn) && cond_branch_p (curr_op->insn)
	&& !curr_op->processed_4_consistent_jumps_p)
      {
	curr_op->processed_4_consistent_jumps_p = TRUE;
	pred_packet = packet;
	if (curr_op->true_jump_p)
	  {
	    next_packet = curr_op->false_target;
	    jump_packet = curr_op->true_target;
	  }
	else
	  {
	    next_packet = curr_op->true_target;
	    jump_packet = curr_op->false_target;
	  }
	while (next_packet != NULL)
	  {
	    if (next_packet->first_op != NULL
		&& !insn_dummy_p (next_packet->first_op->insn))
	      {
		if (uncond_branch_p (next_packet->first_op->insn))
		  break;
	      }
 	    require_jump_p = needs_jump_p (pred_packet, next_packet);
 	    if (require_jump_p)
	      {
		can_be_converted_p = FALSE;
		if (!require_jump_p && pred_packet == packet
		    && !curr_op->target_fixed_p && curr_op->can_be_converted_p)
		  {
		    can_be_converted_p = TRUE;
		    pred_packet1 = packet;
		    /* Look at the jump path.  */
		    while (jump_packet != NULL)
		      {
			if (jump_packet->first_op != NULL
			    && !insn_dummy_p (jump_packet->first_op->insn))
			  {
			    if (uncond_branch_p (jump_packet->first_op->insn))
			      break;
			    if (jump_packet->first_op->next_op != NULL)
			      abort ();
			  }
			for (pred_number = 0;
			     (pred = jump_packet->pred [pred_number]) != NULL;
			     pred_number++)
			  if (pred != pred_packet1)
			    {
			      branch = packet_find_branch (pred, jump_packet,
							   FALSE);
			      if (branch == NULL
				  ||
				  (cond_branch_p (branch->insn)
				   && ((branch->true_jump_p
					&& branch->false_target == jump_packet)
				       ||
				       (!branch->true_jump_p
					&& branch->true_target == jump_packet))
				   && branch->target_fixed_p))
				break;
			    }
			if (pred != NULL)
			  {
			    /* There is another fall into JUMP_PACKET
                               and this path has been already
                               fixed.  */
			    can_be_converted_p = FALSE;
			    break;
			  }
			pred_packet1 = jump_packet;
			if (jump_packet->succ [0] != NULL
			    && jump_packet->succ [1] != NULL)
			  /* We already processed the branch in JUMP_PACKET
                             or will look at it later.  */
			  break;
			jump_packet = jump_packet->succ [0];
		      }
		  }
		if (can_be_converted_p)
		  {
		    /* convert curr_op */
		    curr_op->true_jump_p = !curr_op->true_jump_p;
		    convert_cond_branch (curr_op->insn);
		  }
		else
		  {
		    /* Insert jump.  */
		    new_op = create_simple_jump (pred_packet, next_packet,
						 curr_op);
#if 0
		    {
		      int cost, inconsistence_p;

		      /* ??? The following code will be needed if we
			 add jumps during scheduling itself (see
			 comments for add_consistent_jumps and
			 convert_cond_branch).  */
		      /* We require that the unconditional jump does
			 not create new data dependencies.  */
		      /* Add the jump into ready op list and available
			 ops list of PACKET.  */
		      op_become_ready (new_op);
		      cost = move_cost (new_op, packet, &inconsistence_p);
		      if (cost < 0)
			abort (0);
		      create_avail_list_el (packet, new_op, TRUE,
					    cost, inconsistence_p, curr_time);
		    }
#endif
		  }
		/* We inserted jump or converted the branch.  So we
                   can start processing of the next operation of
                   PACKET.  */
		break;
	      }
	    pred_packet = next_packet;
	    if (next_packet->succ [0] != NULL && next_packet->succ [1] != NULL)
	      /* We already processed the branch in NEXT_PACKET or
                 will look at it later.  */
	      break;
	    next_packet = next_packet->succ [0];
	  }
	curr_op->target_fixed_p = TRUE;
      }
}

/* The following function updates ready ops list after scheduling OP
   and available ops list for packet of OP if AVAIL_P is TRUE.  */

static void
update_ready_avail_ops (op_ptr op, int avail_p)
{
  avail_list_ptr next_el, curr_el;
  dep_ptr curr_dep;
  op_ptr candidate;
  int ready_time, cost, inconsistence_p;
  packet_ptr op_packet = op->packet;

  if (avail_p)
    {
      /* Modify existing list.  Move of OP may transform given
         operation from one requiring making CFG move consistent into
         one not requiring making CFG move consistent.  Consequently
         the cost may also be changed.  Speculated move of OP may make
         available operation unavailable because the operation will
         kill a location of the moved OP.  */
      for (curr_el = op_packet->avail_list; curr_el != NULL; curr_el = next_el)
	{
	  next_el = curr_el->next_avail;
	  cost = move_cost (curr_el->op, op_packet, &inconsistence_p);
	  if (cost < 0)
	    {
	      /* Remove curr_el from available list.  */
	      if (curr_el->prev_avail == NULL)
		op_packet->avail_list = next_el;
	      else
		curr_el->prev_avail->next_avail = next_el;
	      if (next_el != NULL)
		next_el->prev_avail = curr_el->prev_avail;
	      avail_list_el_free (curr_el);
	    }
	  else
	    {
	      curr_el->cost = cost;
	      curr_el->inconsistence_p = inconsistence_p;
	    }
	}
    }
  for (curr_dep = op->out_deps;
       curr_dep != NULL;
       curr_dep = curr_dep->next_out)
    {
      candidate = curr_dep->to;
      ready_time = -1;
      if (avail_p
	  && candidate->it_num < op_packet->min_it_n + iteration_look_ahead)
	{
	  ready_time = get_op_ready_time (candidate, op_packet);
	  if (ready_time < 0)
	    continue;
	  cost = move_cost (candidate, op_packet, &inconsistence_p);
	  if (cost >= 0)
	    create_avail_list_el (op_packet, candidate, TRUE,
				  cost, inconsistence_p, ready_time);
	}
      if (!candidate->ready_p)
	{
	  if (ready_time < 0)
	    {
	      ready_time = get_op_ready_time (candidate, op_packet);
	      if (ready_time < 0)
		continue;
	    }
	  op_become_ready (candidate);
	}
    }
}

/* The following function adds OP of an interior packet to frontier
   PACKET and updates CFG and DDG.  */

static void
schedule_op (op_ptr op, packet_ptr packet)
{
  packet_ptr op_packet = op->packet;
  int basic_block_move_p;
  op_ptr curr_op;
  packet_ptr true_target, false_target, curr_packet, succ, pred;
  int succ_number, pred_number;

  if (op == NULL || op->next_op != NULL)
    abort ();
  basic_block_move_p = ACHIEVABLE (packet, op_packet) == BASIC_BLOCK_PATH;
  if (packet->succ[0] != NULL && packet->succ[1] != NULL
      && cond_branch_p (packet->last_op->insn))
    {
      curr_op = packet->last_op;
#if 0
      /* This code may be present when we add jumps during scheduling.  */
      if (!curr_op->target_fixed_p)
	/* It should be fixed in add_consistent_jumps.  */
	abort ();
#endif
      if (curr_op->true_jump_p)
	curr_op->false_target = NULL;
      else
        curr_op->true_target = NULL;
    }
  if (nonbranch_p (op->insn) || uncond_branch_p (op->insn))
    {
      if (*op_packet->succ == NULL || op_packet->succ[1] != NULL)
        abort ();
      packet_append_op (packet, op, FALSE);
      op_packet->first_op = op_packet->last_op = NULL;
      /* Remove the packet from CFG.  */
      succ = *op_packet->succ;
      for (pred_number = 0;
	   (pred = op_packet->pred [pred_number]) != NULL;
	   pred_number++)
	packet_change_src (succ, op_packet, pred);
      while ((pred = op_packet->pred[0]) != NULL)
	/* Remember: packet_change_dest removes the link from pred by
	   shifting array `pred'.  */
	packet_change_dest (pred, op_packet, succ);
    }
  else if (cond_branch_p (op->insn))
    {
      true_target = op->true_target;
      false_target = op->false_target;
      if (true_target == NULL || false_target == NULL
	  || *op_packet->succ == NULL || op_packet->succ [1] == NULL
	  || op_packet->succ [2] != NULL)
	abort ();
      if (packet->succ[0] != NULL && packet->succ[1] != NULL)
	/* ??? Now we place only one branch into packet.  See comments
           in get_best_candidate_avail_op.  If we permit more one
           branch, the following code should be changed.  */
	abort ();
      /* Create copies of packets in "covered" except for the root and
         the branch.  */
      mark_covered (op, packet);
      for (curr_packet = covered_list;
	   curr_packet != NULL;
	   curr_packet = curr_packet->next_packet)
        if (curr_packet != op_packet && curr_packet != packet)
	  {
	    if (curr_packet->succ[1] != NULL
		&& (curr_packet->succ[0] == op_packet
		    || curr_packet->succ[1] == op_packet)
		&& (curr_packet->succ[0] == true_target
		    || curr_packet->succ[0] == false_target
		    || curr_packet->succ[1] == true_target
		    || curr_packet->succ[1] == false_target))
	      /* We move cond branch through the 2nd cond branch and
                 2nd branch will refer only for one packet.  */
	      abort ();
	    curr_packet->copy = packet_copy (curr_packet);
	    if (true_target->exit_packet_p)
	      curr_packet->exit_packet_p = TRUE;
	    if (false_target->exit_packet_p)
	      curr_packet->copy->exit_packet_p = TRUE;
	  }

      /* Form the second control flow path from packet into the branch
         packet.  */
      for (curr_packet = covered_list;
	   curr_packet != NULL;
	   curr_packet = curr_packet->next_packet)
        if (curr_packet != op_packet && curr_packet != packet)
          {
	    for (succ_number = 0;
		 (succ = curr_packet->succ [succ_number]) != NULL;
		 succ_number++)
	      if (succ != op_packet)
		{
		  if (succ->visit_number2 == last_packet_visit_number2)
		    {
		      /* The successor belongs to "covered" */
		      packet_change_dest (curr_packet->copy, succ, succ->copy);
		      packet_change_src (succ->copy, curr_packet,
					 curr_packet->copy);
		    }
		  else
		    packet_connect (curr_packet->copy, succ, FALSE, FALSE);
		}
	    
	    for (pred_number = 0;
		 (pred = curr_packet->pred [pred_number]) != NULL;
		 pred_number++)
	      if (pred->visit_number2 != last_packet_visit_number2)
		/* The predecessor does not belong to "covered" --
                   side path.  Make the side path entering only into
                   original packets.  */
		packet_change_dest (pred, curr_packet->copy, curr_packet);
	  }
      
      /* Moving op from OP_PACKET into PACKET.  */
      packet_append_op (packet, op, FALSE);
      op_packet->first_op = op_packet->last_op = NULL;
      /* Changes src of {false, true} targets and dest of the
         packet.  */
      succ = packet->succ[0];
      if (succ == op_packet)
	{
	  packet_change_src (true_target, op_packet, packet);
	  packet_change_src (false_target, op_packet, packet);
	  packet_change_dest (packet, op_packet, true_target);
	}
      else
	{
	  if (succ->visit_number2 != last_packet_visit_number2)
	    abort ();
	  packet_change_dest (packet, true_target, succ);
	  packet_change_dest (packet, false_target, succ->copy);
	  while ((pred = op_packet->pred[0]) != NULL)
	    {
	      if (pred->visit_number2 != last_packet_visit_number2
		  || pred == packet)
		abort ();
	      packet_change_dest (pred->copy, op_packet, false_target);
	      packet_change_src (false_target, op_packet, pred->copy);
	      packet_change_dest (pred, op_packet, true_target);
	      packet_change_src (true_target, op_packet, pred);
	    }
	}

      /* Create dependencies for copied packets.  */
      for (curr_packet = covered_list;
	   curr_packet != NULL;
	   curr_packet = curr_packet->next_packet)
        if (curr_packet != op_packet && curr_packet != packet)
	  {
	    dep_ptr in_deps, out_deps, curr_dep, next_dep;

	    in_deps = curr_packet->first_op->in_deps;
	    out_deps = curr_packet->first_op->out_deps;
	    for (curr_dep = in_deps;
		 curr_dep != NULL;
		 curr_dep = curr_dep->next_in)
	      if (curr_dep->from->packet->visit_number2
		  == last_packet_visit_number2
		  && curr_dep->from->packet != packet)
		dep_create (curr_dep->from->packet->copy->first_op,
			    curr_packet->copy->first_op, curr_dep->to_loc,
			    curr_dep->dep_mode, curr_dep->latency);
	      else if (ACHIEVABLE (curr_dep->from->packet,
				   curr_packet->copy) != UNACHIEVABLE)
		dep_create (curr_dep->from, curr_packet->copy->first_op,
			    curr_dep->to_loc, 
			    curr_dep->dep_mode, curr_dep->latency);	      
	    for (curr_dep = out_deps; curr_dep != NULL; curr_dep = next_dep)
	      {
		next_dep = curr_dep->next_out;
		if (curr_dep->to->packet->visit_number2
		    != last_packet_visit_number2)
		  {
		    if (ACHIEVABLE (curr_packet, curr_dep->to->packet)
			== UNACHIEVABLE)
		      dep_delete (curr_dep);
		    if (ACHIEVABLE (curr_packet->copy,
				    curr_dep->to->packet) != UNACHIEVABLE)
		      dep_create (curr_packet->copy->first_op, curr_dep->to,
				  curr_dep->to_loc,
				  curr_dep->dep_mode, curr_dep->latency);
		  }

	      }
	  }
    }
  else
    abort ();
  if (!basic_block_move_p)
    update_deps (op);
  dep_sort_added_in_deps_op ();
  packet_delete (op_packet);
}

/* The following function is used as comparison function to sort
   available list to choose the best operation for scheduling.  */

static int
avail_op_cmp (avail_list_ptr * avail_el1, avail_list_ptr * avail_el2)
{
  avail_list_ptr av1 = *avail_el1;
  avail_list_ptr av2 = *avail_el2;
  
  if (av1->op == av2->op)
    /* No duplications.  */
    abort ();
  if (av1->avail_p && !av2->avail_p)
    return -1;
  else if (!av1->avail_p && av2->avail_p)
    return 1;
  else if (!av1->op->packet->exit_packet_p && av2->op->packet->exit_packet_p)
    return -1;
  else if (av1->op->packet->exit_packet_p && !av2->op->packet->exit_packet_p)
    return 1;
  else if (av1->delay < av2->delay)
    return -1;
  else if (av1->delay > av2->delay)
    return 1;
  else if (av1->cost < av2->cost)
    return -1;
  else if (av1->cost > av2->cost)
    return 1;
  else if (av1->op->it_num < av2->op->it_num)
    return -1;
  else if (av1->op->it_num > av2->op->it_num)
    return 1;
  /* We are trying branches as the last choice for scheduling.  */
  else if (!insn_dummy_p (av2->op->insn)
	   && (uncond_branch_p (av2->op->insn)
	       || cond_branch_p (av2->op->insn))
           && (insn_dummy_p (av1->op->insn)
	       || (!uncond_branch_p (av1->op->insn)
		   && !cond_branch_p (av1->op->insn))))
    return -1;
  else if (!insn_dummy_p (av1->op->insn)
	   && (uncond_branch_p (av1->op->insn)
	       || cond_branch_p (av1->op->insn))
           && (insn_dummy_p (av2->op->insn)
	       || (!uncond_branch_p (av2->op->insn)
		   && !cond_branch_p (av2->op->insn))))
    return 1;
  else
    /* ??? Additional ordering.  May be moving in basic block or
       speculative moving.  */
    return 0;
}

/* The following function creates empty packets for all successors and
   place them between PACKET and the successors.  The function also
   defines available lists for the packets and adds the packets to the
   list of frontier packets.  */

static void
add_next_packets (packet_ptr packet)
{
  packet_ptr *target_ptr, r, old_to;

  if (packet->first_op == NULL)
    {
      /* It is empty packet.  We should merge it lately (with frontier
         packets which have the same successors.  */
      packet->formed_packet_p = FALSE;
      add_frontier_packet (packet);
      return;
    }
  for (target_ptr = packet->succ; *target_ptr != NULL; target_ptr++)
    if (!(*target_ptr)->stop_packet_p)
      {
	r = packet_create (curr_time);
	memcpy (r->cpu_state, packet->cpu_state, cpu_state_length);
	old_to = *target_ptr;
	r->exit_packet_p = old_to->exit_packet_p;
	packet_change_dest (packet, old_to, r);
	packet_connect (r, old_to, FALSE, FALSE);
      }
  for (target_ptr = packet->succ; *target_ptr != NULL; target_ptr++)
    if (!(*target_ptr)->stop_packet_p)
      add_frontier_packet (*target_ptr);
}

/* The function chooses PACKET available op list element with the best
   op for scheduling.  *NO_MORE_CANDIDATE_OP_P == TRUE means that the
   next call we will not find an available op.  */

static avail_list_ptr
get_best_candidate_avail_op (packet_ptr packet, int * no_more_candidate_op_p)
{
  avail_list_ptr pred_el, curr_el;
  packet_ptr curr_packet ATTRIBUTE_UNUSED;
  int pred_number;
  packet_ptr pred;

  *no_more_candidate_op_p = TRUE;
  for (pred_el = NULL, curr_el = packet->avail_list;
       curr_el != NULL;
       pred_el = curr_el, curr_el = curr_el->next_avail)
    {
      if (!curr_el->avail_p)
	/* Current packet available list should contain only available ops.  */
	abort ();
      if (!packet->exit_packet_p && curr_el->op->packet->exit_packet_p)
	/* We never move op from exit packet into the loop body.  */
	continue;
#if 0
      /* This is restriction on choosing op for scheduling.  Such code
         would generate more compact pipelined loops but less
         effective.  ??? How less effective.  We need to examine
         this.  */
      for (curr_packet = first_frontier_packet;
	   curr_packet != NULL;
	   curr_packet = curr_packet->next_frontier_packet)
	if (ACHIEVABLE (curr_packet, curr_el->op->packet) != UNACHIEVABLE)
	  break;
      if (curr_packet != NULL)
	/* Given op belongs more one frontier packet -- ignore it.  */
	continue;
#endif
      
      if ((insn_risky_speculative_move_p (curr_el->op->insn, packet)
	   && !postdom_p (curr_el->op->packet, packet))
	  /* Don't make such move of branch.  Sometimes it results in
             quick increase of insn copy number.  But it is ignored
             for an exit packet because such packets are achieved from
             different iterations and dominator constraint is too
             strict.  */
	  || (!packet->exit_packet_p
	      && (uncond_branch_p (curr_el->op->insn)
		  || cond_branch_p (curr_el->op->insn))
	      && !dom_p (packet, curr_el->op->packet)))
	continue;
      if (curr_el->delay > 0)
	{
	  *no_more_candidate_op_p = FALSE;
	  return NULL;
	}
      memcpy (temp_cpu_state, packet->cpu_state, cpu_state_length);
      if (!cpu_state_issue (curr_el->op->insn, temp_cpu_state))
	*no_more_candidate_op_p = FALSE;
      else
	{
	  int op_can_be_moved_p = TRUE;
	  op_ptr curr_op;
	  
	  if (!cond_branch_p (curr_el->op->insn)
	      && !uncond_branch_p (curr_el->op->insn))
	    for (curr_op = packet->first_op;
		 curr_op != NULL;
		 curr_op = curr_op->next_op)
	      /* ??? Currently gcc (gcc rtl) does not support muti-way
                 branch architectures.  */
	      if (uncond_branch_p (curr_op->insn)
		  || cond_branch_p (curr_op->insn))
		{
		  op_can_be_moved_p = FALSE;
		  break;
		}

	  if (cond_branch_p (curr_el->op->insn)
	      /*|| uncond_branch_p (curr_el->op->insn)*/)
	    for (curr_op = packet->first_op;
		 curr_op != NULL;
		 curr_op = curr_op->next_op)
	      /* ??? Currently we prohibit more one branch in a
		 packet.  Some processors can issue more one branch
		 per cycle.  So we could implement this because we
		 have some infrastructure for this see members and
		 comments for struct op.  The implementation should be
		 depended on semantics of such case (the first branch
		 is taken or the last one).  */
	      if (uncond_branch_p (curr_op->insn)
		  || cond_branch_p (curr_op->insn))
		{
		  op_can_be_moved_p = FALSE;
		  break;
		}
	  if (op_can_be_moved_p && uncond_branch_p (curr_el->op->insn)
	      && curr_el->op->packet->succ[0]->pred [1] != NULL
	      && curr_el->op->packet->pred [0] != packet)
	    op_can_be_moved_p = FALSE;
	  if (op_can_be_moved_p)
	    for (pred_number = 0;
		 (pred = curr_el->op->packet->pred [pred_number]) != NULL;
		 pred_number++)
	      if (pred->succ[1] != NULL
		  && (pred->succ[0] == curr_el->op->packet->succ [0]
		      || pred->succ[1] == curr_el->op->packet->succ [0]
		      || (cond_branch_p (curr_el->op->insn)
			  &&
			  (pred->succ[0] == curr_el->op->packet->succ [1]
			   ||
			   pred->succ[1] == curr_el->op->packet->succ [1]))))
		{
		  /* We move an op through the cond branch and the
		     cond branch will refer only for one packet.
		     ??? We could remove the cond branch if it has no
		     side effects.  */
		  op_can_be_moved_p = FALSE;
		  break;
		}
	  if (op_can_be_moved_p)
	    {
	      *no_more_candidate_op_p = FALSE;
	      return curr_el;
	    }
	}
    }
  return NULL;
}

/* The following function makes available ops list from the ready ops
   list.  */

static void
make_packet_avail_list (packet_ptr packet)
{
  int it_num, ready_time, cost;
  int inconsistence_p;
  op_ptr curr_op;

  /* Remove the current available ops list if it exists.  */
  avail_list_free (packet->avail_list);
  packet->avail_list = NULL;
  for (it_num = min_ready_op_it_num;
       it_num < ready_ops_size
	 && (packet->min_it_n < 0
	     || it_num < packet->min_it_n + iteration_look_ahead);
       it_num++)
    for (curr_op = ready_ops [it_num];
	 curr_op != NULL;
	 curr_op = curr_op->next_ready_op)
      {
	ready_time = get_op_ready_time (curr_op, packet);
	if (ready_time < 0)
	  continue;
	cost = move_cost (curr_op, packet, &inconsistence_p);
	if (cost < 0)
	  continue;
	if (!curr_op->packet->exit_packet_p
	    && (packet->min_it_n < 0 || it_num < packet->min_it_n))
	  packet->min_it_n = it_num;
	create_avail_list_el (packet, curr_op, TRUE,
			      cost, inconsistence_p, ready_time);
      }
}

#ifndef WRONG_ORIGINAL_RCSP

/* This recursive function processes packets (CURR_PACKET) achievable
   from PACKET and adds the packet ops to the available list of
   PACKET.  */

static void
add_achievable_ops (packet_ptr packet, packet_ptr curr_packet)
{
  op_ptr op = curr_packet->first_op;
  packet_ptr succ;
  int succ_number;

  if (curr_packet->formed_packet_p || curr_packet->frontier_packet_p
      || packet->min_it_n < 0)
    abort ();
  if (op == NULL || op->next_op != NULL)
    abort ();
  if (op->it_num >= packet->min_it_n + iteration_look_ahead)
    return;
  create_avail_list_el (packet, op, FALSE, 0, 0, curr_time);
  for (succ_number = 0;
       (succ = curr_packet->succ [succ_number]) != NULL;
       succ_number++)
    add_achievable_ops (packet, succ);
}
#endif /* #ifndef WRONG_ORIGINAL_RCSP */

/* The following function outputs all ready ops into file F with
   output line PREFIX.  */

static void
print_ready_ops (FILE * f, const char * prefix)
{
  int it_num;
  op_ptr curr_op;
  
  for (it_num = min_ready_op_it_num;
       it_num < ready_ops_size;
       it_num++)
    {
      if (ready_ops [it_num] != NULL)
	fprintf (f, "\n%s    ready list (iter = %3d):", prefix, it_num);
      for (curr_op = ready_ops [it_num];
	   curr_op != NULL;
	   curr_op = curr_op->next_ready_op)
	fprintf (f, "\n%s       packet = %3d, op %s",
		 prefix, curr_op->packet->number,
		 insn_name (curr_op->insn, FALSE));
    }
}

/* The following function outputs AVAIL_LIST into file F with output
   line PREFIX.  IF FULL_P is TRUE, the output is believed to contain
   also achievable ops.  */

static void
print_avail_list (FILE * f, avail_list_ptr avail_list, const char * prefix,
		  int full_p)
{
  avail_list_ptr curr_el;
  
  fprintf (f, (full_p ? "\n%s    available & achievable list:"
	       : "\n%s    available list:"), prefix);
  for (curr_el = avail_list; curr_el != NULL; curr_el = curr_el->next_avail)
    {
      fprintf (f, "\n%s       ", prefix);
      if (curr_el->avail_p)
	fprintf (f, "delay = %3d, cost = %3d (%s) ",
		 curr_el->delay, curr_el->cost,
		 (curr_el->inconsistence_p ? "I" : "C"));
      fprintf (f, "packet = %3d, iter = %3d, %sop %s",
	       curr_el->op->packet->number, curr_el->op->it_num,
	       (curr_el->avail_p ? "" : "achievable "),
	       insn_name (curr_el->op->insn, FALSE));
    }
}

/* The following function fills up PACKET by operations from available
   list when functional units reservation is described by
   packet->cpu_state.  The function updates the available list and cpu
   state for PACKET.  F is output file for debugging information.
   PREFIX is prefix of each new line in debugging output.  */

static void
schedule_packet (packet_ptr packet, FILE * f, const char * prefix)
{
  avail_list_ptr el;
  int added_p = FALSE, no_more_op;
  avail_list_ptr curr;
  int succ_number;
  packet_ptr succ;
  int inconsistence_p;

#ifdef RCSP_DEBUG
  if (rcsp_debug_level >= 2)
    fprintf (f, "\n%s\n%s scheduling packet %3d:",
	     prefix, prefix, packet->number);
  if (rcsp_debug_level >= 3)
    print_ready_ops (f, prefix);
#endif
  last_packet_visit_number4++;
  make_packet_avail_list (packet);
  for (;;)
    {
      packet->avail_list
	= avail_list_sort (packet->avail_list, avail_op_cmp);
#ifdef RCSP_DEBUG
      if (rcsp_debug_level >= 3)
	print_avail_list (f, packet->avail_list, prefix, FALSE);
#endif
      el = get_best_candidate_avail_op (packet, &no_more_op);
      if (el == NULL)
	{
	  if (no_more_op && !added_p)
	    {
	      if (first_frontier_packet == NULL)
		abort ();
	      /* We cannot place any op into the packet in this case.  */
	      break;
	    }
	  /* Advance time.  */
	  for (curr = packet->avail_list;
	       curr != NULL;
	       curr = curr->next_avail)
	    if (curr->delay != 0)
	      curr->delay--;
	  curr_time++;
	  cpu_state_advance_cycle (packet->cpu_state);
#ifdef RCSP_DEBUG
	  if (rcsp_debug_level >= 3)
	    fprintf (f, "\n%s   advancing time (time = %3d)",
		     prefix, curr_time);
#endif
	  if (added_p)
	    break;
	}
      else
	{
	  op_ptr op = el->op;

	  packet->issue_time = curr_time;
	  if (el->inconsistence_p)
	    {
	      make_move_consistent (op, packet);
	      if (move_cost (op, packet, &inconsistence_p) < 0
		  || inconsistence_p)
		/* After making CFG move consistent PACKET should be
                   able to be moved and move consistent */
		abort ();
	    }
	  schedule_op (op, packet);
	  if (packet->min_it_n < 0 || op->it_num < packet->min_it_n)
	    packet->min_it_n = op->it_num;
#if 0
	  /* ??? This unfinished code is for adding jumps during
             scheduling itself (see comments for add_consistent_jumps
             and convert_cond_branch).  */
	  if (cond_branch_p (op->insn))
	    /* We are trying to make it here too because a new jump
	       may be added into avail list and consequently into this
	       packet too.  */
	    add_consistent_jumps (packet);
#endif
	  op_become_scheduled (op);
	  /* Remove el from available list.  */
	  if (el->prev_avail == NULL)
	    packet->avail_list = el->next_avail;
	  else
	    el->prev_avail->next_avail = el->next_avail;
	  if (el->next_avail != NULL)
	    el->next_avail->prev_avail = el->prev_avail;
	  avail_list_el_free (el);
	  update_ready_avail_ops (op, TRUE);
	  if (!cpu_state_issue (op->insn, packet->cpu_state))
	    abort ();
#ifdef RCSP_DEBUG
	  if (rcsp_debug_level >= 3)
	    fprintf (f, "\n%s   move op (iter %3d, time = %3d) %s",
		     prefix, op->it_num, curr_time,
		     insn_name (op->insn, FALSE));
#endif
	  added_p = TRUE;
	}
    }
  packet->formed_packet_p = TRUE;
  if (packet->min_it_n < 0)
    abort ();
#ifndef WRONG_ORIGINAL_RCSP
  for (succ_number = 0;
       (succ = packet->succ [succ_number]) != NULL;
       succ_number++)
    add_achievable_ops (packet, succ);
  packet->avail_list = avail_list_sort (packet->avail_list, avail_op_cmp);
#ifdef RCSP_DEBUG
  if (rcsp_debug_level >= 3)
    print_avail_list (f, packet->avail_list, prefix, TRUE);
#endif
#endif /* #ifndef WRONG_ORIGINAL_RCSP */
}

/* The following recursive function traverses CFG starting with PACKET
   and makes all independent ops ready.  */

static void
add_independent_ready_ops (packet_ptr packet)
{
  int succ_number;

  if (packet->visit_number1 == last_packet_visit_number1)
    return;
  packet->visit_number1 = last_packet_visit_number1;
  if (packet->first_op != NULL && packet->first_op->next_op != NULL)
    abort ();
  if (!packet->stop_packet_p
      && packet->first_op != NULL && packet->first_op->in_deps == NULL)
    op_become_ready (packet->first_op);
  for (succ_number = 0;
       packet->succ [succ_number] != NULL;
       succ_number++)
    add_independent_ready_ops (packet->succ [succ_number]);
}

/* The following recursive function traverses all achievable packets
   and marks them.  */

static void
mark_achievable_packets (packet_ptr packet)
{
  int succ_number;

  if (packet->visit_number1 == last_packet_visit_number1)
    return;
  packet->visit_number1 = last_packet_visit_number1;
  for (succ_number = 0;
       packet->succ [succ_number] != NULL;
       succ_number++)
    mark_achievable_packets (packet->succ [succ_number]);
}

/* The following recursive function traverses PACKET and its
   successors and links all unachievable packets.  It is supposed that
   packets are marked by mark_achievable_packets before the function
   call.  It is important that all successors of packet finally will
   be after the packet in the list.  */

static packet_ptr
link_unachievable_packets (packet_ptr packet, packet_ptr first_packet)
{
  int succ_number;

  /* It is achievable packet or we already linked the packet.  */
  if (packet->visit_number1 == last_packet_visit_number1 - 1
      || packet->visit_number1 == last_packet_visit_number1)
    return first_packet;
  packet->visit_number1 = last_packet_visit_number1;
  if (packet->frontier_packet_p)
    abort ();
  for (succ_number = 0;
       packet->succ [succ_number] != NULL;
       succ_number++)
    first_packet = link_unachievable_packets (packet->succ [succ_number],
					      first_packet);
  packet->next_packet = first_packet;
  first_packet = packet;
  return first_packet;
}

/* The following function redirects packet FROM onto TO.  ROOT is a
   top packet from which all other packets are achievable.  F and
   PREFIX are used for debug output.  */

static void
redirect_packet (packet_ptr root, packet_ptr from, packet_ptr to,
		 FILE * f, const char * prefix)
{
  dep_ptr curr_dep, next_dep;
  op_ptr curr_op, next_op;
  packet_ptr curr_packet, next_packet;

#ifdef RCSP_DEBUG
  if (rcsp_debug_level >= 2)
    fprintf (f, "\n%s\n%s changing packet %3d onto packet %3d:",
	     prefix, prefix, from->number, to->number);
#endif
  while (from->pred[0] != NULL)
    /* Remember: packet_change_dest removes the link from
       pred by shifting array `pred'.  */
    packet_change_dest (from->pred[0], from, to);
  last_packet_visit_number1++;
  mark_achievable_packets (root);
  last_packet_visit_number1++;
  link_unachievable_packets (from, NULL);
  for (curr_packet = from; curr_packet != NULL; curr_packet = next_packet)
    {
      next_packet = curr_packet->next_packet;
      while (curr_packet->succ[0] != NULL)
	/* Remember: packet_change_src removes the link from
	   succ by shifting array `succ'.  */
	packet_change_src (curr_packet->succ[0], curr_packet, NULL);
      /* Remove ops from packet.  Ops of packet may be in ready ops
	 list.  */
      for (curr_op = curr_packet->first_op; curr_op != NULL; curr_op = next_op)
	{
	  next_op = curr_op->next_op;
	  for (curr_dep = curr_op->out_deps;
	       curr_dep != NULL;
	       curr_dep = next_dep)
	    {
	      next_dep = curr_dep->next_out;
	      dep_delete (curr_dep);
	    }
	  for (curr_dep = curr_op->in_deps;
	       curr_dep != NULL;
	       curr_dep = next_dep)
	    {
	      next_dep = curr_dep->next_in;
	      dep_delete (curr_dep);
	    }
	  op_delete (curr_op);
	}
      curr_packet->first_op = curr_packet->last_op = NULL;
      packet_delete (curr_packet);
    }
}

/* The following recursive function traverses packets, processes
   packets containing conditional jumps, and adds unconditional jumps
   to make CFG consistent.  */

static void
add_jumps_4_cond_branches (packet_ptr packet)
{
  int succ_number;
  packet_ptr succ;

  if (packet->visit_number5 == last_packet_visit_number5)
    return;
  packet->visit_number5 = last_packet_visit_number5;
  /* We use post order traversing here to process the last op whose
     successor is a loop exit as the first op and possibly remove
     additional jump.  */
  for (succ_number = 0;
       (succ = packet->succ [succ_number]) != NULL;
       succ_number++)
    add_jumps_4_cond_branches (succ);
  add_consistent_jumps (packet);
}

/* The following recursive function finds back edges in the formed
   loop and adds unconditional jumps for them if they are needed.  */

static void
add_jumps_on_back_edges (packet_ptr packet)
{
  int succ_number;
  packet_ptr succ;

  packet->visit_number5 = last_packet_visit_number5;
  for (succ_number = 0;
       (succ = packet->succ [succ_number]) != NULL;
       succ_number++)
    if (succ->visit_number5 != last_packet_visit_number5)
      add_jumps_on_back_edges (succ);
    else if (succ->visit_number4 != last_packet_visit_number4)
      {
	/* packet->succ is a back edge.  */
	if (packet_find_branch (packet, succ, TRUE) == NULL)
	  /* Add jump on back edge.  */
	  (void) create_simple_jump (packet, succ, NULL);
      }
  packet->visit_number4 = last_packet_visit_number4;
}

/* The following recursive function traverses packets, processes
   packets adds the rest of unconditional jumps to make CFG
   consistent.  */

static void
add_the_rest_jumps (packet_ptr packet)
{
  int succ_number;
  packet_ptr succ;

  if (packet->visit_number5 == last_packet_visit_number5)
    return;
  packet->visit_number5 = last_packet_visit_number5;
  for (succ_number = 0;
       (succ = packet->succ [succ_number]) != NULL;
       succ_number++)
    {
      add_the_rest_jumps (succ);
      if (needs_jump_p (packet, succ))
	(void) create_simple_jump (packet, succ, NULL);
    }
}

/* The following recursive function may add jump onto main exit
   packet.  This jump is necessary to guarantee that the main exit
   will be the last packet during outputting packets.  It is supposed
   that for each packet the first output packet will be fall-through
   packet then all jumps which will be output in the same order as
   they are presented in array succ of PACKET.  */

static void
add_exit_jump (packet_ptr packet, int there_are_unprocessed_packets_p)
{
  packet_ptr succ, next_pc_packet;
  int succ_number, unvisited_n;

  if (packet->visit_number5 == last_packet_visit_number5)
    return;
  packet->visit_number5 = last_packet_visit_number5;
  unvisited_n = 0;
  next_pc_packet = NULL;
  for (succ_number = 0;
       (succ = packet->succ [succ_number]) != NULL;
       succ_number++)
    if (packet_find_branch (packet, succ, TRUE) == NULL)
      next_pc_packet = succ;
    else if (succ->visit_number5 != last_packet_visit_number5)
      unvisited_n++;
  if (next_pc_packet != NULL)
    {
      there_are_unprocessed_packets_p
	= there_are_unprocessed_packets_p || unvisited_n != 0;
      if (next_pc_packet->stop_packet_p && next_pc_packet->main_exit_p
	  && there_are_unprocessed_packets_p)
	{
	  (void) create_simple_jump (packet, next_pc_packet, NULL);
	  return;
	}
      add_exit_jump (next_pc_packet, there_are_unprocessed_packets_p);
    }
  for (succ_number = 0;
       (succ = packet->succ [succ_number]) != NULL;
       succ_number++)
    if (succ != next_pc_packet)
      {
	unvisited_n--;
	add_exit_jump (succ,
		       there_are_unprocessed_packets_p || unvisited_n != 0);
      }
}

/* The following function adds unconditional jumps in order to make
   code linear.  ??? If we do not implement adding and scheduling
   unconditional jumps on the fly in the future, we should simplify
   this code by removing functions add_jumps_4_cond_branches and
   add_consistent_jumps.  */

static void
add_jumps (packet_ptr packet)
{
  last_packet_visit_number5++;
  add_jumps_4_cond_branches (packet);
  last_packet_visit_number4++;
  last_packet_visit_number5++;
  add_jumps_on_back_edges (packet);
  last_packet_visit_number5++;
  add_the_rest_jumps (packet);
  last_packet_visit_number5++;
  add_exit_jump (packet, FALSE);
}

#ifdef RCSP_DEBUG

/* The following recursive function checks consistence of CFG starting
   with PACKET.  */

static void
check_CFG (packet_ptr packet)
{
  int pred_number;
  int succ_number;
  packet_ptr pred;
  packet_ptr succ;
  op_ptr curr_op;

  if (packet->visit_number5 == last_packet_visit_number5)
    return;
  packet->visit_number5 = last_packet_visit_number5;
  if (packet->formed_packet_p && packet->frontier_packet_p)
    abort ();
  /* Check consistence of pred and succ: */
  for (pred_number = 0;
       (pred = packet->pred [pred_number]) != NULL;
       pred_number++)
    if (packet_find_in_arr (pred->succ, packet) == NULL)
      abort ();
  for (succ_number = 0;
       (succ = packet->succ [succ_number]) != NULL;
       succ_number++)
    if (packet_find_in_arr (succ->pred, packet) == NULL)
      abort ();
  for (curr_op = packet->first_op; curr_op != NULL; curr_op = curr_op->next_op)
    if (curr_op->false_target != NULL
	&& packet_find_in_arr (packet->succ, curr_op->false_target) == NULL)
      abort ();
    else if (curr_op->true_target != NULL
	     && packet_find_in_arr (packet->succ,
				    curr_op->true_target) == NULL)
      abort ();
  for (succ_number = 0;
       (succ = packet->succ [succ_number]) != NULL;
       succ_number++)
    check_CFG (succ);
}

/* The following recursive function checks consistence of DDG starting
   with PACKET.  */

static void
check_DDG (packet_ptr packet)
{
  op_ptr curr_op;
  dep_ptr curr_dep;
  int succ_number;
  packet_ptr succ;

  if (packet->visit_number5 == last_packet_visit_number5)
    return;
  packet->visit_number5 = last_packet_visit_number5;
  /* Check DDG.  */
  for (curr_op = packet->first_op; curr_op != NULL; curr_op = curr_op->next_op)
    {
      for (curr_dep = curr_op->in_deps;
	   curr_dep != NULL;
	   curr_dep = curr_dep->next_in)
	if (curr_dep->to->packet != packet
	    && ACHIEVABLE (curr_dep->from->packet, packet) == UNACHIEVABLE)
	  abort ();
      for (curr_dep = curr_op->out_deps;
	   curr_dep != NULL;
	   curr_dep = curr_dep->next_out)
	if (curr_dep->from->packet != packet
	    && ACHIEVABLE (packet, curr_dep->to->packet) == UNACHIEVABLE)
	  abort ();
    }
  for (succ_number = 0;
       (succ = packet->succ [succ_number]) != NULL;
       succ_number++)
    check_DDG (succ);
}

/* The following function checks consistence of CFG and DDG starting
   with packet START.  */

static void
check_graphs (packet_ptr start)
{
  last_packet_visit_number5 ++;
  check_CFG (start);
  last_packet_visit_number5 ++;
  check_DDG (start);
}

/* The following macro value is constraint on maximal length of packet
   name.  */
#define PACKET_NAME_LONG 200

/* The following function returns symbolic name of PACKET.  If GRAPH_P
   is TRUE the name will be shorter.  The result is stored in static
   memory.  So the next call will rewrite the memory.  */

static const char *
get_packet_name (packet_ptr packet, int graph_p)
{
  static char name [PACKET_NAME_LONG];
  const char *prefix, *postfix;

  if (packet->formed_packet_p)
    prefix = (graph_p ? "" : "formed ");
  else if (packet->frontier_packet_p)
    prefix = (graph_p ? "F" : "frontier ");
  else
    prefix = (graph_p ? "I" : "");
  if (packet->stop_packet_p)
    postfix = (graph_p ? "S" : " (stop packet)");
  else if (packet->exit_packet_p)
    postfix = (graph_p ? "E" : " (exit packet)");
  else
    postfix = "";
  if (graph_p)
    sprintf (name, "\"%s%s %d:", prefix, postfix, packet->number);
  else
    sprintf (name, "%spacket %4d%s", prefix, packet->number, postfix);
  if (graph_p)
    {
      op_ptr curr_op;
      char *out = name;

      out += strlen (out);
      for (curr_op = packet->first_op;
	   curr_op != NULL;
	   curr_op = curr_op->next_op)
	{
	  out += strlen (out);
#if 1
	  if (curr_op != packet->first_op)
	    {
	      *out++ = '\\';
	      *out++ = 'n';
	    }
#endif
	  sprintf (out," %s(%d)", insn_name (curr_op->insn, TRUE),
		   curr_op->it_num);
	}
      out += strlen (out);
      strcpy (out, "\"");
    }
  if (strlen (name) >= PACKET_NAME_LONG)
    abort ();
  return name;
}

/* The following function prints information about PACKET into file F.
   PREFIX is output after each newline.  If GRAPH_P is TRUE the output
   is in a graphic vizualization program format.  */

static void
print_packet (FILE * f, packet_ptr packet, const char * prefix, int graph_p)
{
  char packet_name [PACKET_NAME_LONG];
  int pred_number, succ_number;
  packet_ptr pred, succ;
  op_ptr curr_op;
  dep_ptr curr_dep;

  strcpy (packet_name, get_packet_name (packet, graph_p));
  if (graph_p)
    fprintf (f, "\n  %s;", packet_name);
  else
    {
      fprintf (f, "\n%s%s", prefix, packet_name);
      fprintf (f, ":\n%s  packet predecessors: ", prefix);
      for (pred_number = 0;
	   (pred = packet->pred [pred_number]) != NULL;
	   pred_number++)
	fprintf (f, " %4d", pred->number);
      fprintf (f, "\n%s  packet successors: ", prefix);
    }
  for (succ_number = 0;
       (succ = packet->succ [succ_number]) != NULL;
       succ_number++)
    if (graph_p)
      fprintf (f, "\n  %s -> %s;", packet_name,
	       get_packet_name (succ, graph_p));
    else
      fprintf (f, " %4d", succ->number);
  if (!graph_p)
    {
      fprintf (f, "\n%s  packet operations:\n%s", prefix, prefix);
      for (curr_op = packet->first_op;
	   curr_op != NULL;
	   curr_op = curr_op->next_op)
	{
	  fprintf (f, "  iter = %3d op = %s", curr_op->it_num,
		   insn_name (curr_op->insn, FALSE));
	  fprintf (f, "\n%s      input dependencies:", prefix);
	  for (curr_dep = curr_op->in_deps;
	       curr_dep != NULL;
	       curr_dep = curr_dep->next_in)
	    fprintf
	      (f, "\n%s        %s (%3d) : packet %3d, iter = %3d, op = %s",
	       prefix,
	       (curr_dep->dep_mode == DEF_USE
		? "DEF-USE"
		: (curr_dep->dep_mode == DEF_DEF ? "DEF-DEF" : "USE-DEF")),
	       curr_dep->latency, curr_dep->from->packet->number,
	       curr_dep->from->it_num,
	       insn_name (curr_dep->from->insn, FALSE)); 
	  fprintf (f, "\n%s      output dependencies:", prefix);
	  for (curr_dep = curr_op->out_deps;
	       curr_dep != NULL;
	       curr_dep = curr_dep->next_out)
	    fprintf
	      (f, "\n%s        %s (%3d) : packet %3d, iter = %3d, op = %s",
	       prefix,
	       (curr_dep->dep_mode == DEF_USE
		? "DEF-USE"
		: (curr_dep->dep_mode == DEF_DEF ? "DEF-DEF" : "USE-DEF")),
	       curr_dep->latency, curr_dep->to->packet->number,
	       curr_dep->to->it_num, insn_name (curr_dep->to->insn, FALSE)); 
	  fprintf (f, "\n%s", prefix);
	}
    }
}

/* The following recursive function prints CFG and DDG starting with
   PACKET into file F.  PREFIX is output after each newline.  If
   GRAPH_P is TRUE the output is in a graphic visualization program
   format.  */

static void
print_packets_1 (FILE * f, packet_ptr packet, const char * prefix,
		 int graph_p)
{
  int succ_number;
  packet_ptr succ;

  if (packet->visit_number5 == last_packet_visit_number5)
    return;
  packet->visit_number5 = last_packet_visit_number5;
  print_packet (f, packet, prefix, graph_p);
  for (succ_number = 0;
       (succ = packet->succ [succ_number]) != NULL;
       succ_number++)
    print_packets_1 (f, succ, prefix, graph_p);
}

/* The following function prints CFG and DDG starting with PACKET into
   file F.  PREFIX is output after each newline.  If GRAPH_P is TRUE
   the output is in a graphic visualization program format.  */

static void
print_packets (FILE * f, packet_ptr start, const char * prefix, int graph_p)
{
  last_packet_visit_number5++;
  if (graph_p)
    {
      fprintf (f, "\ndigraph CFG {\n");
      fprintf (f, "  node [shape=box, fontsize=200];\n");
      fprintf (f, "  edge [dir=forward,tailclip=true,headclip=true];\n");
      fprintf (f, "  ratio=fill;\n");
      fprintf (f, "  page = \"8.5, 11\"; // inches\n");
      fprintf (f, "  size = \"7.5, 10\"; // inches\n");
    }
  print_packets_1 (f, start, prefix, graph_p);
  if (graph_p)
    fprintf (f, "\n} /* end digraph */\n");
}

#endif /* #ifdef RCSP_DEBUG */

/* This entry function makes software pipelining of loop body started
   with START_PACKET.  The function returns TRUE if loop has been
   pipelined successfully.  DEBUG_FILE and PREFIX are used for
   debugging output.  */

packet_ptr
pipeline (packet_ptr start_packet, FILE * debug_file, const char *prefix)
{
  packet_set_t scheduled_before;
  packet_ptr   r;
  packet_ptr n;
  packet_ptr scheduled_packet;
  packet_ptr pred;
  int        ok_p;
  int        loop_ops_number;
  op_ptr     branch ATTRIBUTE_UNUSED;
  int        pred_number ATTRIBUTE_UNUSED;

  if (iteration_look_ahead <= 0 || RCSP_UNROLL_FACTOR <= 1
      || RCSP_EXPANSION_LOOP_FACTOR <= 1)
    abort ();
  last_packet_visit_number1 = 0;
  last_packet_visit_number2 = 0;
  last_packet_visit_number3 = 0;
  last_packet_visit_number4 = 0;
  last_packet_visit_number5 = 0;
  loop_ops_number
    = packets_number / (RCSP_UNROLL_FACTOR * iteration_look_ahead + 1);
  if (loop_ops_number <= 0)
    abort ();
  scheduled_before = packet_set_create ();
  curr_time = 0;
  r = packet_create (curr_time);
  packet_connect (r, start_packet, FALSE, FALSE);
  build_deps (r);
  top_packet->formed_packet_p = TRUE;
  update_ready_avail_ops (top_packet->first_op, FALSE);
  last_packet_visit_number1++;
  add_independent_ready_ops (r);
  cpu_state_reset (r->cpu_state);
#ifdef RCSP_DEBUG
  if (rcsp_debug_level >= 1)
    {
      fprintf (debug_file, "\n%s\n%s ORIGINAL LOOP:\n%s",
	       prefix, prefix, prefix);
      print_packets (debug_file, r, prefix, FALSE);
      check_graphs (r);
    }
#endif
  add_frontier_packet (r);
  while (first_frontier_packet != NULL)
    {
      frontier_packet_list_sort_and_merge (debug_file, prefix);
      n = frontier_packet_list_head_remove ();
      curr_time = n->issue_time;
      schedule_packet (n, debug_file, prefix);
      scheduled_packet = packet_set_insert (scheduled_before, n);
      pred = NULL;
#if 0
      /* ??? This unfinished code is trying to redirect only packets
         which are predecessors of branches.  As consequence it
         results in less added jumps.  Is it worth to do? */
      if (scheduled_packet != n)
	/* There is already such packet.  Can we redirect pred
           packets? */
	for (pred_number = 0;
	     (pred = n->pred [pred_number]) != NULL;
	     pred_number++)
	  {
	    branch = packet_find_branch (pred, n, FALSE);
	    if (branch == NULL
		|| (branch->true_target == n && !branch->true_jump_p)
		|| (branch->false_target == n && branch->true_jump_p))
	      break;
	  }
#endif
      if (scheduled_packet == n || pred != NULL)
	{
	  add_next_packets (n);
	  /* Pipelined loop is already too big.  */
	  if (scheduled_ops_number
	      > RCSP_EXPANSION_LOOP_FACTOR * loop_ops_number)
	    break;
	  if (n->min_it_n >= (RCSP_UNROLL_FACTOR - 1) * iteration_look_ahead)
	    /* The conditional branch of the last iteration which is
               backward jump will have only one successor.  Therefore
               we have one more iteration after the lookahead.  No
               repeated patterns -- do not process other frontier
               packets.  */
	    break;
	}
      else
	redirect_packet (r, n, scheduled_packet, debug_file, prefix);
#ifdef RCSP_DEBUG
      if (rcsp_debug_level >= 1)
	check_graphs (r);
#endif
    }
  ok_p = first_frontier_packet == NULL;
  if (ok_p)
    add_jumps (r);
#ifdef RCSP_DEBUG
  if (rcsp_debug_level >= 1)
    {
      if (!ok_p)
	fprintf (debug_file, "\n%s\n%s THE LOOP HAS NOT BEEN PIPELINED!\n%s\n",
		 prefix, prefix, prefix);
      else
	{
	  fprintf (debug_file, "\n%s\n%s FINAL PIPELINED LOOP:\n%s\n",
		   prefix, prefix, prefix);
	  print_packets (debug_file, r, prefix, FALSE);
	}
    }
  if (rcsp_debug_level >= 5)
    print_packets (debug_file, r, prefix, TRUE);
#endif
  packet_set_delete (scheduled_before);
  return (ok_p ? r : NULL);
}
