/***********************************************************************
                   Copyright (c) 2007 SKY Computers, Inc.
    
     Redistribution and use in source and binary forms are permitted
     provided that this notice is preserved and that due credit is given to
     SKY Computers, Inc. The name of SKY Computers, Inc. may not be used to
     endorse or promote products derived from this software without
     specific prior written permission. This software is provided ``as is''
     without express or implied warranty.
    ************************************************************************/

/*
 *--------------------------------------------------------------------------
 *
 >++
 *
 *   File: th_main.c
 *
 *   Function: To provide the startup of the other process threads and to
 *       provide some basic control functions.
 *
 *   Calling Sequence: See below. 
 *  
 *   Implementation Details:
 *       1. Create the common data structures.
 *       2. Start the other threads.
 *       3. Handle termination.
 *
 *   Restrictions:
 *       1. None.
 *
 >**
 *   Revision Information:
 *    Date       By   Rev    Changes Made...
 *    10/14/07   bwj  0.2    New Module.
 *
 >--
 *--------------------------------------------------------------------------
 */

/* Include files */
#include "sky_ex_inc.h"

#define EX_MAIN   (1)
#include "th_tcb.h"

/* Globals */
TCB *gTcbBgn;

/*
 *  TimeTrac Externals -- 
 *      Do the following if you want the TimeTrac variables to
 *      disappear when the TimeTrac function calls disappear
 *      (see the Makefile).
 */
#ifdef TIME_TRAC

/* Time_Trac variables, handles, functions. */
TTHandle Trace_Log[NUM_THREADS];

TTEventHandle tt_hello_bgn[NUM_THREADS], tt_hello_end[NUM_THREADS];
TTEventHandle tt_vran_bgn[NUM_THREADS], tt_vran_end[NUM_THREADS];
TTEventHandle tt_vadd_bgn[NUM_THREADS], tt_vadd_end[NUM_THREADS];
TTEventHandle tt_vmul_bgn[NUM_THREADS], tt_vmul_end[NUM_THREADS];
TTEventHandle tt_vsub_bgn[NUM_THREADS], tt_vsub_end[NUM_THREADS];
TTEventHandle tt_vsin_bgn[NUM_THREADS], tt_vsin_end[NUM_THREADS];

TTEventHandle MN_init_bgn, MN_init_end;
TTEventHandle MN_run_bgn, MN_run_end;
TTEventHandle MN_reset_bgn, MN_reset_end;

#endif  /* TIME_TRAC */

/* Compute thread entry points. */
extern int compute_main (TCB *t);

/* ---------------------------------------------------------------------- */

/*
 >++
 *
 *   void init_time_trac (int rank):
 *       int rank: Assigned to the calling thread (always 0 in this example)
 *
 *       Set up the TimeTrac file (open the file, define the range events) for 
 *       the main processing thread and structures.
 *
 *       Returns: Nothing
 >--
 */

void init_time_trac (rank)
{
  int status;
  int ret;
  char trace_name[80];
  char MsgStr[MAX_STR_SIZE];
  
  sprintf (trace_name, "%02d_Main", rank);
  
  ret = time_trac_open (trace_name, 0, TT_MAX_EVENTS, 0xffffffff, TT_NO_AUTO_SAVE, TT_CREATE_FILE, 1, &Trace_Log[rank]);
  if (ret != TTE_SUCCESS)
    {
      sprintf (MsgStr, "MN %02d: WARNING -- TimeTrac file (%s) could not be opened.", rank, trace_name);
    }
  
  status = time_trac_reg_range_event (Trace_Log[rank], "a. MN Init", "Brown", 1, &MN_init_bgn, &MN_init_end);
  VERIFY_TIMETRAC_STATUS (status);

  status = time_trac_reg_range_event (Trace_Log[rank], "b. MN Run", "Black", 1, &MN_run_bgn, &MN_run_end);
  VERIFY_TIMETRAC_STATUS (status);

  status = time_trac_reg_range_event (Trace_Log[rank], "c. MN Reset", "BROWN", 1, &MN_reset_bgn, &MN_reset_end);
  VERIFY_TIMETRAC_STATUS (status);
}

/* ---------------------------------------------------------------------- */ 

/*
 >++
 *
 *   void tt_init_co (TCB *tcb):
 *       TCB *tcb: Pointer to the thread control block describing this thread.
 *
 *       Set up the TimeTrac file (open the file, define the range events) for 
 *       a compute thread and structures. Which thread is defined by the TCB.
 *
 *       Returns: Nothing
 >--
 */

void tt_init_co (TCB *tcb)
{
  unsigned group_mask = 0xffffffff;
  unsigned group_id = 0x1;
  int use_thread_safe = 0;
  int num_events = 10000;
  int context_depth = 1;
  int status = 4;
  char filename[256];
  
  /* Create a unique trace file name for each thread. */
  sprintf (filename, "%02d_Compute", tcb->Rank);
  
  /* Get the TimeTrac handle */
  status = time_trac_open (filename, use_thread_safe, num_events, group_mask,
			   TT_NO_AUTO_SAVE, TT_CREATE_FILE, context_depth, &Trace_Log[tcb->Rank]);
  VERIFY_TIMETRAC_STATUS (status);
  
  /* 
   *  Create all of the range events that we wish to log. Note that the names 
   *  have been assigned in a manner that makes "sort-by-name very easy.
   */
  status = time_trac_reg_range_event (Trace_Log[tcb->Rank], "a. hello", "Green",
				      group_id, &tt_hello_bgn[tcb->Rank], &tt_hello_end[tcb->Rank]);
  VERIFY_TIMETRAC_STATUS (status);
  
  status = time_trac_reg_range_event (Trace_Log[tcb->Rank], "b. vinput", "Brown",
				      group_id, &tt_vran_bgn[tcb->Rank], &tt_vran_end[tcb->Rank]);
  VERIFY_TIMETRAC_STATUS (status);
  
  status = time_trac_reg_range_event (Trace_Log[tcb->Rank], "c. vadd", "Orange",
				      group_id, &tt_vadd_bgn[tcb->Rank], &tt_vadd_end[tcb->Rank]);
  VERIFY_TIMETRAC_STATUS (status);
  
  status = time_trac_reg_range_event (Trace_Log[tcb->Rank], "d. vmult", "Violet",
				      group_id, &tt_vmul_bgn[tcb->Rank], &tt_vmul_end[tcb->Rank]);
  VERIFY_TIMETRAC_STATUS (status);
  
  status = time_trac_reg_range_event (Trace_Log[tcb->Rank], "e. vsub", "Blue",
				      group_id, &tt_vsub_bgn[tcb->Rank], &tt_vsub_end[tcb->Rank]);
  VERIFY_TIMETRAC_STATUS (status);
}

/* ---------------------------------------------------------------------- */ 

/*
 >++
 *
 *   void tt_term (): 
 *
 *       Save all of the TimeTrac events and then close the TimeTrac file.
 *
 *       Returns: Nothing
 >--
 */

int tt_term (TCB *tcb)
{
  int status;
  
  /* Save the events by overwritting any existing file, close the TimeTrac file. */
  status = time_trac_save (Trace_Log[tcb->Rank], TT_OVERWRITE_FILE);
  VERIFY_TIMETRAC_STATUS (status);

  status = time_trac_close (Trace_Log[tcb->Rank]);
  VERIFY_TIMETRAC_STATUS (status);
}

/*--------------------------------------------------------------------------*/

/*
 >++
 *
 *   int t_set_common (int rank, TCB *t, EPB *p):
 *       int rank: of the thread of interest
 *       TCB *t: Task block to be initialized.
 *       EPB *p: Global parameter block containing system wide values
 *
 *       Given the rank, the task block, and the parameter list, initialize the 
 *       common values in the task control block.
 *   
 *       Returns: 0 on success.
 >--
 */

int t_set_common (int rank, TCB *t, EPB *p)
{
  t->Rank = rank;
  t->Comm = 0;
  t->Status = 0;
  t->Faults = 0;
  
  return (0);
}

/*--------------------------------------------------------------------------*/

/*
 >++
 *
 *   int main (int argc, char **argv, char **arge): 
 *
 *       This is the master thread that starts all of the other threads. It 
 *       synchronizes the startup, tells each to continue, and then saves the 
 *       TimeTrac files and terminates when done.   
 *   
 *       Returns: Exits with 0 when done, -1 on failure.
 >--
 */

main (int argc, char **argv, char **arge)
{
  int i;
  int rank;
  TCB *tcb;
  EPB *epb;
  pthread_t ex_thread[NUM_THREADS];
  EPB *ex_params;

  /* The main program has rank == 0. */
  rank = 0;
  
  /* Get the TimeTrac stuff started for the main thread. */
  init_time_trac (rank);
  time_trac_record (Trace_Log[rank], MN_init_bgn, rank);

  /* Initialize the memory allocator, allocate structures. */
  ex_mem_init (rank);
  gTcbBgn = (TCB *) ex_mem_alloc (rank, sizeof(TCB) * NUM_THREADS, 64, "Thread Control Blocks");
  tcb = &gTcbBgn[rank];
  epb = (EPB *) ex_mem_alloc (rank, sizeof(TCB) * NUM_THREADS, 64, "Thread Parameter Block");

  /* 
   *  Start all of the compute threads.
   */
  for (i = 1; i < NUM_THREADS; i++)
    {
      /* Set parameters for this thread ...  */
      tcb = &gTcbBgn[i];
      t_set_common (i, tcb, ex_params);
      tcb->Rank = i;
      tcb->Comm = COMM_NONE;
      tcb->Status = STAT_NONE;
            
      /* Set up the TimeTrac parameters for this thread. */
      tt_init_co (tcb);
      
      /* ... and create the thread. */
      if (pthread_create (&ex_thread[rank], NULL, (void *) &compute_main, (void *) tcb) != 0)
        {
	  printf ("MN %2d: PANIC -- Th %d Thread Create Failed (%s, line %d).", 
		  rank, i, __FILE__, __LINE__);
	  exit (-1);
        }
    }

  ex_html_mem_dump (rank);
  
  /* Stop one trace and start the other. */
  time_trac_record (Trace_Log[0], MN_init_end, rank);
  time_trac_record (Trace_Log[0], MN_run_bgn, 0);
  
  /* 
   *  We have started all threads, now wait for all threads to 
   *  complete startup by entering the WAIT state.
   */
  for (i = 1; i < NUM_THREADS; i++)
    {
      tcb = &gTcbBgn[i];
      while (tcb->Status != STAT_WAIT)
        {
	  // printf ("MN %2d: Waiting for thread %d (now = %s).\n", 
	  //   rank, i, stat_name[tcb->Status]);
	  
	  usleep (1000);
        }
    }
  
  /* 
   *  Tell all of the compute threads to go.
   */
  for (i = 1; i < NUM_THREADS; i++)
    {
      /* Set ..  */
      tcb = &gTcbBgn[i];
      tcb->Status = STAT_NONE;
      tcb->Comm = COMM_COMP;
    }
  
  /* 
   *  We have told all threads to compute, now wait for all threads to 
   *  complete their work by entering the wait state.
   */
  for (i = 1; i < NUM_THREADS; i++)
    {
      tcb = &gTcbBgn[i];
      while (tcb->Status != STAT_DONE)
        {
	  //printf ("MN %2d: Waiting for thread %d (now = %s).", 
	  //   rank, i, stat_name[tcb->Status]);
	  
	  usleep (1000);
        }
    }
  
  /* End of testing, do a orderly shutdown. */
  printf ("MN %2d: Stopped, doing cleanup.\n", rank);
  
  time_trac_record (Trace_Log[0], MN_run_end, 0);
  
  /*  Tell all threads to exit.  */
  for (i = 0; i < NUM_THREADS; i++)
    {
      tcb = &gTcbBgn[i];
      tt_term (tcb);
      tcb->Comm = COMM_EXIT;
    }
  
  usleep (100000);
  
  printf ("MN %2d: Application exiting.\n", rank);
  usleep (100000);
  exit (0);
}

/* --------------------------- End of Module ---------------------------- */