Club Amiga de Montreal - CAM/CAM_CD_1.iso/files/618a.lha/NeuralNetwork/neural_network.cc

C/C++ Source or Header | 1992-03-01 | 24KB | 810 lines

#include <stdio.h>
#include <new.h>
#include <math.h>
#include <File.h>
#include "Neural_network.h"

void Neural_network::allocate_matrices ()
{

  // Time to allocate the entire neural_net structure
  // Activation matrices
  hidden1_act = new double [num_hidden1];
  hidden2_act = new double [num_hidden2];
  output_act = new double [num_outputs];

  // Weight matrices
  input_weights = new double [num_hidden1 * num_inputs];
  hidden1_weights = new double [num_hidden2 * num_hidden1];
  hidden2_weights = new double [num_outputs * num_hidden2];

  // Learning rate matrices for each weight's learning rate.
  // Needed for delta-bar-delta algorithm
  input_learning_rate = new double [num_hidden1 * num_inputs];
  hidden1_learning_rate = new double [num_hidden2 * num_hidden1];
  hidden2_learning_rate = new double [num_outputs * num_hidden2];

  // Learning rate deltas for each weight's learning rate.
  // Needed for delta-bar-delta algorithm.
  input_learning_delta = new double [num_hidden1 * num_inputs];
  hidden1_learning_delta = new double [num_hidden2 * num_hidden1];
  hidden2_learning_delta = new double [num_outputs * num_hidden2];

  // Delta bar matrices for each weight's delta bar.
  // Needed for delta-bar-delta algorithm.
  input_learning_delta_bar = new double [num_hidden1 * num_inputs];
  hidden1_learning_delta_bar = new double [num_hidden2 * num_hidden1];
  hidden2_learning_delta_bar = new double [num_outputs * num_hidden2];

  // Weight delta matrices for each weights delta.
  // Needed for BackPropagation algorithm.
  input_weights_sum_delta = new double [num_hidden1 * num_inputs];
  hidden1_weights_sum_delta = new double [num_hidden2 * num_hidden1];
  hidden2_weights_sum_delta = new double [num_outputs * num_hidden2];

  // Sum of delta * weight matrices for each weight.
  // Needed for BackPropagation algorithm.
  hidden1_sum_delta_weight = new double [num_hidden2 * num_hidden1];
  hidden2_sum_delta_weight = new double [num_outputs * num_hidden2];

  /* Done neural net allocation */
}

void Neural_network::initialize_matrices (double range)
{
  int    x,y;
  double rand_max;

  training_examples = 0;
  examples_since_update = 0;

  rand_max = RAND_MAX;
  /* Initialize all weights from -range to +range randomly */
  for (x = 0; x < num_hidden1; ++x)
    {
      hidden1_sum_delta_weight [x] = 0.0;

      for (y = 0; y < num_inputs; ++y)
        {
          input_weights [x * num_inputs + y] = rand () / rand_max * range;
          if ( rand () < (RAND_MAX / 2) )
              input_weights [x * num_inputs + y] *= -1.0;

          input_weights_sum_delta [x * num_inputs + y] = 0.0;
          input_learning_rate [x * num_inputs + y] = learning_rate;
          input_learning_delta [x * num_inputs + y] = 0.0;
          input_learning_delta_bar [x * num_inputs + y] = 0.0;
        }
    }

  for (x = 0; x < num_hidden2; ++x)
    {
      hidden2_sum_delta_weight [x] = 0.0;

      for (y = 0; y < num_hidden1; ++y)
        {
          hidden1_weights [x * num_hidden1 + y] = rand () / rand_max * range;
          if ( rand () < (RAND_MAX / 2) )
              hidden1_weights [x * num_hidden1 + y] *= -1.0;

          hidden1_weights_sum_delta [x * num_hidden1 + y] = 0.0;
          hidden1_learning_rate [x * num_hidden1 + y] = learning_rate;
          hidden1_learning_delta [x * num_hidden1 + y] = 0.0;
          hidden1_learning_delta_bar [x * num_hidden1 + y] = 0.0;
        }
    }

  for (x = 0; x < num_outputs; ++x)
    {
      for (y = 0; y < num_hidden2; ++y)
        {
          hidden2_weights [x * num_hidden2 + y] = rand () / rand_max * range;
          if ( rand () < (RAND_MAX / 2) )
              hidden2_weights [x * num_hidden2 + y] *= -1.0;

          hidden2_weights_sum_delta [x * num_hidden2 + y] = 0.0;
          hidden2_learning_rate [x * num_hidden2 + y] = learning_rate;
          hidden2_learning_delta [x * num_hidden2 + y] = 0.0;
          hidden2_learning_delta_bar [x * num_hidden2 + y] = 0.0;
        }
    }

}

void Neural_network::deallocate_matrices ()
{

  // Time to destroy the entire neural_net structure
  // Activation matrices
  delete hidden1_act;
  delete hidden2_act;
  delete output_act;

  // Weight matrices
  delete input_weights;
  delete hidden1_weights;
  delete hidden2_weights;

  // Learning rate matrices for each weight's learning rate.
  // Needed for delta-bar-delta algorithm
  delete input_learning_rate;
  delete hidden1_learning_rate;
  delete hidden2_learning_rate;

  // Learning rate deltas for each weight's learning rate.
  // Needed for delta-bar-delta algorithm.
  delete input_learning_delta;
  delete hidden1_learning_delta;
  delete hidden2_learning_delta;

  // Delta bar matrices for each weight's delta bar.
  // Needed for delta-bar-delta algorithm.
  delete input_learning_delta_bar;
  delete hidden1_learning_delta_bar;
  delete hidden2_learning_delta_bar;

  // Weight delta matrices for each weights delta.
  // Needed for BackPropagation algorithm.
  delete input_weights_sum_delta;
  delete hidden1_weights_sum_delta;
  delete hidden2_weights_sum_delta;

  // Sum of delta * weight matrices for each weight.
  // Needed for BackPropagation algorithm.
  delete hidden1_sum_delta_weight;
  delete hidden2_sum_delta_weight;

  /* Done neural net deallocation */
}

Neural_network::Neural_network (int number_inputs, int number_hidden1,
                                int number_hidden2, int number_outputs,
                                double t_epsilon, double t_skip_epsilon,
                                double t_learning_rate, double t_theta,
                                double t_phi, double t_K, double range) :
                num_inputs (number_inputs), num_hidden1 (number_hidden1),
                num_hidden2 (number_hidden2), num_outputs (number_outputs),
                epsilon (t_epsilon), skip_epsilon (t_skip_epsilon),
                learning_rate (t_learning_rate), theta (t_theta), phi (t_phi),
                K (t_K), training_examples (0), examples_since_update (0)
{
  allocate_matrices ();
  initialize_matrices (range);
}


Neural_network::Neural_network (char *filename, int& file_error,
                                double t_epsilon, double t_skip_epsilon,
                                double t_learning_rate, double t_theta,
                                double t_phi, double t_K) :
                epsilon (t_epsilon), skip_epsilon (t_skip_epsilon),
                learning_rate (t_learning_rate), theta (t_theta), phi (t_phi),
                K (t_K), examples_since_update (0),
                hidden1_act (0), hidden2_act (0), output_act (0),
                input_weights (0), hidden1_weights (0), hidden2_weights (0),
                input_learning_rate (0), hidden1_learning_rate (0),
                hidden2_learning_rate (0), input_learning_delta (0),
                hidden1_learning_delta (0), hidden2_learning_delta (0),
                input_learning_delta_bar (0), hidden1_learning_delta_bar (0),
                hidden2_learning_delta_bar (0), input_weights_sum_delta (0),
                hidden1_weights_sum_delta (0), hidden2_weights_sum_delta (0),
                hidden1_sum_delta_weight (0), hidden2_sum_delta_weight (0)
{
  file_error = read_weights (filename);
}

int Neural_network::read_weights (char *filename)
{
  File    *fp;
  int     x;
  long    iter;

  if ( ((fp = new File (filename,"rt")) == NULL) ||
       (!fp->is_open ()) )
    {
      printf ("Could not read weights from file '%s'\n",filename);
      return (-1);
    }

  /* First read in how many iterations have been performed so far */
  fp->scan ("%ld",&iter);
  printf ("Iterations = %ld\n",iter);

  /* Next read in how many input nodes, hidden1 nodes, hidden2 nodes */
  /* and output nodes. */
  fp->scan ("%d%d%d%d",&num_inputs,&num_hidden1,&num_hidden2,&num_outputs);

  // Deallocate previous matrices
  deallocate_matrices ();

  /* Allocate new matrices with new size */
  allocate_matrices ();

  // Initialize all matrices and variables
  initialize_matrices (1.0);
  training_examples = iter;

  /* Read input->hidden1 weights from file. */
  for (x = 0; x < num_inputs * num_hidden1; ++x)
    {
      fp->scan ("%lf",&input_weights [x]);
    }

  /* Read hidden1->hidden2 weights from file. */
  for (x = 0; x < num_hidden1 * num_hidden2; ++x)
    {
      fp->scan ("%lf",&hidden1_weights [x]);
    }

  /* Read hidden2->output weights from file. */
  for (x = 0; x < (num_hidden2 * num_outputs); ++x)
    {
      fp->scan ("%lf",&hidden2_weights [x]);
    }

  /* Now all the weights have been loaded */
 delete fp;

 return (0);
}



int Neural_network::save_weights (char *filename)
{
  File *fp;
  int   x;

  if ( ((fp = new File (filename,"wt")) == NULL) ||
       (!fp->is_open ()) )
    {
      printf ("Could not save weights to file '%s'\n",filename);
      return (-1);
    }

  /* First write out how many iterations have been performed so far */
  fp->form (" %ld\n",training_examples);

  /* Next write out how many input nodes, hidden1 nodes, hidden2 nodes */
  /* and output nodes. */
  fp->form (" %d %d %d %d\n",num_inputs,num_hidden1,num_hidden2,num_outputs);

  /* Write input->hidden1 weights to output. */
  for (x = 0; x < num_inputs * num_hidden1; ++x)
    {
      fp->form ("%10.5f ",input_weights [x]);
      if ( (x % 5) == 4 )
          fp->form ("\n");
    }
  fp->form ("\n\n");

  /* Write hidden1->hidden2 weights to output. */
  for (x = 0; x < num_hidden1 * num_hidden2; ++x)
    {
      fp->form ("%10.5f ",hidden1_weights [x]);
      if ( (x % 5) == 4 )
          fp->form ("\n");
    }
  fp->form ("\n\n");

  /* Write hidden2->output weights to output. */
  for (x = 0; x < (num_hidden2 * num_outputs); ++x)
    {
      fp->form ("%10.5f ",hidden2_weights [x]);
      if ( (x % 5) == 4 )
          fp->form ("\n");
    }
  fp->form ("\n\n");

  fp->flush ();
  printf ("Flushed file\n");
  fp->close ();
  printf ("Closed file\n");

  delete fp;

  /* Now all the weights have been saved */
  return (0);
}



void Neural_network::set_size_parameters (int number_inputs,
                                 int number_hidden1, int number_hidden2,
                                 int number_outputs, double range)
{
  double *new_input_weights,*new_hidden1_weights,*new_hidden2_weights;
  double rand_max;
  int    x;

  rand_max = RAND_MAX;

  // Allocate new weight matrices with new size
  new_input_weights = new double [number_hidden1 * number_inputs];
  new_hidden1_weights = new double [number_hidden2 * number_hidden1];
  new_hidden2_weights = new double [number_outputs * number_hidden2];

  // Copy over all weights
  // Input weights
  for (x = 0; x < number_hidden1 * number_inputs; ++x)
    {
      // IF the new size is larger than the old size, THEN make new connections
      // a random weight between +-range.
      if ( x >= (num_hidden1 * num_inputs) )
        {
          new_input_weights [x] = rand () / rand_max * range;
          if ( rand () < (RAND_MAX / 2) )
              new_input_weights [x] *= -1.0;
        }
      else
          new_input_weights [x] = input_weights [x];
    }

  // Hidden1 weights
  for (x = 0; x < number_hidden2 * number_hidden1; ++x)
    {
      // IF the new size is larger than the old size, THEN make new connections
      // a random weight between +-range.
      if ( x >= (num_hidden2 * num_hidden1) )
        {
          new_hidden1_weights [x] = rand () / rand_max * range;
          if ( rand () < (RAND_MAX / 2) )
              new_hidden1_weights [x] *= -1.0;
        }
      else
          new_hidden1_weights [x] = hidden1_weights [x];
    }

  // Hidden2 weights
  for (x = 0; x < number_outputs * number_hidden2; ++x)
    {
      // IF the new size is larger than the old size, THEN make new connections
      // a random weight between +-range.
      if ( x >= (num_outputs * num_hidden2) )
        {
          new_hidden2_weights [x] = rand () / rand_max * range;
          if ( rand () < (RAND_MAX / 2) )
              new_hidden2_weights [x] *= -1.0;
        }
      else
          new_hidden2_weights [x] = hidden2_weights [x];
    }

  // All weights have been copied.

  // Change size paramters
  num_inputs = number_inputs;
  num_hidden1 = number_hidden1;
  num_hidden2 = number_hidden2;
  num_outputs = number_outputs;

  // Deallocate all matrices
  deallocate_matrices ();

  // Allocate new nerual network matrices with the correct size and initialize
  allocate_matrices ();
  initialize_matrices (1.0);

  // Now deallocate new randomly initialized weight matrices and assign them
  // to the new weight matrices that have the correct weight values.
  delete input_weights;
  delete hidden1_weights;
  delete hidden2_weights;

  input_weights = new_input_weights;
  hidden1_weights = new_hidden1_weights;
  hidden2_weights = new_hidden2_weights;

}


void Neural_network::back_propagation (double input [],
                                       double desired_output [],
                                       int& done)
{
  int     x,y;
  int     size;
  double  delta,sum_delta,*weight,*p_sum_delta,*p_learning_delta;

  /* First check if training complete. */
  for (x = num_outputs - 1; x >= 0; --x)
    {
      if ( fabs (desired_output [x] - output_act [x]) > epsilon )
        {
          done = 0;
        }
    }

  /* Go backward through list for speed */
  size = num_hidden2;
  /* First calculate deltas of weights from output to hidden layer 2. */
  for (x = num_outputs - 1; x >= 0; --x)
    {
      weight = &hidden2_weights [x * size];
      p_sum_delta = &hidden2_weights_sum_delta [x * size];
      p_learning_delta = &hidden2_learning_delta [x * size];
      for (y = num_hidden2 - 1; y >= 0; --y)
        {
          /* Formula delta = (desired - actual) * derivative
             derivative = S(1 - S)
             Also calculate sum of deltas * weight for next layer.
          */
          delta = (desired_output [x] - output_act [x])
                   * SLOPE * output_act [x] * (1.0 - output_act [x]);

          hidden2_sum_delta_weight [y] += delta * weight [y];

          p_learning_delta [y] += delta;

          /* Now multiply by activation and sum in weights sum delta */
          p_sum_delta [y] += delta * hidden2_act [y];
        }
    }


  /* Next calculate deltas of weights between hidden layer2 and hidden
     layer 1 */
  size = num_hidden1;
  for (x = num_hidden2 - 1; x >= 0; --x)
    {
      sum_delta = hidden2_sum_delta_weight [x];
      weight = &hidden1_weights [x * size];
      p_sum_delta = &hidden1_weights_sum_delta [x * size];
      p_learning_delta = &hidden1_learning_delta [x * size];
      for (y = num_hidden1 - 1; y >= 0; --y)
        {
          /* Formula delta = SUM (previous deltas*weight)
                             * derivative
             previous deltas already muliplied by weight.
             derivative = S(1 - S)

             Also calculate sum of deltas * weight to save from doing
             it for next layer.
          */

          delta = sum_delta * hidden2_act [x] *
                  (1.0 - hidden2_act [x]) * SLOPE;

          hidden1_sum_delta_weight [y] += delta * weight [y];

          p_learning_delta [y] += delta;

          /* Now multiply by activation and sum in weights_sum_delta */
          p_sum_delta [y] += delta * hidden1_act [y];
        }
      hidden2_sum_delta_weight [x] = 0.0;
    }

  /* Finally calculate deltas of weights between hidden layer 1 and input
     layer */
  size = num_inputs;
  for (x = num_hidden1 - 1; x >= 0; --x)
    {
      sum_delta = hidden1_sum_delta_weight [x];
      p_sum_delta = &input_weights_sum_delta [x * size];
      p_learning_delta = &input_learning_delta [x * size];
      for (y = num_inputs - 1; y >= 0; --y)
        {
          /* Formula delta = SUM (previous deltas*weight)
                             * derivative * activation of input
             previous deltas already muliplied by weight
             derivative = S(1 - S)
          */
          delta = sum_delta * hidden1_act [x] *
                  (1.0 - hidden1_act [x]) * SLOPE;

          p_learning_delta [y] += delta;

          p_sum_delta [y] += (delta * input [y]);
        }
      hidden1_sum_delta_weight [x] = 0.0;
    }

  /* Now all deltas have been calculated and added into their appropriate
     neuron connection. */
  ++examples_since_update;

}


double Neural_network::calc_forward (double input [], double desired_output [],
                                     int& num_wrong, int& skip, int print_it,
                                     int& actual_printed)
{
  int     x,y,wrong;
  int     size;
  double  *weight,error,abs_error;

  skip = 1;
  wrong = 0;
  error = 0.0;

  /* Go backward for faster processing */
  /* Calculate hidden layer 1's activation */
  size = num_inputs;
  for (x = num_hidden1 - 1; x >= 0;  --x)
    {
      hidden1_act [x] = 0.0;
      weight = &input_weights [x * size];
      for (y = num_inputs - 1; y >= 0; --y)
        {
          hidden1_act [x] += (input [y] * weight [y]);
        }
      hidden1_act [x] = S(hidden1_act [x]);
    }

  /* Calculate hidden layer 2's activation */
  size = num_hidden1;
  for (x = num_hidden2 - 1; x >= 0; --x)
    {
      hidden2_act [x] = 0.0;
      weight = &hidden1_weights [x * size];
      for (y = num_hidden1 - 1; y >= 0; --y)
        {
          hidden2_act [x] += (hidden1_act [y] * weight [y]);
        }
      hidden2_act [x] = S(hidden2_act [x]);
    }

  /* Calculate output layer's activation */
  size = num_hidden2;
  for (x = num_outputs - 1; x >= 0; --x)
    {
      output_act [x] = 0.0;
      weight = &hidden2_weights [x * size];
      for (y = num_hidden2 - 1; y >= 0; --y)
        {
          output_act [x] += hidden2_act [y] * weight [y];
        }
      output_act [x] = S(output_act [x]);
      abs_error = fabs (output_act [x] - desired_output [x]);
      error += abs_error;
      if ( abs_error > epsilon )
          wrong = 1;
      if ( abs_error > skip_epsilon )
          skip = 0;
    }

  if ( wrong )
      ++num_wrong;

  if ( print_it == 2 )
    {
      for (x = 0; x < num_outputs; ++x)
        {
          printf ("%6.4f ",output_act [x]);
        }
      ++actual_printed;
    }
  else if ( print_it && wrong )
    {
      for (x = 0; x < num_outputs; ++x)
        {
          printf ("%6.4f ",fabs (desired_output [x] - output_act [x]));
        }
      ++actual_printed;
    }

  return (error);

}


void Neural_network::update_weights ()
{
  int     x,y;
  int     size;
  double  rate,*p_ldelta,*p_ldelta_bar,*weight,*p_lrate,*p_sum_delta;

  // Check to see if any changes have been calculated.
  if ( examples_since_update == 0 )
    {
      return;
    }

  /* Go backward for slightly faster processing */
  /* First add deltas of weights from output to hidden layer 2. */
  size = num_hidden2;
  for (x = num_outputs - 1; x >= 0; --x)
    {
      p_ldelta = &hidden2_learning_delta [x * size];
      p_ldelta_bar = &hidden2_learning_delta_bar [x * size];
      weight = &hidden2_weights [x * size];
      p_lrate = &hidden2_learning_rate [x * size];
      p_sum_delta = &hidden2_weights_sum_delta [x * size];

      for (y = num_hidden2 - 1; y >= 0; --y)
        {
          rate = p_ldelta [y] * p_ldelta_bar [y];
          if ( rate < 0.0 )
            {
              p_lrate [y] -= (phi * p_lrate [y]);
            }
          else if ( rate > 0.0 )
            {
              p_lrate [y] += K;
            }

          weight [y] += (p_lrate [y] * p_sum_delta [y]);
          p_sum_delta [y] = 0.0;
          p_ldelta_bar [y] *= theta;
          p_ldelta_bar [y] += ((1.0 - theta) * p_ldelta [y]);
          p_ldelta [y] = 0.0;
        }
    }

  /* Next add deltas of weights between hidden layer2 and hidden
     layer 1 */
  size = num_hidden1;
  for (x = num_hidden2 - 1; x >= 0; --x)
    {
      p_ldelta = &hidden1_learning_delta [x * size];
      p_ldelta_bar = &hidden1_learning_delta_bar [x * size];
      weight = &hidden1_weights [x * size];
      p_lrate = &hidden1_learning_rate [x * size];
      p_sum_delta = &hidden1_weights_sum_delta [x * size];

      for (y = num_hidden1 - 1; y >= 0; --y)
        {
          rate = p_ldelta [y] * p_ldelta_bar [y];
          if ( rate < 0.0 )
            {
              p_lrate [y] -= (phi * p_lrate [y]);
            }
          else if ( rate > 0.0 )
            {
              p_lrate [y] += K;
            }

          weight [y] += (p_lrate [y] * p_sum_delta [y]);
          p_sum_delta [y] = 0.0;
          p_ldelta_bar [y] *= theta;
          p_ldelta_bar [y] += ((1.0 - theta) * p_ldelta [y]);
          p_ldelta [y] = 0.0;
        }
    }

  /* Finally add deltas of weights between hidden layer 1 and input
     layer */
  size = num_inputs;
  for (x = num_hidden1 - 1; x >= 0; --x)
    {
      p_ldelta = &input_learning_delta [x * size];
      p_ldelta_bar = &input_learning_delta_bar [x * size];
      weight = &input_weights [x * size];
      p_lrate = &input_learning_rate [x * size];
      p_sum_delta = &input_weights_sum_delta [x * size];

      for (y = num_inputs - 1; y >= 0; --y)
        {
          rate = p_ldelta [y] * p_ldelta_bar [y];
          if ( rate < 0.0 )
            {
              p_lrate [y] -= (phi * p_lrate [y]);
            }
          else if ( rate > 0.0 )
            {
              p_lrate [y] += K;
            }

          weight [y] += (p_lrate [y] * p_sum_delta [y]);
          p_sum_delta [y] = 0.0;
          p_ldelta_bar [y] *= theta;
          p_ldelta_bar [y] += ((1.0 - theta) * p_ldelta [y]);
          p_ldelta [y] = 0.0;
        }
    }

  /* Now all deltas have been added into their appropriate neuron
     connection. */
  training_examples += examples_since_update;
  examples_since_update = 0;

}


int Neural_network::calc_forward_test (double input [],
                                       double desired_output [],
                                       int print_it, double correct_eps,
                                       double good_eps)
{
  int x,y,wrong,good;

  wrong = 0;
  good = 0;

  /* Go backward for faster processing */
  /* Calculate hidden layer 1's activation */
  for (x = num_hidden1 - 1; x >= 0;  --x)
    {
      hidden1_act [x] = 0.0;
      for (y = num_inputs - 1; y >= 0; --y)
        {
          hidden1_act [x] += (input [y] *
                              input_weights [x * num_inputs + y]);
        }
      hidden1_act [x] = S(hidden1_act [x]);
    }

  /* Calculate hidden layer 2's activation */
  for (x = num_hidden2 - 1; x >= 0; --x)
    {
      hidden2_act [x] = 0.0;
      for (y = num_hidden1 - 1; y >= 0; --y)
        {
          hidden2_act [x] += (hidden1_act [y] *
                                 hidden1_weights [x*num_hidden1 + y]);
        }
      hidden2_act [x] = S(hidden2_act [x]);
    }

  /* Calculate output layer's activation */
  for (x = num_outputs - 1; x >= 0; --x)
    {
      output_act [x] = 0.0;
      for (y = num_hidden2 - 1; y >= 0; --y)
        {
          output_act [x] += hidden2_act [y] *
                                hidden2_weights [x * num_hidden2 + y];
        }
      output_act [x] = S(output_act [x]);

      if ( fabs (output_act [x] - desired_output [x]) > good_eps )
          wrong = 1;
      else if ( fabs (output_act [x] - desired_output [x]) > correct_eps )
          good = 1;
    }

  if ( print_it )
    {
      for (x = 0; x < num_outputs; ++x)
        {
          printf ("%6.4f ",output_act [x]);
        }
    }

  if ( wrong )
      return (WRONG);
  else if ( good )
      return (GOOD);
  else
      return (CORRECT);
}


void Neural_network::kick_weights (double range)
{
  int    x;
  double rand_max;
  double variation;

  rand_max = RAND_MAX;
  /* Add from -range to +range to all weights randomly */
  for (x = 0; x < (num_hidden1 * num_inputs); ++x)
    {
      variation = rand () / rand_max * range;
      if ( rand () < (RAND_MAX / 2) )
          variation *= -1.0;
      input_weights [x] += variation;
    }

  for (x = 0; x < (num_hidden2 * num_hidden1); ++x)
    {
      variation = rand () / rand_max * range;
      if ( rand () < (RAND_MAX / 2) )
          variation *= -1.0;
      hidden1_weights [x] += variation;
    }

  for (x = 0; x < (num_outputs * num_hidden2); ++x)
    {
      variation = rand () / rand_max * range;
      if ( rand () < (RAND_MAX / 2) )
          variation *= -1.0;
      hidden2_weights [x] += variation;
    }

}