/***************************************
  $Id $

  UP_util.c 

  Status: NOT REVIEWED, NOT TESTED

  Author(s):       Denis Walker

  ******************/ /******************
  Modification History:
        denis (20021021) Created.
  ******************/ /******************
  Copyright (c) 2002                              RIPE NCC

  All Rights Reserved

  Permission to use, copy, modify, and distribute this software and its
  documentation for any purpose and without fee is hereby granted,
  provided that the above copyright notice appear in all copies and that
  both that copyright notice and this permission notice appear in
  supporting documentation, and that the name of the author not be
  used in advertising or publicity pertaining to distribution of the
  software without specific, written prior permission.

  THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
  ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS; IN NO EVENT SHALL
  AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
  DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
  AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 ***************************************/

#include <glib.h>
#include <assert.h>

#include "rip.h"
#include "ep.h"
#include "rt.h"
#include "km.h"
#include "lg.h"
#include "lu_whois.h"
#include "dbupdate.h"
#include "up_pre_process.h"
#include "up_auto_nic.h"
#include "up_util.h"
#include "ca_defs.h"

/* Report an internal software error, send an ack if possoble and exit out of dbupdate
   Receives RT context
            LG context
            options structure
            error message string
   Returns  none
*/

void UP_internal_error(RT_context_t *rt_ctx, LG_context_t *lg_ctx,
                     options_struct_t *options, char *err_msg)
{
  /* ER_perror(FAC_UP, err_type, err_msg); */
  LG_log(lg_ctx, LG_FUNC,">UP_internal_error: entered");
  
  LG_log(lg_ctx, LG_FATAL,"%s", err_msg);
  LG_log(lg_ctx, LG_PERROR,"%s", err_msg);
  
  RT_internal_error(rt_ctx);
  /* close input to the reporting context */
  RT_end(rt_ctx);

  /* Send ack message */
  /* If it is a mail message and the error occured before unfolding the input then
     we will not have the email address, so no ack will be sent, but it will still
     be logged.
     If it is not a mail message then an ack will be sent and logged */
  NT_process_acknowledgement(rt_ctx, lg_ctx, options);
  
  /************************* TIDY UP AND SHUTDOWN  ***************************/

  /* close down the report */
  RT_destroy(rt_ctx);
  
  /* free any remaining memory */
  /* this is also done at the end of the main routine */
  if (options->input_file_name)
    free(options->input_file_name);
  if (options->keywords)
    free(options->keywords);
  if (options->time_str)
    free(options->time_str);
  if (options->origin)
    free(options->origin);
  if (options->mail_hdr_data.from)
    free(options->mail_hdr_data.from);
  if (options->mail_hdr_data.cc)
    free(options->mail_hdr_data.cc);
  if (options->mail_hdr_data.subject)
    free(options->mail_hdr_data.subject);
  if (options->mail_hdr_data.date)
    free(options->mail_hdr_data.date);
  if (options->mail_hdr_data.replyto)
    free(options->mail_hdr_data.replyto);
  if (options->mail_hdr_data.msgid)
    free(options->mail_hdr_data.msgid);

  LG_log(lg_ctx, LG_FATAL,"UP_internal_error: exit(1)");
  /* close down the logging */
  LG_ctx_free(lg_ctx);
  /* close the state log file, but do not delete the file */
  fclose(options->state);

  /* report abnormal termination to the wrapper */
  exit(20);
}



/* removes the '\n's and '\r's at the end of the string, and returns it
   Receives string
   Returns  string
*/

char *up_remove_EOLs(char *arg)
{
  while ( strlen(arg) > 0 &&
          (arg[strlen(arg) - 1] == '\n' || 
           arg[strlen(arg) - 1] == '\r') )
  {
    arg[strlen(arg) - 1] = '\0';
  }
 
  return arg;
}



/* Creates a temp file name using the type specified in the name
   It is first called before the LG context is set up.
   Receives LG context
            type of file (eg input, state)
   Returns  temp file name string
*/

char *UP_get_temp_filename(LG_context_t *lg_ctx, char *type)
{
  char *tmpdir = NULL;
  char hostname[MAXHOSTNAMELEN];
  char *tmpfilename;
  int length;
  int pid;

  if ( lg_ctx )
    LG_log(lg_ctx, LG_FUNC,">UP_get_temp_filename: entered type=[%s]", type);

  gethostname(hostname, MAXHOSTNAMELEN);
  pid = getpid(); 
  tmpdir = ca_get_tmpdir;

  length = strlen(tmpdir) +  strlen(hostname) + strlen(type) + 64 + 5;
  tmpfilename = (char *)malloc(length);

  snprintf(tmpfilename, length, "%s/dbupdate.%s.%s.%ld.tmp", tmpdir, type, hostname, pid );
  free(tmpdir);

  if ( lg_ctx )
    LG_log(lg_ctx, LG_FUNC,"<UP_get_temp_filename: exiting tmpfilename [%s]", tmpfilename);
  return tmpfilename;
}      



/* Adds the contents of the given file to the update log
   and to the state log for debugging
   Receives RT context
            LG context
            options structure
            input file handle
   Returns  none
*/

void UP_add_to_upd_log(RT_context_t *rt_ctx, LG_context_t *lg_ctx, 
                            options_struct_t *options, FILE *infile)
{
  char *time_str;
  char datestr[10]; 
  char *updlogfile = NULL;
  char *updlog = NULL;
  char buf[1024];
  char *line = NULL;
  char *lc_line = NULL;
  struct tm *tmstr;
  time_t now;
  time_t cur_time;
  FILE *log_file;

  LG_log(lg_ctx, LG_FUNC,">UP_add_to_upd_log: entered");
  
  /* We need to get the a date string to construct the updlog file name */
  time(&now);
  tmstr = localtime(&now);
  strftime(datestr, 10, "%Y%m%d", tmstr);

  /* now that we have the date string, we can construct updlog file name */
  updlog = ca_get_updlog;
  updlogfile = (char *)malloc(strlen(updlog) + strlen(datestr) + 2);
  snprintf(updlogfile, strlen(updlog) + strlen(datestr) + 2,
           "%s.%s", updlog, datestr);
  
 
  if (( log_file = fopen(updlogfile, "a")) == NULL)
  {
    LG_log(lg_ctx, LG_FATAL, "UP_add_to_upd_log: Can't open upd log file [%s]", updlogfile);
    free(updlogfile);
    UP_internal_error(rt_ctx, lg_ctx, options, "UP_add_to_upd_log: Can't open upd log file\n");
  }
  free(updlogfile);
   
  /* get time */
  cur_time = time(NULL);
  time_str = strdup(ctime(&cur_time));
  /* cut the '\n' at the end */
  time_str[strlen(time_str) - 1] = '\0';

  /* write the record seperater to the upd log */
  if (options->mail_input)
  {
    fprintf(log_file, "\n>>> time: %s MAIL UPDATE <<<\n\n", time_str);
  }
  else if (options->origin)
  {
    /* include the origin */
    fprintf(log_file, "\n>>> time: %s SYNC UPDATE (%s) <<<\n\n", 
                       time_str, options->origin);
    /* add the keywords from the command options */
    fprintf(log_file, "keywords - %s\n\n",
                       options->keywords ? options->keywords : "None" );
  }
  else
  {
    fprintf(log_file, "\n>>> time: %s UPDATE <<<\n\n", time_str);
    /* add the keywords from the command options */
    fprintf(log_file, "keywords - %s\n\n",
                       options->keywords ? options->keywords : "None" );
  }
  free(time_str);
  
  LG_log(lg_ctx, LG_DEBUG, "UP_add_to_upd_log: Input data");
  LG_log(lg_ctx, LG_DEBUG, ">[");

  /* write the full update to the upd log and state file */
  while ( fgets(buf, 1023, infile) != NULL ) 
  {
    line = strdup(buf);
    line = up_remove_EOLs(line);
    lc_line = strdup(line);
    g_strdown(lc_line);
    if ( ! strstr(lc_line, "override") )
    {
      fprintf(log_file, "%s", buf);
      LG_log(lg_ctx, LG_DEBUG, "%s", line);
    }
    else
    {
      /* this line contains the override password */
      LG_log(lg_ctx, LG_DEBUG, "%s  --  This line _NOT_ written to upd log", line);
    }
    free(line);
    free(lc_line);
  }
  LG_log(lg_ctx, LG_DEBUG, "<]");

  fclose(log_file);
  LG_log(lg_ctx, LG_FUNC,"<UP_add_to_upd_log: exiting");
}



/* Gets a GList of attr structures and checks if one of their 
   (upper case)values starts with substr.
   Receives LG context
            list of attribute structures
            string to search for
   Returns  1 if found
            0 if NOT found
*/

int UP_strstr_in_attr_list(LG_context_t *lg_ctx, GList *list, const char *substr)
{
  GList *item = NULL;
  char *word; 

  LG_log(lg_ctx, LG_FUNC,">UP_strstr_in_attr_list: entered with search string [%s]", substr);
 
  for( item = list; item != NULL ; item = g_list_next(item) )
  {
   word = rpsl_attr_get_clean_value((rpsl_attr_t *)(item->data));
   g_strup(word);
   if (strstr(word, substr) == word)
   {
     free(word);
     LG_log(lg_ctx, LG_FUNC,"<UP_strstr_in_attr_list: exiting with value 1 (sub string found)");
     return 1;
   }
   free(word);
  }
  /* none of them matched, so return 0 */
  LG_log(lg_ctx, LG_FUNC,"<UP_strstr_in_attr_list: exiting with value 0 (no sub string found)");
  return 0; 
}



/* takes the source string from an object and sets up the current source
   data from the list of config data for multiple sources 
   Receives RT context
            LG context
            options structure
            source string
            source data structure pointer
   Returns  UP_OK for success and UP_FAIL for source not found 
            source data in structure
*/

int up_set_current_source_data(RT_context_t *rt_ctx, LG_context_t *lg_ctx,
                                 options_struct_t *options,
                                 char *obj_source, source_data_t *source_data)
{
  int src_idx;
  int found = UP_FAIL;
  int num_sources = 0;
  ca_updDbSource_t **upd_source_hdl;
  
  LG_log(lg_ctx, LG_FUNC,">up_set_current_source_data: entered source [%s]", obj_source);

  /* retrieve source variables for the multiple sources */
  /* upd_source_hdl is a pointer to an array of pointers to source data 
     held in the ca module */
  LG_log(lg_ctx, LG_INFO,"up_set_current_source_data: get sources from config file");
  upd_source_hdl = ca_get_UpdSourceHandle(CA_UPDSOURCE);

  if (upd_source_hdl[0] == NULL)
  {
    LG_log(lg_ctx, LG_FATAL,
        "up_set_current_source_data: There must be at least one updateable source in the config file. Exiting");
    UP_internal_error(rt_ctx, lg_ctx, options,
        "up_set_current_source_data: There must be at least one updateable source in the config file. Exiting");
  }
  
  num_sources = ca_get_UpdSourceNum();
  LG_log(lg_ctx, LG_DEBUG, "up_set_current_source_data: number of sources read from config file[%d]", num_sources);
  
  for ( src_idx=0; src_idx<num_sources; src_idx++ )
  {
    if ( ! strcasecmp( obj_source, upd_source_hdl[src_idx]->name) )
    {
      /* found the source in the list */
      found = UP_OK;
      LG_log(lg_ctx, LG_DEBUG, "up_set_current_source_data: source found");
      
      if ( source_data->current_source )
        free(source_data->current_source);
      source_data->current_source = strdup(upd_source_hdl[src_idx]->name);
      source_data->update_host = upd_source_hdl[src_idx]->whoisd_host;
      if ( source_data->query_host )
        free(source_data->query_host);
      source_data->query_host = strdup(source_data->update_host);
      source_data->update_port = upd_source_hdl[src_idx]->updPort;
      source_data->query_port = upd_source_hdl[src_idx]->qryPort;
      source_data->DBhost = upd_source_hdl[src_idx]->updDb.host;
      source_data->DBport = upd_source_hdl[src_idx]->updDb.port;
      source_data->DBname = upd_source_hdl[src_idx]->updDb.dbName;
      source_data->DBuser = upd_source_hdl[src_idx]->updDb.user;
      source_data->DBpasswd = upd_source_hdl[src_idx]->updDb.password; 

      /* log the config variables for debugging */ 
      LG_log(lg_ctx, LG_DEBUG, "up_set_current_source_data: CURRENT SOURCE is: [%s]", 
                                     source_data->current_source);
      LG_log(lg_ctx, LG_DEBUG, "up_set_current_source_data: UPDATE_HOST is: [%s]", 
                                     source_data->update_host);
      LG_log(lg_ctx, LG_DEBUG, "up_set_current_source_data: UPDATE_PORT is: [%d]", 
                                     source_data->update_port);
      LG_log(lg_ctx, LG_DEBUG, "up_set_current_source_data: QUERY_HOST is: [%s]",  
                                     source_data->query_host);
      LG_log(lg_ctx, LG_DEBUG, "up_set_current_source_data: QUERY_PORT is: [%d]",  
                                     source_data->query_port);   
      LG_log(lg_ctx, LG_DEBUG, "up_set_current_source_data: DBhost is: [%s]", 
                                     source_data->DBhost);
      LG_log(lg_ctx, LG_DEBUG, "up_set_current_source_data: DBport is: [%d]",  
                                     source_data->DBport);   
      LG_log(lg_ctx, LG_DEBUG, "up_set_current_source_data: DBname is: [%s]", 
                                     source_data->DBname);
      LG_log(lg_ctx, LG_DEBUG, "up_set_current_source_data: DBuser is: [%s]", 
                                     source_data->DBuser);
      LG_log(lg_ctx, LG_DEBUG, "up_set_current_source_data: DBpasswd is: [%s]", 
                                     source_data->DBpasswd);
      break;
    }
  }

  if ( found == UP_FAIL )
  {
    RT_invalid_source(rt_ctx);
    LG_log(lg_ctx, LG_ERROR,"up_set_current_source_data: source in object not recognised [%s]", obj_source);
  }

  LG_log(lg_ctx, LG_FUNC,"<up_set_current_source_data: exiting with value %s", UP_ret2str(found));
  return found;
}



/* gets the source from the object and sets up the current source values
   Receives RT context
            LG context
            options structure
            parsed object
            ptr to source string
            source data structure pointer
   Returns  UP_OK for success and UP_FAIL for source not found 
            source string
            source data in structure
*/

int up_get_source(RT_context_t *rt_ctx, LG_context_t *lg_ctx,
                            options_struct_t *options,
                            rpsl_object_t *object, char **obj_source, 
                            source_data_t *source_data)
{
  int retval;
  GList *attr_list = NULL;

  LG_log(lg_ctx, LG_FUNC,">up_get_source: entered ");

  /* find source from object */
  attr_list = rpsl_object_get_attr(object, "source");
  if ( attr_list )
    *obj_source = rpsl_attr_get_clean_value((rpsl_attr_t *)(attr_list->data));

  if ( ! attr_list || ! (*obj_source) )
  {
    /* should only happen if the object has no source: attribute or the attribute has no value, 
       but the parser should reject that case */
    LG_log(lg_ctx, LG_ERROR,"up_get_source: no source found");
    retval = UP_FAIL;
  }
  else
  {
    /* set up current source data values */
    retval = up_set_current_source_data(rt_ctx, lg_ctx, options, (*obj_source), source_data);
    if ( retval != UP_OK )
    {
      free(*obj_source);
      *obj_source = NULL;
    }
    rpsl_attr_delete_list(attr_list);
  }

  LG_log(lg_ctx, LG_FUNC,"<up_get_source: exiting returning %s", UP_ret2str(retval));
  return retval;
}



/*  Function to rewrite a line of text with only one blankspace between each word.
    remove any tab characters.
   Receives destination of string
            source of string
   Returns  sets new string in destination
*/

void up_string_pack(char *dest, const char *source)
{

/*
 * This while loop continues until the NULL character is copied into
 * the destination string.  If a tab character is copied into the
 * destination string, it is replaced with a blank-space character.
 *
 * Multiple blank-space and/or tab characters are skipped in the source
 * string until any other character is found.
 */

  while (1)
  {
    /* copy the next character, but replace tab with space */
    *dest = (*source == '\t') ? ' ' : *source;

    /* Exit if have copied the end of the string. */
    if (*dest == '\0')
      return;

    ++dest;
    if ( (*source == ' ') || (*source == '\t') )
    {
      /* the character just copied was a space or tab */
      ++source;
      /* skip past any sebsequent space or tabs until we find any other character */
      while ( (*source == ' ') || (*source == '\t') )
        ++source;
    }
    else
      ++source;
  }
}



/* replaces the erase_str occurences with insert_str in str 
   Receives string
            erase string
            replace string
   Returns  new string
*/

char * up_replace_strings(char *str, const char *erase_str, const char *insert_str)
{
  GString *g_str;  
  int pos;
  char *result_str;
  
  /* if erase_str is NULL return a copy of input string */
  if (erase_str == NULL)
    return strdup(str);
  
  g_str = g_string_new(str);
  
  /* replace erase_str with insert_str */
  while (strstr(g_str->str, erase_str) != NULL)
  {
    pos = strstr(g_str->str, erase_str) - g_str->str;
    g_str = g_string_erase(g_str, pos, strlen(erase_str));
    if(insert_str != NULL)
      g_str = g_string_insert(g_str, pos, insert_str);
  }

  /* save the result string */
  result_str = strdup(g_str->str);

  /* free the GString structure (NULL means 'don't free the char string') */
  g_string_free(g_str, 0);

  return result_str;
}

/* used by g_list_foreach to delete the objects in a list 
   Receives object pointer
            NULL
   Returns  none
*/

void up_rpsl_object_delete(gpointer data, gpointer user_data)
{
    rpsl_object_delete((rpsl_object_t *)data);
}


/* checks if two objects are identical or not.
    Takes two objects, one as char *, the other as
	a parsed object, 

    Algorithm is very simple: All strings of tabs and 
    white spaces are collapsed into a single white space,
    and then the strings are compared (strcmp) 
   Receives LG context
            old object string
            parsed object
   Returns  1 for identical and 0 for different
*/

int up_identical(LG_context_t *lg_ctx, const char *old_version, rpsl_object_t *object)
{
  char *arg[2];
  int str_idx;
  rpsl_object_t *object2;
  rpsl_error_t error;
  int result = 0;
  char *temp_str[2]; 
  char *temp;

  LG_log(lg_ctx, LG_FUNC,">up_identical: entered");
  
  arg[0] = strdup(old_version);
  object2 = rpsl_object_copy(object);
  rpsl_object_remove_attr_name(object2, "delete", &error);
  rpsl_object_remove_attr_name(object2, "override", &error);
  arg[1] = rpsl_object_get_text(object2,RPSL_STD_COLUMN);
  rpsl_object_delete(object2);

  for ( str_idx=0; str_idx<2; str_idx++ )
  {
    arg[str_idx] = g_strstrip(arg[str_idx]);

    temp_str[str_idx] = (char *)malloc(strlen(arg[str_idx]) + 1); 
    up_string_pack(temp_str[str_idx], arg[str_idx]);

    /* if there are still \r's at the end of strings, remove them */
    if ((temp_str[str_idx][strlen(temp_str[str_idx]) - 1]) == '\r')
      temp_str[str_idx][strlen(temp_str[str_idx]) - 1] = '\0';

    /* there may be white spaces at the end of the strings now, remove them */
    if ((temp_str[str_idx][strlen(temp_str[str_idx]) - 1]) == ' ')
      temp_str[str_idx][strlen(temp_str[str_idx]) - 1] = '\0';

    /* remove the white space just before the EOL (since this is not taken care of by
       the up_string_pack func) */
    temp = up_replace_strings(temp_str[str_idx], " \n", "\n");
    free(temp_str[str_idx]);
    temp_str[str_idx] = temp;
    free(arg[str_idx]);
  }

  LG_log(lg_ctx, LG_DEBUG,"up_identical: length of object [%d] old object [%d]",
                  strlen(temp_str[1]), strlen(temp_str[0]) );
  LG_log(lg_ctx, LG_DEBUG,"up_identical: compressed object [\n%s]\n\ncompressed old object [\n%s]",
                  temp_str[1], temp_str[0] );
  result = strcmp(temp_str[0], temp_str[1]);

  free(temp_str[0]);
  free(temp_str[1]);

  LG_log(lg_ctx, LG_FUNC,"<up_identical: exiting with result [%s]",  result ? "not identical" : "identical");
  return ! result;
}



/* gets the object and old object (if exists) and determines the operation to be carried out
   checks for some operation dependent error conditions
   Receives RT context
            LG context
            options structure
            parsed object
            object string
            old object string
            old object
            pointer to operation
   Returns  UP_OK for success
            UP_FAIL for error
            operation
*/

int up_determine_operation(RT_context_t *rt_ctx, LG_context_t *lg_ctx,
                            options_struct_t *options, rpsl_object_t *object,
                            char *old_object_str, rpsl_object_t *old_obj,
                            int *operation)
{
  int retval = UP_OK;
  const char *type;

  LG_log(lg_ctx, LG_FUNC,">up_determine_operation: entered");

  /* determine operation */
  if ( rpsl_object_is_deleted(object) )
    *operation = UP_DELETE;
  else if ( old_object_str )
    *operation = UP_MODIFY;
  else
    *operation = UP_CREATE;
  LG_log(lg_ctx, LG_DEBUG,"up_determine_operation: operation is [%s]", 
                                      UP_op2str(*operation));

  /* check for operation dependent errors */
  type = rpsl_object_get_class(object);
  if ( *operation == UP_DELETE && ! old_object_str )
  {
    /* this is a delete operation but the object does not exist in the database */
    LG_log(lg_ctx, LG_ERROR,"up_determine_operation: delete operation, but object does not exist in the database");
    RT_non_existent_object(rt_ctx);
    retval = UP_FAIL;
  }
  else if ( *operation == UP_DELETE && ! up_identical(lg_ctx, old_object_str, object) )
  {
    /* this is a delete operation but the object is not identical to the one that 
       exists in the database */
    LG_log(lg_ctx, LG_ERROR,"up_determine_operation: object for deletion is not identical to the one in the database");
    RT_versions_dont_match(rt_ctx, old_obj);
    retval = UP_FAIL;
  }
  else if ( *operation == UP_MODIFY && up_identical(lg_ctx, old_object_str, object) )
  {
    /* this is a modify operation but the object is identical to the one that
       exists in the database */    
    LG_log(lg_ctx, LG_DEBUG,"up_determine_operation: object for modify is identical to the one in the database, no operation actioned");
    RT_versions_match(rt_ctx);
    *operation = UP_NOOP_OP;
    retval = UP_NOOP;
  }

  if ( options->enforced_new && old_object_str )
  {
    /* the enforced new keyword was specified 
    but this object already exists in the database */
    LG_log(lg_ctx, LG_ERROR,"up_determine_operation: enforced new set, but object already exists in the database");
    RT_enforcednew_exists(rt_ctx);
    /* set operation to create to generate correct summary in ack message */
    *operation = UP_CREATE;
    retval = UP_FAIL;
  }

  LG_log(lg_ctx, LG_FUNC,"<up_determine_operation: exiting with value %s", UP_ret2str(retval));
  return retval;
}



/* Performs external checks
   Receives RT context
            LG context
            parsed object
   Returns  UP_OK for success
            UP_FAIL for error
*/

int up_external_checks(RT_context_t *rt_ctx, LG_context_t *lg_ctx,
                            rpsl_object_t *object, char *object_str )
{
  int retval = UP_OK;
  
  /* currently there are no external checks to be done */
  
  return retval;
}



/* Prepares the object for commiting to the database.
   Some of the checks may alter the data in the parsed object.
   All checks are performed, even if some fail, then error reports
   can be made to include multiple errors if necessary. So the
   return value retval must |= calls to various functions.
   Receives RT context
            LG context
            options structure
            key_info pointer
            parsed object
            operation
            auto_nic pointer
            object source
   Returns  UP_OK if successful
            UP_FAIL otherwise
*/

int up_pre_process_object(RT_context_t *rt_ctx, LG_context_t *lg_ctx,
                           options_struct_t *options, key_info_t *key_info,
                           rpsl_object_t *preproc_obj,int operation,
                           char *auto_nic, char *obj_source)
{
  int retval = UP_OK;
  int key_status;
  const char *type = NULL;
  int str_idx, ctry_idx;
  char *country_str = NULL;
  char *countries[COUNTRY_LIST_SIZE];
  char **temp_vector;
  KM_key_return_t *key_data = NULL;

  LG_log(lg_ctx, LG_FUNC,">up_pre_process_object: entered");

  /* Perform all checks to report any errors.
     All functions called return 0 (UP_OK) for success, non zero for error.
     All function calls must do an |= on return value.
     For delete operations we omit some check. As long as the object is identical
     we will allow the deletion to reduce the legacy objects with wrong formats. */


  if ( operation != UP_DELETE )
  {
    /* get country details from config file */
    LG_log(lg_ctx, LG_INFO,"up_pre_process_object: get country codes from config file");
    country_str = ca_get_country;
    /* construct countries array from country string variable */
    temp_vector = g_strsplit(country_str, "\n", 0);
    for (str_idx=0, ctry_idx=0; temp_vector[str_idx] != NULL; str_idx++)
    {
      temp_vector[str_idx] = g_strstrip(temp_vector[str_idx]);
      if (strlen(temp_vector[str_idx]) > 0)
	  {
        countries[ctry_idx] = strdup(temp_vector[str_idx]);
        g_strup(countries[ctry_idx++]);
      }
    }
    countries[ctry_idx] = NULL; /* mark the end of array */
    g_strfreev(temp_vector);
    free(country_str);
    LG_log(lg_ctx, LG_DEBUG,"up_pre_process_object: number of countries [%d]", ctry_idx);

    retval |= UP_check_country_attr(rt_ctx, lg_ctx, preproc_obj, countries);

    retval |= UP_check_nicsuffixes(rt_ctx, lg_ctx, options, preproc_obj, countries);

    /* free the countries list */
    for (ctry_idx=0; countries[ctry_idx] != NULL; ctry_idx++)
      free(countries[ctry_idx]);

    retval |= UP_check_changed_attr(rt_ctx, lg_ctx, preproc_obj);

    retval |= UP_check_filter_set_object(rt_ctx, lg_ctx, preproc_obj);

    retval |= UP_check_peering_set_object(rt_ctx, lg_ctx, preproc_obj);

    /* check for references to AUTO- nic handles */
    if ( ! retval && UP_has_ref_to_AUTO_nic_hdl(lg_ctx, preproc_obj) )
    {
      /* replace references to auto nic handles with the already assigned nic handles */
      retval |= UP_replace_refs_to_AUTO_nic_hdl(rt_ctx, lg_ctx, options, preproc_obj);
    }

    /* check if object has any AUTO- nic handles */
    if ( ! retval && UP_has_AUTO_nic_hdl(lg_ctx, preproc_obj) )
    {
      /* the object has an AUTO- nic hdl */
      LG_log(lg_ctx, LG_DEBUG,"up_pre_process_object: object contains AUTO- nic handles");

      /* replace auto nic handle with special code for ripupdate */
      retval |= UP_replace_AUTO_nic_hdl(rt_ctx, lg_ctx, options, preproc_obj, 
                                           auto_nic, obj_source);
    }
  }
  
  type = rpsl_object_get_class(preproc_obj);
  if ( ! strcasecmp(type, "key-cert") )
  {
    /* special processing needed for key-cert objects */
    if ( operation != UP_DELETE )
    {
      /* create the generated attributes for a 
         create or modify key-cert object */
      retval |= UP_generate_keycert_attrs(rt_ctx, lg_ctx, key_info, preproc_obj);
    }
    else
    {
      /* we still need the key_id for delete */
      key_status = UP_get_key_data(rt_ctx, lg_ctx, key_info, &key_data, preproc_obj);
      KM_key_return_free(key_data);
      if ( key_status != KM_OK )
      {
        retval |= UP_FAIL;
      }
    }
  }

  LG_log(lg_ctx, LG_FUNC,"<up_pre_process_object: exiting with value %s", UP_ret2str(retval));
  return retval;
}



/* Performs transactions on any external data repositories
   Currently only the public keyring.
   Receives RT context
            LG context
            key_info pointer
            parsed object
            operation
   Returns  UP_OK if successful
            UP_FAIL otherwise
*/

int up_external_transact(RT_context_t *rt_ctx, LG_context_t *lg_ctx,
                         key_info_t *key_info, rpsl_object_t *preproc_obj,
                         int operation)
{
  int retval = UP_OK;
  const char *type;
  KM_status_t KM_status;

  LG_log(lg_ctx, LG_FUNC,">up_external_transact: entered with operation [%s]", UP_op2str(operation));

  type = rpsl_object_get_class(preproc_obj);
  if ( ! strcasecmp(type, "key-cert") )
  {
    /* A bit of a fudge here. When is a commit not a commit?
       Perform transaction on the public keyring using a copy of the keyring 
       So it is actually a commit, but the keyring is not changed.
       The commit later will simply mv the copy to the keyring. */
    LG_log(lg_ctx, LG_INFO,"up_external_transact: commiting to copy of public keyring");
    if ( operation == UP_CREATE )
    {
      KM_status = KM_key_add(KM_PGP, key_info->key);
      if ( KM_status != KM_OK )
      {
        LG_log(lg_ctx, LG_ERROR,"up_external_transact: key add failed");
        RT_key_add_error(rt_ctx, KM_status);
        LG_log(lg_ctx, LG_ERROR,"up_external_transact: return status %s", KM_return_string(KM_status));
        retval = UP_FAIL;
      }
    }
    else if ( operation == UP_MODIFY )
    {
      KM_status = KM_key_modify(KM_PGP, key_info->key_id, key_info->key);
      if ( KM_status != KM_OK )
      {
        LG_log(lg_ctx, LG_ERROR,"up_external_transact: key modify failed");
        RT_key_modify_error(rt_ctx, KM_status);
        LG_log(lg_ctx, LG_ERROR,"up_external_transact: return status %s", KM_return_string(KM_status));
        retval = UP_FAIL;
      }
    }
    else
    {
      KM_status = KM_key_remove(KM_PGP, key_info->key_id);
      if ( KM_status != KM_OK )
      {
        LG_log(lg_ctx, LG_ERROR,"up_external_transact: key remove failed");
        RT_key_remove_error(rt_ctx, KM_status);
        LG_log(lg_ctx, LG_ERROR,"up_external_transact: return status %s", KM_return_string(KM_status));
        retval = UP_FAIL;
      }
    }
  }
  
  LG_log(lg_ctx, LG_FUNC,"<up_external_transact: exiting with value %s", UP_ret2str(retval));
  return retval;
}



/* Commits to any external data repositories
   Currently only the public keyring.
   Receives RT context
            LG context
            options structure
            parsed object
   Returns  UP_OK if successful
            UP_FAIL otherwise
*/

int up_external_commit(RT_context_t *rt_ctx, LG_context_t *lg_ctx,
                           options_struct_t *options,
                           rpsl_object_t *preproc_obj)
{
  int retval = UP_OK;
  const char *type;
  KM_status_t KM_status;

  LG_log(lg_ctx, LG_FUNC,">up_external_commit: entered");

  type = rpsl_object_get_class(preproc_obj);
  if ( ! strcasecmp(type, "key-cert") )
  {
    LG_log(lg_ctx, LG_INFO,"up_external_commit: commiting to public keyring (mv copy)");
    KM_status = KM_commit(KM_PGP);
    if ( KM_status != KM_OK )
    {
      /* any error on the commit is serious */
      LG_log(lg_ctx, LG_FATAL,"up_external_commit: status return from keyring commit [%s]", KM_return_string(KM_status));
      UP_internal_error(rt_ctx, lg_ctx, options, "up_external_commit: ERROR commiting to public key ring\n");
    }
  }
  
  LG_log(lg_ctx, LG_FUNC,"<up_external_commit: exiting with value %s", UP_ret2str(retval));
  return retval;
}



/* Rollback transactions for any external data repositories
   Currently only the public keyring.
   Receives RT context
            LG context
            options structure
            parsed object
   Returns  UP_OK if successful
            UP_FAIL otherwise
*/

int up_external_rollback(RT_context_t *rt_ctx, LG_context_t *lg_ctx,
                           options_struct_t *options,
                           rpsl_object_t *preproc_obj)
{
  int retval = UP_OK;
  const char *type;
  KM_status_t KM_status;

  LG_log(lg_ctx, LG_FUNC,">up_external_rollback: entered");

  type = rpsl_object_get_class(preproc_obj);
  if ( ! strcasecmp(type, "key-cert") )
  {
    LG_log(lg_ctx, LG_INFO,"up_external_rollback: rollback transaction for public keyring (don't mv copy)");
    KM_status = KM_rollback(KM_PGP);
    if ( KM_status != KM_OK )
    {
      /* any error on the rollback is serious */
      LG_log(lg_ctx, LG_FATAL,"up_external_rollback: status return from keyring commit [%s]", KM_return_string(KM_status));
      UP_internal_error(rt_ctx, lg_ctx, options, "up_external_rollback: ERROR on rollback for public key ring\n");
    }
  }
  
  LG_log(lg_ctx, LG_FUNC,"<up_external_rollback: exiting with value %s", UP_ret2str(retval));
  return retval;
}



/* Obtains a transaction ID for an object to be sent to ripupd
   Receives RT context
            LG context
            options structure
            source data structure
   Returns  new transaction id
*/

long up_get_transaction_id(RT_context_t *rt_ctx, LG_context_t *lg_ctx,
                           options_struct_t *options, source_data_t *source_data)
{
  SQ_connection_t *sql_connection;
  SQ_result_set_t *result;
  int error;
  long new_id;

  LG_log(lg_ctx, LG_FUNC,">up_get_transaction_id: entered");

  sql_connection = SQ_get_connection(source_data->DBhost, source_data->DBport,
                                        source_data->DBname, source_data->DBuser, 
                                        source_data->DBpasswd); 
  if (!sql_connection)
  {
    /* No SQL connection is fatal */
    LG_log(lg_ctx, LG_FATAL,"up_get_transaction_id: No SQL connection");
    UP_internal_error(rt_ctx, lg_ctx, options, "up_get_transaction_id: ERROR No SQL connection\n");
  }
  error = SQ_execute_query(sql_connection, "INSERT INTO tid VALUES(NULL)", &result);
  if (error)
  {
    /* query fail is fatal */
    LG_log(lg_ctx, LG_FATAL,"up_get_transaction_id: Execute query failed [%s]", SQ_error(sql_connection));
    UP_internal_error(rt_ctx, lg_ctx, options, "up_get_transaction_id: ERROR Execute query failed\n");
  }

  new_id = mysql_insert_id(sql_connection);

  SQ_close_connection(sql_connection);  

  LG_log(lg_ctx, LG_FUNC,"<up_get_transaction_id: exiting with new id [%ld]", new_id);
  return new_id;
}



/* Extracts the error no from the string returned from RIPupd.
   Receives RT context
            LG context
            options structure
            result string from ripupd
            pointer to conection error flag
   Returns  error number
*/
   
int up_get_error_num(RT_context_t *rt_ctx, LG_context_t *lg_ctx,
                        options_struct_t *options,
                        const char *string, int *connect_err)
{
  char error_no[10];
  char **temp = NULL;
  int line_idx, char_idx, num_idx;
  int err = 0;

  LG_log(lg_ctx, LG_FUNC,">up_get_error_num: entered, ripupd_result [%s]", 
                                string ? string : "");
     
  /* if the string is NULL or empty, fatal error */
  if (string == NULL || strlen(string) == 0)
  {
    LG_log(lg_ctx, LG_FATAL,"up_get_error_num: ripupd_result string is empty");
    UP_internal_error(rt_ctx, lg_ctx, options, "up_get_error_num: NULL return from ripupdate\n");
  }

  /* split the string into lines */
  temp = g_strsplit(string , "\n", 0);
  num_idx = 0;
  for (line_idx = 0; temp[line_idx] != NULL; line_idx++)
  {
    if ( strstr(temp[line_idx], "%ERROR") == temp[line_idx]  )
    {
      /* Skip past %ERROR and look for the number following */
      LG_log(lg_ctx, LG_DEBUG,"up_get_error_num: error number line found\n%s", temp[line_idx]);
      for ( char_idx=strlen("%ERROR"); temp[line_idx][char_idx] != '\0' && num_idx<9; char_idx++ )
      {
        if ( isdigit((int)temp[line_idx][char_idx]) )
        {
          /* save the digits */
          error_no[num_idx++] = temp[line_idx][char_idx];
        }
        else if ( num_idx )
        {
          /* first non digit AFTER the number */
          break;
        }
        else
        {
          if ( temp[line_idx][char_idx] == ':' )
          {
            /* a : before the number means this is a conection type error */
            *connect_err = 1;   
          }
        }
      }
    }
  }
  g_strfreev(temp);

  if ( ! num_idx )
  {
    /* we have not found an error number */
    LG_log(lg_ctx, LG_FATAL,"up_get_error_num: ripupd_result string has no error number");
    UP_internal_error(rt_ctx, lg_ctx, options, "up_get_error_num: ripupd_result string has no error number\n");
  }

  error_no[num_idx] = '\0';
  LG_log(lg_ctx, LG_DEBUG,"up_get_error_num: error number string %s", error_no);
  err = atoi(error_no);

  LG_log(lg_ctx, LG_FUNC,"<up_get_error_num: exiting with error num %d", err);
  return err;
}

/* Interprets the result string coming from RIPupd
   Gets the error number. If not 0 (success) finds the error message(s) and passes them
   to RT without processing them.
   assigned_NIC is filled in if this is a person/role creation with AUTO nic hdl 
   assigned_NIC must be allocated enough memory before up_interpret_ripudb_result is called 
   If the caller does not expect a NIC hdl back, then assigned_NIC MUST be NULL

   Receives RT context
            LG context
            options structure
            result string from ripupd
            pointer to assigned_nic
   Returns  UP_OK if successful
            UP_FAIL otherwise
            sets assigned_NIC
*/

int up_interpret_ripudb_result(RT_context_t *rt_ctx, LG_context_t *lg_ctx, 
                                  options_struct_t *options,
                                  char *ripupd_result, char *assigned_nic )
{
  int retval = UP_OK;
  int err = 0;
  int line_idx;
  int connect_err = 0;
  char *pos = NULL;
  char *error_str = NULL;
  char **temp = NULL;
  char *temp_str;

  LG_log(lg_ctx, LG_FUNC,">up_interpret_ripudb_result: entered, ripupd_result [%s]", ripupd_result);

  /* Get the error number from the return string.
     0 means no error in this context */
  err = up_get_error_num(rt_ctx, lg_ctx, options, ripupd_result, &connect_err);

  if ( ! err && assigned_nic != NULL)
  { 
    /* if the update was successful and the caller of the function expected 
       to get a NIC handle, get the nic-hdl from the returned message */
//    retval = UP_get_assigned_nic(rt_ctx, lg_ctx, options, ripupd_result, assigned_nic);
    retval = get_assigned_nic(rt_ctx, lg_ctx, options, ripupd_result, assigned_nic);
  }
  else if ( err )
  {
/**************************************************************************************/    
/*                     TEMPORARY CODE TO USE EXISTING RIPUPD                          */
  
    /* According to the error no got from RIPupdate, construct an error string  */
    switch (err)
    {
      case ERROR_U_MEM: 
      case ERROR_U_DBS:
      case ERROR_U_BADOP:
      case ERROR_U_COP:
      case ERROR_U_NSUP:
      case ERROR_U_BUG:
        temp_str = (char *)malloc(1024);
        snprintf(temp_str, 1024, "***Error: Please contact database admin: Error no %i", err);
        LG_log(lg_ctx, LG_DEBUG,"up_interpret_ripudb_result: error string [%s]", temp_str);
        break; 
      case ERROR_U_OBJ:
        temp_str = (char *)malloc(1024);
        /* if the object contains refs to unknown objects */
        if (strstr(ripupd_result, "dummy") != NULL || 
              strstr(ripupd_result, "reference cannot be resolved") != NULL )
        {
    	  /* if the response from RIPupd contains "dummy not allowed" string 
             or a reference that cannot be resolved */
    	  snprintf(temp_str, 1024, "***Error: Unknown object referenced");
        }
	    else if (strstr(ripupd_result, "key-cert") != NULL)
	    {
        /* if the response from RIPupd contains "no key-cert object" string */
        snprintf(temp_str, 1024, "***Error: Unknown key-cert object referenced"); 
        }
        else
        {
        /* then, the object is referenced from other objects */
    	  snprintf(temp_str, 1024, "***Error: Object is referenced from other objects");
        }
        LG_log(lg_ctx, LG_DEBUG,"up_interpret_ripudb_result: error string [%s]", temp_str);
        break; 
      case ERROR_U_AUT:
        temp_str = (char *)malloc(1024);
        snprintf(temp_str, 1024, "***Error: Membership authorisation failure");
        LG_log(lg_ctx, LG_DEBUG,"up_interpret_ripudb_result: error string [%s]", temp_str);
        break; 
      default: 
        temp_str = (char *)malloc(1024);
        snprintf(temp_str, 1024, "***Error: Please contact database admin: Error no %i", err);
        LG_log(lg_ctx, LG_DEBUG,"up_interpret_ripudb_result: error string [%s]", temp_str);
        break; 
    }

    RT_RIP_update_error(rt_ctx, temp_str);
  
goto temp_skip;  
/**************************************************************************************/    
    /* find the error message following the colon(:) 
       there may be more than one error message line in the reply */
    /* split the string into lines */
    temp = g_strsplit(ripupd_result , "\n", 0);
    for (line_idx = 0; temp[line_idx] != NULL; line_idx++)
    {
      if ( strstr(temp[line_idx], "E") == temp[line_idx] ||
            strstr(temp[line_idx], "W") == temp[line_idx] ||
            strstr(temp[line_idx], "I") == temp[line_idx] ||
            (strstr(temp[line_idx], "%ERROR") == temp[line_idx] && connect_err) )
      {
        /* Look for the colon(:) */
        pos = strchr(temp[line_idx], ':');
        if ( pos && connect_err )
        {
          /* connection error messages have a different format, eg
             %ERROR:406: not authorized to update the database
             so find the next colon(:) */
          temp_str = pos+1;
          pos = strchr(temp_str, ':');
        }
        if ( pos )
        {
          /* colon(:) found, the error message follows */
          if ( ! error_str )
          {
            error_str = malloc(strlen(pos + 1) +2);
            strcpy(error_str, pos + 1);
            strcat(error_str, "\n");
          }
          else
          {
            error_str = realloc(error_str, strlen(error_str) + strlen(pos + 1) +2);
            strcat(error_str, pos + 1);
            strcat(error_str, "\n");
          }
        }
        else
        {
          /* there is no error message, or the colon was missing,
             this should not happen, but just in case */
          LG_log(lg_ctx, LG_FATAL, "up_interpret_ripudb_result: no :error message returned from ripupdate");
          UP_internal_error(rt_ctx, lg_ctx, options, "up_interpret_ripudb_result: no :error message returned from ripupdate\n");
        }
      }
    }
    RT_RIP_update_error(rt_ctx, error_str);
    free(error_str);
    g_strfreev(temp);
/**************************************************************************************/    
temp_skip:
/**************************************************************************************/    
    retval = UP_FAIL;
  }

  LG_log(lg_ctx, LG_FUNC,"<up_interpret_ripudb_result: exiting with value %s", UP_ret2str(retval));
  return retval;
}



/* Opens a socket to ripupd, sends the object,
   receives the reply and closes the socket. 
   operation is the UP integer value. This is converted to the ripupdate string
   'ADD' ,'DEL' or 'UPD' by the macro UP_op2ripopstr(operation)
   Receives RT context
            LG context
            options structure
            source data structure
            parsed object
            operation
            pointer to result_string
   Returns  UP_OK if successful
            UP_FAIL otherwise
            sets result_string
*/

int up_send_object_db(RT_context_t *rt_ctx, LG_context_t *lg_ctx,
                           options_struct_t *options,
                           source_data_t *source_data,
                           rpsl_object_t *preproc_obj, 
                           int operation, char **result_string )
{
  int retval = UP_OK;
  int sockfd, numbytes;  
  char buf[DB_MAXDATASIZE + 1];
  char *to_be_sent = NULL;
  long tr_id;
  char *tr_id_str;
  sk_conn_st condat;
  rpsl_object_t *send_obj;
  rpsl_error_t return_error;
  
  LG_log(lg_ctx, LG_FUNC,">up_send_object_db: entered, operation [%s]", UP_op2str(operation));
  
  /* get the object text */
  if ( operation == UP_DELETE )
  {
    /* remove the DELETE: attribute */
    send_obj = rpsl_object_copy(preproc_obj);
    rpsl_object_remove_attr_name(send_obj,"delete",&return_error);
    to_be_sent = rpsl_object_get_text(send_obj,0);
    rpsl_object_delete(send_obj);
  }
  else
    to_be_sent = rpsl_object_get_text(preproc_obj,0);
  LG_log(lg_ctx, LG_DEBUG,"up_send_object_db: object to be sent [\n%s]", to_be_sent);

  /* get the transaction ID, to be sent to RIPupdate*/
  tr_id = up_get_transaction_id(rt_ctx, lg_ctx, options, source_data);
  /* convert it into a string */
  tr_id_str = (char *)malloc(64);
  sprintf(tr_id_str, "%ld", tr_id);
  LG_log(lg_ctx, LG_DEBUG,"up_send_object_db: transaction ID [%s]", tr_id_str);

  /* open a connection to ripupdate */
  retval = SK_connect(&sockfd, source_data->update_host,
                            source_data->update_port, RIP_TIMEOUT);
  if ( retval != SK_OK )
  {
    /* any error on connection to ripupdate is serious */
    LG_log(lg_ctx, LG_FATAL,"up_send_object_db: status return from socket connect [%d]", retval);
    UP_internal_error(rt_ctx, lg_ctx, options, "up_send_object_db: ERROR on socket connect to ripupdate\n");
  }
  else
    retval = UP_OK;

  /* set up the connection data */
  SK_cd_make(&condat, sockfd, 0);

  /* send the data to ripupdate */
  if (SK_write(sockfd, UP_op2ripopstr(operation), strlen(UP_op2ripopstr(operation)), NULL, 0) == -1)
      goto UP_SEND_OBJECT_DB_ERROR;
  if (SK_write(sockfd, " ", strlen(" "), NULL, 0) == -1)
      goto UP_SEND_OBJECT_DB_ERROR;
  if (SK_write(sockfd, tr_id_str, strlen(tr_id_str), NULL, 0) == -1)
      goto UP_SEND_OBJECT_DB_ERROR;
  if (SK_write(sockfd, "\n\n" , 2, NULL, 0) == -1)
      goto UP_SEND_OBJECT_DB_ERROR;
  if (SK_write(sockfd, to_be_sent, strlen(to_be_sent), NULL, 0) == -1)
      goto UP_SEND_OBJECT_DB_ERROR;
  if (SK_write(sockfd, "\n\n", 2, NULL, 0)  == -1)
      goto UP_SEND_OBJECT_DB_ERROR;
  /* SK_write the ACK now */
  if (SK_write(sockfd, "ACK ",strlen("ACK "), NULL, 0)  == -1)
      goto UP_SEND_OBJECT_DB_ERROR;
  if (SK_write(sockfd, tr_id_str, strlen(tr_id_str), NULL, 0) == -1)
      goto UP_SEND_OBJECT_DB_ERROR;
  if (SK_write(sockfd, "\n\n", 2, NULL, 0)  == -1)
      goto UP_SEND_OBJECT_DB_ERROR;

  LG_log(lg_ctx, LG_INFO,"up_send_object_db: send complete");
      
  /* get the reply from ripupdate */
  while (( numbytes=SK_cd_gets(&condat, buf, DB_MAXDATASIZE) ) > 0)
  {
    buf[numbytes] = '\0';
    LG_log(lg_ctx, LG_DEBUG,"up_send_object_db: received from socket [\n%s]", buf);

    if (*result_string == NULL)
      *result_string = strdup(buf);
    else
    {
      *result_string = (char *)realloc(*result_string, 
    				     strlen(*result_string) + strlen(buf) + 1);
      *result_string = strcat(*result_string, buf);
    }

    if (strstr(*result_string,"\n\n") != NULL)
    { 
      /* if the result_string contains an empty line at the end, we will close */
      LG_log(lg_ctx, LG_DEBUG,"up_send_object_db: received blank line from socket, stop reading");
      break;
    };
  }

  SK_cd_free(&condat);
  SK_close(sockfd);
  free(to_be_sent);
  free(tr_id_str);

  if ( numbytes < 0 )
  {
    /* a read error from ripupdate is serious */
    LG_log(lg_ctx, LG_FATAL,"up_send_object_db: read error from ripupdate");
    LG_log(lg_ctx, LG_FATAL,"up_send_object_db: data received so far [%s]",
                             *result_string ? *result_string : "" );
    UP_internal_error(rt_ctx, lg_ctx, options, "up_send_object_db: ERROR reading from ripupdate\n");
  }

  LG_log(lg_ctx, LG_FUNC,"<up_send_object_db: exiting with value %s", UP_ret2str(retval));
  return retval;

  /* write error exit */
  UP_SEND_OBJECT_DB_ERROR:
    /* a write error to ripupdate is serious */
    LG_log(lg_ctx, LG_FATAL,"up_send_object_db: write error to ripupdate");
    UP_internal_error(rt_ctx, lg_ctx, options, "up_send_object_db: ERROR writing to ripupdate\n");
}



/* Commits to source data base
   Receives RT context
            LG context
            options structure
            source data structure
            parsed object
            operation
            pointer to assigned_nic
   Returns  UP_OK if successful
            UP_FAIL otherwise
*/

int up_source_commit(RT_context_t *rt_ctx, LG_context_t *lg_ctx,
                       options_struct_t *options,
                       source_data_t *source_data,
                       rpsl_object_t *preproc_obj,
                       int operation, char *assigned_nic)
{
  int retval = UP_OK;
  char *ripupd_result = NULL;

  LG_log(lg_ctx, LG_FUNC,">up_source_commit: entered with operation [%s]", UP_op2str(operation));
  LG_log(lg_ctx, LG_FUNC,">up_source_commit: the object is [%s]", rpsl_object_get_text(preproc_obj,0));

  /* send the object to the database */
  retval = up_send_object_db(rt_ctx, lg_ctx, options, source_data, preproc_obj, 
                                       operation, &ripupd_result);
  
  /* interpret the results returned by ripupdate */
  if ( retval == UP_OK )
    retval = up_interpret_ripudb_result(rt_ctx, lg_ctx, options,
                                             ripupd_result, assigned_nic);
  
  if ( ripupd_result )
    free(ripupd_result);
  
  /* if the commit was not successful then rollback any external transactions */
  if ( retval != UP_OK )
  {
    LG_log(lg_ctx, LG_DEBUG,"up_source_commit: commit failed, do external rollback");
    up_external_rollback(rt_ctx, lg_ctx, options, preproc_obj);
  }
  else
  {
    /* otherwise commit the external transactions */
    LG_log(lg_ctx, LG_DEBUG,"up_source_commit: commit succeeded, do external commit");
    up_external_commit(rt_ctx, lg_ctx, options, preproc_obj);
  }

  LG_log(lg_ctx, LG_FUNC,"<up_source_commit: exiting with value [%s]", UP_ret2str(retval));
  return retval;
}



/* Make any final changes to the object for reporting
   Receives RT context
            LG context
            options structure
            preproc object
            old object
            auto_nic (if it exists otherwise NULL)
            assigned nic (if it exists otherwise NULL)
            operation
            list of mntners used in auth
   Returns  UP_OK for success
            UP_FAIL for any error
*/

int up_post_process_object(RT_context_t *rt_ctx, LG_context_t *lg_ctx, 
                             options_struct_t *options, source_data_t *source_data,
                             rpsl_object_t *preproc_obj, rpsl_object_t *old_obj,
                             char *auto_nic, char *assigned_nic, int operation,
                             GList *mntner_used )
{
  int retval = UP_OK;
//  char *postproc_obj_str = NULL;
  rpsl_object_t *postproc_obj = NULL;

  LG_log(lg_ctx, LG_FUNC,">up_post_process_object: entered with assigned_nic [%s]", 
                            (assigned_nic && strlen(assigned_nic)) ? assigned_nic : "not set");

  postproc_obj = rpsl_object_copy(preproc_obj);

  /* check if this object has an AUTO- nic-hdl and replace it with the assigned one */
  if ( assigned_nic && strlen(assigned_nic) )
    retval = UP_put_assigned_nic(rt_ctx, lg_ctx, options, auto_nic, assigned_nic, postproc_obj);

//  postproc_obj_str = rpsl_object_get_text(postproc_obj,RPSL_STD_COLUMN);
//  assert( postproc_obj_str != NULL );      /* must have a valid object at this stage */
  RT_postproc_obj(rt_ctx, postproc_obj);

  /* processing of this object is completed */
  /* prepare notif messages */
  LG_log(lg_ctx, LG_DEBUG,"up_post_process_object: object processing complete, prepare notif messages");
  if ( operation == UP_DELETE )
    NT_write_all_ntfs(rt_ctx, lg_ctx, options, source_data, NULL, old_obj, mntner_used);
  else
    NT_write_all_ntfs(rt_ctx, lg_ctx, options, source_data, postproc_obj, old_obj, mntner_used);
//  free(postproc_obj_str);

  rpsl_object_delete(postproc_obj);
  LG_log(lg_ctx, LG_FUNC,"<up_post_process_object: exiting with value [%s]", UP_ret2str(retval));
  return retval;
}


/* Gets an object string from the input submission with it's associated credentials.
   Processes the object.
   Receives RT context
            LG context
            options structure
            object string
            list of credentials
   Returns  one of the UP return_codes
*/

int up_process_object(RT_context_t *rt_ctx, LG_context_t *lg_ctx, 
                           options_struct_t *options, char *object_str, 
                           GList *credentials)
{
  int retval;
  int au_retval;
  int operation = 0;
  int parsed_ok = 0;
  char *old_object_str = NULL;
  char *auto_nic = NULL;
  char *assigned_nic = NULL;
  char *obj_source = NULL;
  rpsl_object_t *object = NULL;
  rpsl_object_t *old_object = NULL;
  rpsl_object_t *preproc_obj = NULL;
  LU_server_t *current_server = NULL;
  GList *mntner_used = NULL;

  source_data_t source_data = {NULL, NULL, NULL, 0, 0, NULL, 0, NULL, NULL, NULL, NULL};
  key_info_t    key_info    = {NULL, NULL};

  LG_log(lg_ctx, LG_FUNC,">up_process_object: entered with object [\n%s]", object_str);

  /* initialise the lookup module */
  LU_init(lg_ctx);

  /* parse the object string */
  object = rpsl_object_init(object_str);
  RT_set_object(rt_ctx, object);
  if ( rpsl_object_has_error(object, RPSL_ERRLVL_ERROR) )
  {
    LG_log(lg_ctx, LG_DEBUG,"up_process_object: object parsed with errors");
    RT_syntax_error(rt_ctx);
    parsed_ok = 0;
  }
  else
  {
    LG_log(lg_ctx, LG_DEBUG,"up_process_object: object parsed OK");
    RT_syntax_ok(rt_ctx);
    parsed_ok = 1;
  }
      
  /* if it did NOT parse OK and it is NOT a delete then this is an error.
     if it did not parse ok and it is a delete it MAY be ok if it is a
     legacy object */
  if ( ! parsed_ok && ! rpsl_object_is_deleted(object) )
  {
    LG_log(lg_ctx, LG_DEBUG,"up_process_object: parse errors and not deletion");
    retval = UP_FAIL;
    operation = UP_SYNTAX_ERR;
    goto up_process_object_exit;
  }

  /* find source from object and set current source */
  retval = up_get_source(rt_ctx, lg_ctx, options, object, &obj_source, &source_data);
  if ( retval != UP_OK )
    goto up_process_object_exit;

  /* set up current server for lokups */
  current_server = LU_whois_init(source_data.query_host, source_data.query_port, UPDATE_QUERY_TIMEOUT);
  source_data.current_server = current_server;

  /* get old object, if there is one */
  retval = LU_get_object(current_server, &old_object, object, obj_source);
  if ( retval != LU_OKAY )
  {
    /* any lookup error is considered a fatal error */
    LG_log(lg_ctx, LG_FATAL,"up_process_object: lookup error");
    UP_internal_error(rt_ctx, lg_ctx, options, "up_process_object: lookup error\n");
  }
  if ( old_object )
    old_object_str = rpsl_object_get_text(old_object, RPSL_STD_COLUMN);

  /* determine the operation to be carried out on this object */
  retval = up_determine_operation(rt_ctx, lg_ctx, options, object,
                                   old_object_str, old_object, &operation);
  if ( retval != UP_OK )
    goto up_process_object_exit;

  /* perform a set of pre-processing operations and checks on the object */
  preproc_obj = rpsl_object_copy(object);
  /* set up storage location for the auto code sent to ripupdate  */
  auto_nic = (char *)calloc(AUTO_NIC_LENGTH, sizeof(char)); 
  retval = up_pre_process_object(rt_ctx, lg_ctx, options, &key_info, preproc_obj, 
                                    operation, auto_nic, obj_source);
  if ( retval != UP_OK )
  {
    LG_log(lg_ctx, LG_DEBUG,"up_process_object: pre-processing failed, checking for auth errors");
//    goto up_process_object_exit;
  }

  /* auth checks */
  /* Use object and NOT preproc_obj for auth checks in case of AUTO- nic-hdls 
     In preproc_obj AUTO- nic-hdls have been replaced by special place holders ready for
     sending to ripupdate and we don't want these displayed in the auth info messages */
  AU_set_lookup(current_server);
  au_retval = AU_check_auth(object, credentials, op2au_op(operation), 
                            rt_ctx, &mntner_used);

  if (au_retval != AU_AUTHORISED)
  {
    LG_log(lg_ctx, LG_DEBUG,"up_process_object: check auth returned %s", AU_ret2str(au_retval));
    if ( au_retval == AU_FWD )
    {
      /* This was a maintainer, irt or as-block creation request with no override.
         If we are not in test mode and no pre-processing errors were found,
         forward it to <HUMAILBOX>  */
      if ( retval == UP_OK && ! options->test_mode )
      {
        LG_log(lg_ctx, LG_DEBUG,"up_process_object: send forward create message");
        NT_forw_create_req(rt_ctx, lg_ctx, options, object, credentials);
        retval = UP_FWD;
        operation = UP_FWD_OP;
      }
    }
    /* otherwise auth failed */
    else if ( au_retval == AU_UNAUTHORISED_CONT )
    {
      /* the auth failed on this object, but we are able to ack this object
         and safely continue to process other objects in this update message */
      /* prepare forward messages */
      LG_log(lg_ctx, LG_DEBUG,"up_process_object: prepare forward messages");
      if ( operation == UP_DELETE )
        NT_write_all_frwds(rt_ctx, lg_ctx, options, &source_data, NULL, old_object, mntner_used);
      else
        NT_write_all_frwds(rt_ctx, lg_ctx, options, &source_data, object, old_object, mntner_used);
      retval = UP_FAIL;
    }
    else
    {
      /* the auth failed in a way that is not possible to continue to
         process any other objects in this message. 
         Report a fatal error */
      retval = UP_FATAL;
      LG_log(lg_ctx, LG_FATAL,"up_process_object: AU_check_auth fatal error");
      if (mntner_used)
      {
        g_list_foreach(mntner_used, up_rpsl_object_delete, NULL);
        g_list_free(mntner_used);
      }
      UP_internal_error(rt_ctx, lg_ctx, options, "up_process_object: AU_check_auth fatal error\n");
    }
  }
  /* if either pre-processing or auth failed, or processing is complete (UP_FWD) */
  if ( retval != UP_OK )
    goto up_process_object_exit;
  
  /* external checks */
  retval = up_external_checks(rt_ctx, lg_ctx, object, object_str);
  if ( retval != UP_OK )
    goto up_process_object_exit;

  /* if an AUTO- nic-hdl has been found set up storage location
     for the actual nic handle to be created by ripupdate */
  if ( strlen(auto_nic) )
    assigned_nic = (char *)calloc(AUTO_NIC_LENGTH, sizeof(char)); 
  
  /* perform transactions to external data repositories */
  retval = up_external_transact(rt_ctx, lg_ctx, &key_info, preproc_obj, operation);
  if ( retval != UP_OK )
    goto up_process_object_exit;

  /* Commit to source database */
  retval = up_source_commit(rt_ctx, lg_ctx, options, &source_data, preproc_obj, 
                                     operation, assigned_nic);
  if ( retval != UP_OK )
    goto up_process_object_exit;
  
  /* post process */
  retval =  up_post_process_object(rt_ctx, lg_ctx, options, &source_data, 
                                     preproc_obj, old_object, auto_nic, assigned_nic,
                                     operation, mntner_used );

  /* tidyup before returning retval */
  up_process_object_exit:
    RT_unset_object(rt_ctx, op2rt_upd_op(operation), 
                      (retval==UP_OK || retval==UP_NOOP || retval==UP_FWD) ? 1 : 0 );
    rpsl_object_delete(object);
    /* cleanup LU module */
    if ( current_server )
      LU_cleanup(current_server);
    if ( old_object )
      rpsl_object_delete(old_object);
    if ( old_object_str )
      free(old_object_str);
    if ( preproc_obj )
      rpsl_object_delete(preproc_obj);
    if ( auto_nic )
      free(auto_nic);
    if ( assigned_nic )
      free(assigned_nic);
    if ( obj_source )
      free(obj_source);
    if ( key_info.key )
      free(key_info.key);
    if ( key_info.key_id )
      free(key_info.key_id);
    if ( source_data.current_source )
      free(source_data.current_source);
    if ( source_data.query_host )
      free(source_data.query_host);
    if (mntner_used)
    {
      g_list_foreach(mntner_used, up_rpsl_object_delete, NULL);
      g_list_free(mntner_used);
    }
    LG_log(lg_ctx, LG_FUNC,"<up_process_object: exiting with value %s", UP_ret2str(retval));
    return retval;
}


/* Gets a submission from the input with it's associated credentials.
   Receives RT context
            LG context
            options structure
            submission
   Returns  one of the UP return_codes
*/

int UP_process_submission(RT_context_t *rt_ctx, LG_context_t *lg_ctx, 
                           options_struct_t *options,
                           EP_blob_credential_t *submission )
{
  int retval = UP_OK;
  char **split_lines = NULL;
  char *line = NULL;
  char *object_str = NULL;
  int line_idx, line_cnt;
  int valid_first_line = 0;
  int source_line_found = 0;
  int in_object = 0;
  int not_object = 0;
  char *new_data_str = NULL;
  const gchar *data_str = NULL;
  const GList *error_list = NULL;
  const GList *error_list_item = NULL;
  gint elevel, ecode;
  gchar *edescr = NULL;
  GList *credentials;
  GSList *list_of_AUTO_refd_objects = NULL;   /* for objects referencing an AUTO nic handle */
  GSList *item;
  rpsl_object_t *object = NULL;

  LG_log(lg_ctx, LG_FUNC,">UP_process_submission: entered");

  /* get the data and credentials from this submission */
  LG_log(lg_ctx, LG_INFO, "UP_process_submission: get submission data");
  data_str = EP_get_blob(submission);
  LG_log(lg_ctx, LG_DEBUG, "UP_process_submission: submission data [\n%s]", data_str);
  LG_log(lg_ctx, LG_INFO, "UP_process_submission: get submission credentials");
  credentials = EP_get_credentials(submission);
  
  /* add an extra blank line to the end of the data,
     just in case the last object was not followed by a blank line */
  new_data_str = strdup(data_str);
  new_data_str = realloc(new_data_str, strlen(new_data_str) + 2);
  strcat(new_data_str, "\n");
  
  /* split the new_data_str on \n */
  split_lines = g_strsplit(new_data_str, "\n", 0);
  
  /* process the lines in the submission */
  LG_log(lg_ctx, LG_INFO, "UP_process_submission: start to process the submission data");
  for (line_idx=0; split_lines[line_idx] != NULL; line_idx++)
  {
    line = split_lines[line_idx];  /* just to make the next few lines easier to read */
    /* remove '\n's and '\r' first */
    line = up_remove_EOLs(line);
    /* remove trailing white space */
    line = g_strchomp(line); 
    
    /* build up an object string */
    /* Whilst doing this check if it looks like an object --
       look for the first line starting with an alphabetic char and containing a colon before a white space
       look for a line starting with 'source:', but not the first line */

    if (strlen(line) != 0)
    {
      /* add this line to the current object string */
      if (object_str == NULL)
      {
        /* this is the first line of the current object string */
        /* check if it starts with an alphabetic char and 
           contains a colon (:) before any white spaces */
        if ( (*line >= 'a' && *line <= 'z') || (*line >= 'A' && *line <= 'Z') )
        {
          for ( line_cnt=0; line_cnt < strlen(line); line_cnt++ )
          {
            if ( line[line_cnt] == ' ' || line[line_cnt] == '\t'
                   || line[line_cnt] == '\r' )
              break;
            if ( line[line_cnt] == ':' )
            {
              LG_log(lg_ctx, LG_DEBUG, "UP_process_submission: valid_first_line = 1");
              valid_first_line = 1;
              break;
            }
          }
        }
        in_object = 1;
        object_str = (char *)malloc(strlen(line) + 2);
        strcpy(object_str, line);
        strcat(object_str, "\n"); /* add EOL again (we removed it before) */
      }
      else
      {
        object_str = (char *)realloc(object_str, strlen(object_str) + strlen(line) + 2);
        strcat(object_str, line);
        strcat(object_str, "\n"); /* add EOL again (we removed it before) */
		g_strdown(line);
        if ( strstr(line, "source:") == line )
        {
          LG_log(lg_ctx, LG_DEBUG, "UP_process_submission: source_line_found = 1");
          source_line_found = 1;
        }
      }
    }
    else
    {
      /* blank line found, end of object */
      if ( in_object )      /* allow for multiple blank lines */
      {
        LG_log(lg_ctx, LG_INFO, "UP_process_submission: end of object");
        LG_log(lg_ctx, LG_DEBUG, "UP_process_submission: object string [\n%s]", object_str);
        /* has it passed the test for looking like a real object */
        if ( valid_first_line && source_line_found )
        {
          not_object = 0;
          LG_log(lg_ctx, LG_DEBUG, "UP_process_submission: looks like an object");
          /* parse the object string */
	      object = rpsl_object_init(object_str);
          error_list = rpsl_object_errors(object);
          if (error_list)
          {
            LG_log(lg_ctx, LG_DEBUG,"UP_process_submission: errors found on initial object parse");
            for ( error_list_item = error_list; error_list_item != NULL; error_list_item = g_list_next(error_list_item) )
            {
              elevel = ((rpsl_error_t *)(error_list_item->data))->level;
              ecode = ((rpsl_error_t *)(error_list_item->data))->code;
              edescr = strdup(((rpsl_error_t *)(error_list_item->data))->descr);

              LG_log(lg_ctx, LG_DEBUG,"UP_process_submission: level [%d]  code [%d]  [%s]", elevel, ecode, edescr);
              if ( ecode == RPSL_ERR_ONLYCOMMENTS || ecode == RPSL_ERR_BADCLASS
                        ||ecode == RPSL_ERR_UNKNOWNCLASS )
              {
                /* does not look like an object after all */
                not_object = 1;
                LG_log(lg_ctx, LG_DEBUG,"UP_process_submission: This does not look like an object after all");
              }
              free(edescr);
            }
          }

          if ( not_object )
          {
            /* this does not look like an object */
            LG_log(lg_ctx, LG_DEBUG, "UP_process_submission: unparsable input");
            RT_unparsable_input(rt_ctx, object_str);
          }
          else
          {
            LG_log(lg_ctx, LG_DEBUG, "UP_process_submission: recognised object");
            if ( ! rpsl_object_has_error(object, RPSL_ERRLVL_ERROR) && UP_has_ref_to_AUTO_nic_hdl(lg_ctx, object) )
            {
		      /* object has a reference to an auto- nichdl and no syntax errors
		         so defer the processing to allow the auto- nichdl to be created first */
              /* put this object string on the list of objects referencing AUTO nic handles */
              LG_log(lg_ctx, LG_DEBUG, "UP_process_submission: put object on list_of_AUTO_refd_objects");
              list_of_AUTO_refd_objects = g_slist_append(list_of_AUTO_refd_objects, strdup(object_str));
            }
            else
            {
              /* process this object 
                 OR the return value so that we can keep track of any failures */
              retval |= up_process_object(rt_ctx, lg_ctx, options,
                                           object_str, credentials);

              /* do not continue to process objects after a fatal error */
              if (retval & UP_FATAL)
              {
                retval = UP_FATAL;  /* so that UP_ret2str macro works */
                /* free up memory */
                if ( list_of_AUTO_refd_objects )
                {
                  for ( item=list_of_AUTO_refd_objects; item != NULL; item=g_slist_next(item) )
                    free((char *)(item->data));
                  g_slist_free(list_of_AUTO_refd_objects);
                }
                free(object_str);
                g_strfreev(split_lines);
                free(new_data_str);
                LG_log(lg_ctx, LG_FUNC,"<UP_process_submission: exiting with value %s", UP_ret2str(retval));
                return retval;
              }

              /* keep a tally */
              if (retval == UP_OK)
                options->count_successful++;
		      else if (retval == UP_FAIL)
                options->count_unsuccessful++;
            }
          }
        }
        else
        {
          /* this does not look like an object */
          LG_log(lg_ctx, LG_DEBUG, "UP_process_submission: does not look like an object");
          RT_unparsable_input(rt_ctx, object_str);
        }

        /* reset flags */
        free(object_str);
        object_str = NULL;
        valid_first_line = 0;
        source_line_found = 0;
        in_object = 0;
      }
    }
  }
  /* end of submission */
  LG_log(lg_ctx, LG_INFO, "UP_process_submission: end of submission");
  g_strfreev(split_lines);
  free(new_data_str);

  /* now process the list of objects referencing AUTO nic handles */
  if ( list_of_AUTO_refd_objects )
  {
    LG_log(lg_ctx, LG_DEBUG, "UP_process_submission: process list_of_AUTO_refd_objects");
    for ( item=list_of_AUTO_refd_objects; item != NULL; item=g_slist_next(item) )
    {
      object_str = (char *)(item->data);
      LG_log(lg_ctx, LG_DEBUG, "UP_process_submission: object string [\n%s]", object_str);
      /* process this object
         OR the return value so that we can keep track of any failures */
      retval |= up_process_object(rt_ctx, lg_ctx, options,
                                   object_str, credentials);

      /* do not continue to process objects after a fatal error */
      if (retval & UP_FATAL)
      {
        retval = UP_FATAL;  /* so that UP_ret2str macro works */
        /* free up memory */
        for ( item=item; item != NULL; item=g_slist_next(item) )
          free((char *)(item->data));
        g_slist_free(list_of_AUTO_refd_objects);
        LG_log(lg_ctx, LG_FUNC,"<UP_process_submission: exiting with value %s", UP_ret2str(retval));
        return retval;
      }

      /* keep a tally */
      if (retval == UP_OK)
        options->count_successful++;
	  else if (retval == UP_FAIL)
        options->count_unsuccessful++;

      free(object_str);
    }
  
    g_slist_free(list_of_AUTO_refd_objects);
  }

  LG_log(lg_ctx, LG_FUNC,"<UP_process_submission: exiting with value %s", UP_ret2str(retval));
  return retval;
}
