[unstable] Produce some reasonable character set support when

postponing messages.

Additionally, this patch fixes a nasty pointer leak in
load_charset() [noted with electric fence], and a completely
mis-lead attempt to use bsearch().  Apparently, nobody ever tested
the utf8 decoder for quite some time.

10 files changed, 204 insertions, 62 deletions

diff --git a/OPS b/OPS
index 02cddbee..e5add27a 100644
--- a/OPS
+++ b/OPS
@@ -28,7 +28,8 @@ OP_COMPOSE_EDIT_TYPE "edit attachment type"
 OP_COMPOSE_GET_ATTACHMENT "get a temporary copy of an attachment"
 OP_COMPOSE_ISPELL "run ispell on the message"
 OP_COMPOSE_NEW_MIME "compose new attachment using mailcap entry"
-OP_COMPOSE_NORECODE "toggle recoding of this attachment"
+OP_COMPOSE_TOGGLE_RECODE "toggle recoding of this attachment"
+OP_COMPOSE_RECODE "recode this attachment to/from the local charset"
 OP_COMPOSE_POSTPONE_MESSAGE "save this message to send later"
 OP_COMPOSE_RENAME_FILE "rename/move an attached file"
 OP_COMPOSE_SEND_MESSAGE "send the message"
diff --git a/charset.c b/charset.c
index 58a2d205..1700a253 100644
--- a/charset.c
+++ b/charset.c
@@ -32,6 +32,7 @@
 #include <sys/types.h>
 #include <dirent.h>
 #include <unistd.h>
+#include <errno.h>
 
 #include "mutt.h"
 #include "charset.h"
@@ -77,7 +78,7 @@ static void fix_symbol (char *symbol, CHARMAP * m);
 
 static void canonical_charset (char *dest, size_t dlen, const char *name)
 {
-  int i;
+  size_t i;
 
   if (!strncasecmp (name, "x-", 2))
     name = name + 2;
@@ -96,7 +97,7 @@ static void canonical_charset (char *dest, size_t dlen, const char *name)
 static CHARSET *charset_new (size_t hash_size)
 {
   CHARSET *cp = safe_malloc (sizeof (CHARSET));
-  short i;
+  size_t i;
 
   cp->n_symb = 256;
   cp->u_symb = 0;
@@ -244,10 +245,10 @@ static void fix_symbol (char *symbol, CHARMAP * m)
 
   for (s = symbol, d = symbol; *s; *d++ = *s++)
   {
-    if (*s == m->escape_char)
-      s++;
+    if (*s == m->escape_char && !*++s)
+      break;
   }
-
+  
   *d = *s;
 }
 
@@ -420,17 +421,21 @@ static int load_charset (const char *filename, CHARSET ** csp, short multbyte)
       break;
     else if (i == CL_DESCR)
     {
-      dprint (5, (debugfile, "load_charset: Got character description: < %s > -> %x\n",
+      dprint (5, (debugfile, "load_charset: Got character description: <%s> -> %x\n",
 		  cd->symbol, cd->repr));
-      hash_delete (cs->symb_to_repr, cd->symbol, NULL, NULL);
-      hash_insert (cs->symb_to_repr, cd->symbol, cd, 0);
 
       if (!multbyte)
       {
 	if (0 <= cd->repr && cd->repr < 256)
 	{
+	  hash_delete (cs->symb_to_repr, cd->symbol, NULL, NULL);
+	  hash_insert (cs->symb_to_repr, cd->symbol, cd, 0);
+
 	  if (cs->description[cd->repr])
+	  {
+	    hash_delete (cs->symb_to_repr, cs->description[cd->repr]->symbol, cs->description[cd->repr], NULL);
 	    chardesc_free (&cs->description[cd->repr]);
+	  }
 	  else
 	    cs->u_symb++;
 
@@ -450,12 +455,17 @@ static int load_charset (const char *filename, CHARSET ** csp, short multbyte)
 	    cs->description[i] = NULL;
 	  cs->n_symb = new_size;
 	}
-
+	hash_delete (cs->symb_to_repr, cd->symbol, NULL, NULL);
+	hash_insert (cs->symb_to_repr, cd->symbol, cd, 0);
 	cs->description[cs->u_symb++] = cd;
 	cd = NULL;
       }
     }
-
+    if (cd)
+    {
+      dprint (5, (debugfile, "load_charset: character description still present: <%s>->%x\n",
+		  cd->symbol, cd->repr));
+    }
     chardesc_free (&cd);
   }
 
@@ -476,8 +486,9 @@ bail:
 
 static CHARDESC *repr2descr (int repr, CHARSET * cs)
 {
-  CHARDESC key;
-
+  CHARDESC *key;
+  CHARDESC **r;
+  
   if (!cs || repr < 0)
     return NULL;
 
@@ -489,9 +500,21 @@ static CHARDESC *repr2descr (int repr, CHARSET * cs)
       return NULL;
   }
 
-  key.repr = repr;
-  return bsearch (&key, cs->description, cs->u_symb,
-		  sizeof (CHARDESC *), _cd_compar);
+  key = safe_malloc (sizeof(CHARDESC));
+  key->repr = repr;
+  key->symbol = "<unknown>"; /* otherwise, the
+			     * debug code may 
+			     * segfault. ouch.
+			     */
+
+  r = bsearch (&key, cs->description, cs->u_symb,
+	       sizeof (CHARDESC *), _cd_compar);
+
+  safe_free ((void **) &key);
+
+  if (r) return *r;
+  
+  return NULL;
 }
 
 /* Build a translation table.  If a character cannot be
@@ -832,7 +855,7 @@ struct utf8_state
 
 static struct utf8_state *new_utf8_state (void)
 {
-  return safe_calloc (sizeof (struct utf8_state), 1);
+  return safe_calloc (1, sizeof (struct utf8_state));
 }
 
 static void free_utf8_state (struct utf8_state **sp)
@@ -878,7 +901,7 @@ static void state_fput_utf8(STATE *st, char u, CHARSET *chs, struct utf8_state *
     if(sfu->bp + 1 >= sfu->blen)
     {
       sfu->blen = (sfu->blen + 80) * 2;
-      safe_realloc((void **) &sfu->buffer, sfu->blen);
+      safe_realloc((void **) &sfu->buffer, sfu->blen + 1);
     }
     sfu->buffer[sfu->bp++] = u;
   }
@@ -888,7 +911,7 @@ static void state_fput_utf8(STATE *st, char u, CHARSET *chs, struct utf8_state *
 
 DECODER *mutt_open_decoder (STATE *s, BODY *b, int istext)
 {
-  DECODER *dp = safe_calloc (sizeof (DECODER), 1);
+  DECODER *dp = safe_calloc (1, sizeof (DECODER));
   
   dp->s = s;
   
@@ -930,3 +953,67 @@ void mutt_decoder_putc (DECODER *dp, char c)
   else
     state_prefix_putc (mutt_display_char ((unsigned char) c, dp->map), dp->s);
 }
+
+/* FIXME: utf-8 support */
+
+int mutt_recode_file (const char *fname, const char *src, const char *dest)
+{
+  FILE *fp, *tmpfp;
+  char tempfile[_POSIX_PATH_MAX];
+  int c;
+  int rv = -1;
+  
+  CHARSET_MAP *map;
+
+  if (mutt_is_utf8 (dest) ^ mutt_is_utf8(src))
+  {
+    mutt_error (_("We can't currently handle utf-8 at this point."));
+    return -1;
+  }
+
+  if ((fp = fopen (fname, "r+")) == NULL)
+  {
+    mutt_error (_("Can't open %s: %s."), fname, strerror (errno));
+    return -1;
+  }
+  
+  mutt_mktemp (tempfile);
+  if ((tmpfp = safe_fopen (tempfile, "w+")) == NULL)
+  {
+    mutt_error (_("Can't open %s: %s."), tempfile, strerror (errno));
+    fclose (fp);
+    return -1;
+  }
+
+  map = mutt_get_translation (src, dest);
+  
+  while ((c = fgetc (fp)) != EOF)
+    if (fputc (mutt_display_char ((unsigned char) c, map), tmpfp) == EOF)
+      goto bail;
+
+  fclose (fp); fp = NULL;
+  rewind (tmpfp);
+
+  /* don't use safe_fopen here - we're just going
+   * to overwrite the old file.
+   */
+
+  if ((fp = fopen (fname, "w")) == NULL)
+    goto bail;
+  
+  while ((c = fgetc (tmpfp)) != EOF)
+    if (fputc (c, fp) == EOF)
+      goto bail;
+
+  rv = 0;
+  unlink (tempfile);
+  
+bail:
+  if (rv == -1)
+    mutt_error (_("Error while recoding %s. See %s for recovering your data."),
+		fname, tempfile);
+
+  if (fp) fclose (fp);
+  if (tmpfp) fclose (tmpfp);
+  return rv;
+}
diff --git a/charset.h b/charset.h
index 08f73a1b..39522428 100644
--- a/charset.h
+++ b/charset.h
@@ -67,19 +67,16 @@ typedef struct decoder
 }
 DECODER;    
 
+
 CHARSET *mutt_get_charset(const char *);
 CHARSET_MAP *mutt_get_translation(const char *, const char *);
-
-unsigned char mutt_display_char(unsigned char, CHARSET_MAP *);
-
+DECODER *mutt_open_decoder (STATE *, BODY *, int);
 int mutt_display_string(char *, CHARSET_MAP *);
 int mutt_is_utf8(const char *);
-
-void mutt_decode_utf8_string(char *, CHARSET *);
-
-DECODER *mutt_open_decoder (STATE *, BODY *, int);
+int mutt_recode_file (const char *, const char *, const char *);
+unsigned char mutt_display_char(unsigned char, CHARSET_MAP *);
 void mutt_close_decoder (DECODER **);
-
+void mutt_decode_utf8_string(char *, CHARSET *);
 void mutt_decoder_putc (DECODER *, char);
 
 #endif
diff --git a/compose.c b/compose.c
index 811db861..ccf2849a 100644
--- a/compose.c
+++ b/compose.c
@@ -390,7 +390,7 @@ static int change_attachment_charset (BODY *b)
 {
   char buff[SHORT_STRING];
 
-  if (b->type != TYPETEXT)
+  if (!mutt_is_text_type (b->type, b->subtype))
   {
     mutt_error _("Can't change character set for non-text attachments!");
     return 0;
@@ -403,11 +403,13 @@ static int change_attachment_charset (BODY *b)
     
   if (mutt_is_utf8(buff))
   {
-    mutt_error (_("UTF-8 encoding attachments has not yet been implemented."));
-    return 0;
+    if (!b->noconv)
+    {
+      mutt_error (_("UTF-8 encoding attachments has not yet been implemented."));
+      return 0;
+    }
   }
-  
-  if (mutt_get_charset (buff) == NULL)
+  else if (mutt_get_charset (buff) == NULL)
   {
     mutt_error (_("Character set %s is unknown."), buff);
     return 0;
@@ -761,21 +763,63 @@ int mutt_compose_menu (HEADER *msg,   /* structure for new message */
         menu->redraw = change_attachment_charset(idx[menu->current]->content);
         break;
 
-      case OP_COMPOSE_NORECODE:
+#define CURRENT idx[menu->current]->content
+      
+      case OP_COMPOSE_TOGGLE_RECODE:
+      {      
         CHECK_COUNT;
-        if (idx[menu->current]->content->type != TYPETEXT)
+        if (!mutt_is_text_type (CURRENT->type, CURRENT->subtype))
         {
-	  mutt_error (_("Recoding only affects text/plain attachments."));
+	  mutt_error (_("Recoding only affects text attachments."));
+	  break;
+	}
+	if (mutt_is_utf8 (mutt_get_parameter ("charset", CURRENT->parameter)))
+	{
+	  mutt_error (_("We currently can't encode to utf-8."));
 	  break;
 	}
-        idx[menu->current]->content->noconv = !idx[menu->current]->content->noconv;
-        if (idx[menu->current]->content->noconv)
+        CURRENT->noconv = !CURRENT->noconv;
+        if (CURRENT->noconv)
 	  mutt_message (_("The current attachment won't be converted."));
         else
 	  mutt_message (_("The current attachment will be converted."));
 	menu->redraw = REDRAW_CURRENT;
         break;
-      
+      }
+
+      case OP_COMPOSE_RECODE:
+      {
+	const char *chs;
+	int rv;
+
+        CHECK_COUNT;
+        if (!mutt_is_text_type (CURRENT->type, CURRENT->subtype))
+        {
+	  mutt_error (_("Recoding only affetcs text attachments."));
+	  break;
+	}
+
+	if (!(chs = mutt_get_parameter ("charset", CURRENT->parameter)))
+	  chs = SendCharset;
+
+        if (CURRENT->noconv)
+	  rv = mutt_recode_file (CURRENT->filename, chs, Charset);
+	else
+	  rv = mutt_recode_file (CURRENT->filename, Charset, chs);
+
+	mutt_update_encoding (CURRENT);
+	
+	if (rv == 0)
+	{
+	  mutt_message (_("Recoding successful."));
+	  CURRENT->noconv = !CURRENT->noconv;
+	}
+
+	menu->redraw = REDRAW_CURRENT;
+	break;
+      }
+#undef CURRENT
+
       case OP_COMPOSE_EDIT_DESCRIPTION:
 	CHECK_COUNT;
 	strfcpy (buf,
diff --git a/functions.h b/functions.h
index 46c606e2..125dedcf 100644
--- a/functions.h
+++ b/functions.h
@@ -285,8 +285,9 @@ struct binding_t OpCompose[] = {
   { "edit-to",		OP_COMPOSE_EDIT_TO,		"t" },
   { "edit-type",	OP_COMPOSE_EDIT_TYPE,		"\024" },
   { "write-fcc",	OP_COMPOSE_WRITE_MESSAGE,	"w" },
+  { "recode-attachment",OP_COMPOSE_RECODE,		NULL },
   { "toggle-unlink",	OP_COMPOSE_TOGGLE_UNLINK,	"u" },
-  { "toggle-recode",    OP_COMPOSE_NORECODE,		NULL },
+  { "toggle-recode",    OP_COMPOSE_TOGGLE_RECODE,	NULL },
   { "update-encoding",	OP_COMPOSE_UPDATE_ENCODING,	"U" },
   { "view-attach",	OP_VIEW_ATTACH,			M_ENTER_S },
   { "send-message",	OP_COMPOSE_SEND_MESSAGE,	"y" },
diff --git a/hash.c b/hash.c
index d855cb0f..5d59e71e 100644
--- a/hash.c
+++ b/hash.c
@@ -50,9 +50,10 @@ HASH *hash_create (int nelem)
 }
 
 /* table        hash table to update
-   key          key to hash on
-   data         data to associate with `key'
-   allow_dup    if nonzero, duplicate keys are allowed in the table */
+ * key          key to hash on
+ * data         data to associate with `key'
+ * allow_dup    if nonzero, duplicate keys are allowed in the table 
+ */
 int hash_insert (HASH * table, const char *key, void *data, int allow_dup)
 {
   struct hash_elem *ptr;
@@ -108,20 +109,18 @@ void hash_delete_hash (HASH * table, int hash, const char *key, const void *data
 		       void (*destroy) (void *))
 {
   struct hash_elem *ptr = table->table[hash];
-  struct hash_elem *last = NULL;
-  for (; ptr; last = ptr, ptr = ptr->next)
+  struct hash_elem **last = &table->table[hash];
+
+  for (; ptr; last = &ptr->next, ptr = ptr->next)
   {
     /* if `data' is given, look for a matching ->data member.  this is
-       required for the case where we have multiple entries with the same
-       key */
-    if (data == ptr->data || (!data && mutt_strcmp (ptr->key, key) == 0))
+     * required for the case where we have multiple entries with the same
+     * key
+     */
+    if ((data == ptr->data) || (!data && mutt_strcmp (ptr->key, key) == 0))
     {
-      if (last)
-	last->next = ptr->next;
-      else
-	table->table[hash] = ptr->next;
-      if (destroy)
-	destroy (ptr->data);
+      *last = ptr->next;
+      if (destroy) destroy (ptr->data);
       FREE (&ptr);
       return;
     }
@@ -129,7 +128,8 @@ void hash_delete_hash (HASH * table, int hash, const char *key, const void *data
 }
 
 /* ptr		pointer to the hash table to be freed
-   destroy()	function to call to free the ->data member (optional) */
+ * destroy()	function to call to free the ->data member (optional) 
+ */
 void hash_destroy (HASH **ptr, void (*destroy) (void *))
 {
   int i;
diff --git a/main.c b/main.c
index 5a13a159..1c989536 100644
--- a/main.c
+++ b/main.c
@@ -361,9 +361,10 @@ int main (int argc, char **argv)
   textdomain (PACKAGE);
 #endif
 
+  setlocale (LC_CTYPE, "");
+
   mutt_error = mutt_nocurses_error;
   SRAND (time (NULL));
-  setlocale (LC_CTYPE, "");
   umask (077);
 
   memset (Options, 0, sizeof (Options));
diff --git a/postpone.c b/postpone.c
index f4083fd1..afb6d346 100644
--- a/postpone.c
+++ b/postpone.c
@@ -416,6 +416,7 @@ int mutt_parse_pgp_hdr (char *p, int set_signas)
 
 int mutt_prepare_edit_message (CONTEXT *ctx, HEADER *newhdr, HEADER *hdr)
 {
+  PARAMETER *par;
   MESSAGE *msg = mx_open_message (ctx, hdr->msgno);
   char file[_POSIX_PATH_MAX];
 
@@ -435,8 +436,7 @@ int mutt_prepare_edit_message (CONTEXT *ctx, HEADER *newhdr, HEADER *hdr)
      * message.
      */
     newhdr->content = hdr->content->parts;
-    b = hdr->content->parts;
-    while (b != NULL)
+    for (b = hdr->content->parts; b; b = b->next)
     {
       file[0] = '\0';
       if (b->filename)
@@ -455,11 +455,16 @@ int mutt_prepare_edit_message (CONTEXT *ctx, HEADER *newhdr, HEADER *hdr)
       safe_free ((void *) &b->filename);
       b->filename = safe_strdup (file);
       b->unlink = 1;
+
+      if (mutt_is_text_type (b->type, b->subtype))
+	b->noconv = 1;
+      
       mutt_stamp_attachment (b);
       mutt_free_body (&b->parts);
-      b = b->next;
     }
     hdr->content->parts = NULL;
+    if (hdr->content->type == TYPEMESSAGE && hdr->content->hdr)
+      hdr->content->hdr->content = NULL;
   }
   else
   {
@@ -478,6 +483,12 @@ int mutt_prepare_edit_message (CONTEXT *ctx, HEADER *newhdr, HEADER *hdr)
     newhdr->content->type = hdr->content->type;
     newhdr->content->xtype = safe_strdup (hdr->content->xtype);
     newhdr->content->subtype = safe_strdup (hdr->content->subtype);
+
+    for (par = hdr->content->parameter; par; par = par->next)
+      mutt_set_parameter (par->attribute, par->value, &newhdr->content->parameter);
+
+    if (mutt_is_text_type (newhdr->content->type, newhdr->content->subtype))
+      newhdr->content->noconv = 1;
     
     newhdr->content->use_disp = 0;	/* no content-disposition */
     newhdr->content->unlink = 1;	/* delete when we are done */
diff --git a/recvattach.c b/recvattach.c
index a5f8c84a..e5a6a3d8 100644
--- a/recvattach.c
+++ b/recvattach.c
@@ -175,13 +175,13 @@ const char *mutt_attach_fmt (char *dest,
       if (!optional)
       {
 	snprintf (fmt, sizeof (fmt), "%%%ss", prefix);
-	if (aptr->content->type == TYPETEXT && 
+	if (mutt_is_text_type (aptr->content->type, aptr->content->subtype) &&
 	    mutt_get_send_charset (charset, sizeof (charset), aptr->content, 0))
 	  snprintf (dest, destlen, fmt, charset);
 	else
 	  snprintf (dest, destlen, fmt, "");
       }
-      else if (aptr->content->type != TYPETEXT || 
+      else if (!mutt_is_text_type (aptr->content->type, aptr->content->subtype) ||
 	       !mutt_get_send_charset (charset, sizeof (charset), aptr->content, 0))
         optional = 0;
       break;
@@ -189,7 +189,7 @@ const char *mutt_attach_fmt (char *dest,
       /* XXX */
       if (!optional)
       {
-	snprintf (fmt, sizeof (fmt), "%%sc", prefix);
+	snprintf (fmt, sizeof (fmt), "%%%sc", prefix);
 	snprintf (dest, destlen, fmt, aptr->content->type != TYPETEXT ||
 		  aptr->content->noconv ? 'n' : 'c');
       }
diff --git a/sendlib.c b/sendlib.c
index 2b039df7..05d1d3b2 100644
--- a/sendlib.c
+++ b/sendlib.c
@@ -894,7 +894,7 @@ char *mutt_get_send_charset (char *d, size_t dlen, BODY *b, short f)
     else if (Charset)
       p = Charset;
   }
-  
+
   if (p)
   {
     strfcpy (d, NONULL(p), dlen);