Merging the external character set patch into unstable.

Additionally, this change introduces a M_CHARCONV state flag which
gives us some more control about when character set conversions are
actually done.  Current versions of mutt would happily apply
character set conversions when, e.g., saving a text/plain attachment
to a file.  (We had at least one corrupt russing translation file
due to this mis-feature.)

Additionally, we clean up some of the character set related code in
handler.c.  Most of that is now done by the decoder functions in
charset.c.

1 files changed, 110 insertions, 102 deletions

diff --git a/sendlib.c b/sendlib.c
index b72753f7..2b039df7 100644
--- a/sendlib.c
+++ b/sendlib.c
@@ -116,13 +116,16 @@ static char MsgIdPfx = 'A';
 
 static void transform_to_7bit (BODY *a, FILE *fpin);
 
-static void encode_quoted (FILE * fin, FILE *fout, int istext)
+static void encode_quoted (FILE * fin, FILE *fout, int istext, CHARSET_MAP *map)
 {
   int c, linelen = 0;
   char line[77], savechar;
 
   while ((c = fgetc (fin)) != EOF)
   {
+    if(istext && map)
+      c = mutt_display_char(c, map);
+
     /* Escape lines that begin with "the message separator". */
     if (linelen == 5 && !mutt_strncmp ("From ", line, 5))
     {
@@ -297,7 +300,7 @@ static void b64_putc(char c, FILE *fout)
 }
   
   
-static void encode_base64 (FILE * fin, FILE *fout, int istext)
+static void encode_base64 (FILE * fin, FILE *fout, int istext, CHARSET_MAP *map)
 {
   int ch, ch1 = EOF;
   
@@ -305,6 +308,9 @@ static void encode_base64 (FILE * fin, FILE *fout, int istext)
   
   while((ch = fgetc(fin)) != EOF)
   {
+    if(istext && map)
+      ch = mutt_display_char(ch, map);
+
     if(istext && ch == '\n' && ch1 != '\r')
       b64_putc('\r', fout);
     b64_putc(ch, fout);
@@ -314,68 +320,22 @@ static void encode_base64 (FILE * fin, FILE *fout, int istext)
   fputc('\n', fout);
 }
 
-#ifdef PERMIT_DEPRECATED_UUENCODED_MESSAGES
-
-#define UUENC(c) ((c) ? ((c) & 077) + ' ' : '`')
-
-static void encode_uuenc (FILE * fin, FILE * fout)
+static void encode_8bit(FILE *fin, FILE *fout, int istext, CHARSET_MAP *map)
 {
-  register int ch, linelen;
-  register unsigned char *p;
-  unsigned char line[80];
+  int ch;
 
-  while ((linelen = fread(line, 1, 45, fin)))
+  if(!istext || !map)
   {
-    ch = UUENC(linelen);
-    fputc (ch, fout);
-
-    for (p = line; linelen>0; linelen -= 3, p += 3)
-    {
-      ch = *p >> 2;
-      ch = UUENC(ch);
-      fputc(ch, fout);
-
-      if (linelen>1)
-      {
-	ch = ((*p & 0x3) << 4) | (p[1] >> 4);
-	ch = UUENC(ch);
-	fputc(ch, fout);
-      }
-      else
-      {
-	ch = (*p & 0x3) << 4;
-	ch = UUENC(ch);
-	fputc(ch, fout);
-	break;
-      }
-
-      if (linelen>2)
-      {
-	ch = ((p[1] & 0xf) << 2) | (p[2] >> 6);
-	ch = UUENC(ch);
-	fputc(ch, fout);
-      }
-      else
-      {
-	ch = (p[1] & 0xf) << 2;
-	ch = UUENC(ch);
-	fputc(ch, fout);
-	break;
-      }
-
-      ch = p[2] & 0x3f;
-      ch = UUENC(ch);
-      fputc(ch, fout);
-    }
-
-    fputc('\n', fout);
+    mutt_copy_stream(fin, fout);
+    return;
+  }
+  
+  while((ch = fgetc(fin)) != EOF)
+  {
+    fputc(mutt_display_char(ch, map), fout);
   }
-  ch = UUENC('\0');
-  fputc(ch, fout);
-  fputc('\n', fout);
 }
-
-#endif
+  
 
 int mutt_write_mime_header (BODY *a, FILE *f)
 {
@@ -459,12 +419,11 @@ int mutt_write_mime_header (BODY *a, FILE *f)
 int mutt_write_mime_body (BODY *a, FILE *f)
 {
   char *p, boundary[SHORT_STRING];
+  char send_charset[SHORT_STRING];
   FILE *fpin;
   BODY *t;
-#ifdef PERMIT_DEPRECATED_UUENCODED_MESSAGES
-  char *r;
-#endif
-
+  CHARSET_MAP *map = NULL;
+  
   if (a->type == TYPEMULTIPART)
   {
     /* First, find the boundary to use */
@@ -509,25 +468,19 @@ int mutt_write_mime_body (BODY *a, FILE *f)
     return -1;
   }
 
+  if (a->type == TYPETEXT)
+    map = mutt_get_translation (Charset, mutt_get_send_charset (send_charset, sizeof(send_charset), a, 1));
+
   if (a->encoding == ENCQUOTEDPRINTABLE)
-    encode_quoted (fpin, f, mutt_is_text_type (a->type, a->subtype));
+    encode_quoted (fpin, f, mutt_is_text_type (a->type, a->subtype), 
+		   a->type == TYPETEXT && (!a->noconv) ? map : NULL);
   else if (a->encoding == ENCBASE64)
-    encode_base64 (fpin, f, mutt_is_text_type (a->type, a->subtype));
-#ifdef PERMIT_DEPRECATED_UUENCODED_MESSAGES
-  else if (a->encoding == ENCUUENCODED)
-  {
-    /* Strip off the leading path... */
-    if ((r = strrchr (a->filename, '/')))
-      r++;
-    else
-      r = a->filename;
-    fprintf (f, "begin 600 %s\n", r);
-    encode_uuenc (fpin, f);
-    fprintf (f, "end\n");
-  }
-#endif
+    encode_base64 (fpin, f, mutt_is_text_type (a->type, a->subtype), 
+		   a->type == TYPETEXT && (!a->noconv) ? map : NULL);
   else
-    mutt_copy_stream (fpin, f);
+    encode_8bit (fpin, f, mutt_is_text_type (a->type, a->subtype),
+		      a->type == TYPETEXT && (!a->noconv) ? map : NULL);
+
   fclose (fpin);
 
   return (ferror (f) ? -1 : 0);
@@ -549,12 +502,14 @@ void mutt_generate_boundary (PARAMETER **parm)
 }
 
 /* analyze the contents of a file to determine which MIME encoding to use */
-static CONTENT *mutt_get_content_info (const char *fname)
+static CONTENT *mutt_get_content_info (const char *fname, BODY *b)
 {
   CONTENT *info;
   FILE *fp;
   int ch, from=0, whitespace=0, dot=0, linelen=0;
 
+  if(b && !fname) fname = b->filename;
+  
   if ((fp = fopen (fname, "r")) == NULL)
   {
     dprint (1, (debugfile, "mutt_get_content_info: %s: %s (errno %d).\n",
@@ -737,18 +692,6 @@ static int lookup_mime_type (char *d, const char *s)
   return (cur_n);
 }
 
-static char *set_text_charset (CONTENT *info)
-{
-  if ((Charset == NULL || mutt_strcasecmp (Charset, "us-ascii") == 0)
-      && info->hibin)
-    return ("unknown-8bit");
-
-  if (info->hibin)
-    return (Charset);
-
-  return ("us-ascii");
-}
-
 void mutt_message_to_7bit (BODY *a, FILE *fp)
 {
   char temp[_POSIX_PATH_MAX];
@@ -871,6 +814,25 @@ static void transform_to_7bit (BODY *a, FILE *fpin)
   }
 }
 
+static const char *get_text_charset (BODY *b, CONTENT *info)
+{
+  char send_charset[SHORT_STRING];
+  char *chsname;
+
+  chsname = mutt_get_send_charset (send_charset, sizeof (send_charset), b, 1);
+  
+  /* if charset is unknown assume low bytes are ascii compatible */
+
+  if ((chsname == NULL || mutt_strcasecmp (chsname, "us-ascii") == 0)
+      && info->hibin)
+    return ("unknown-8bit");
+
+  if (info->hibin)
+    return (chsname);
+
+  return ("us-ascii");
+}
+
 /* determine which Content-Transfer-Encoding to use */
 static void mutt_set_encoding (BODY *b, CONTENT *info)
 {
@@ -912,19 +874,66 @@ void mutt_stamp_attachment(BODY *a)
   a->stamp = time(NULL);
 }
 
+/* Get the character set which is to be used for sending */
+
+char *mutt_get_send_charset (char *d, size_t dlen, BODY *b, short f)
+{
+  char *p = NULL;
+
+  if (b && b->type != TYPETEXT)
+    return NULL;
+
+  if (b) 
+    p = mutt_get_parameter ("charset", b->parameter);
+
+  /* override the special "us-ascii" and "unknown-8bit" character sets */
+  if (!p || (f && (!mutt_strcasecmp (p, "us-ascii") || !mutt_strcasecmp (p, "unknown-8bit"))))
+  {
+    if (SendCharset && *SendCharset)
+      p = SendCharset;
+    else if (Charset)
+      p = Charset;
+  }
+  
+  if (p)
+  {
+    strfcpy (d, NONULL(p), dlen);
+    return d;
+  }
+
+  /* something is seriously wrong. */
+  return NULL;
+}
+
+/* set a body structure's character set */
+
+void mutt_set_body_charset(BODY *b, const char *chs)
+{
+  char send_charset[SHORT_STRING];
+  
+  if(b->type != TYPETEXT)
+    return;
+
+  if(!chs && !(chs = mutt_get_send_charset(send_charset, sizeof(send_charset), NULL, 1)))
+    return;
+
+  mutt_set_parameter ("charset", chs, &b->parameter);
+}
+
+
 /* Assumes called from send mode where BODY->filename points to actual file */
 void mutt_update_encoding (BODY *a)
 {
   CONTENT *info;
 
-  if ((info = mutt_get_content_info (a->filename)) == NULL)
+  if ((info = mutt_get_content_info (a->filename, a)) == NULL)
     return;
 
   mutt_set_encoding (a, info);
   mutt_stamp_attachment(a);
   
   if (a->type == TYPETEXT)
-    mutt_set_parameter ("charset", set_text_charset (info), &a->parameter);
+    mutt_set_body_charset(a, get_text_charset(a, info));
 
 #ifdef _PGPPATH
   /* save the info in case this message is signed.  we will want to do Q-P
@@ -977,26 +986,26 @@ BODY *mutt_make_message_attach (CONTEXT *ctx, HEADER *hdr, int attach_msg)
   if (!attach_msg && option (OPTMIMEFORWDECODE))
   {
     chflags |= CH_MIME | CH_TXTPLAIN;
-    cmflags = M_CM_DECODE;
+    cmflags = M_CM_DECODE | M_CM_CHARCONV;
 #ifdef _PGPPATH
     pgp &= ~PGPENCRYPT;
 #endif
   }
 #ifdef _PGPPATH
   else
-    if(option(OPTFORWDECRYPT)
+    if (option (OPTFORWDECRYPT)
        && (hdr->pgp & PGPENCRYPT))
   {
-    if(mutt_is_multipart_encrypted(hdr->content))
+    if (mutt_is_multipart_encrypted (hdr->content))
     {
       chflags |= CH_MIME | CH_NONEWLINE;
       cmflags = M_CM_DECODE_PGP;
       pgp &= ~PGPENCRYPT;
     }
-    else if(mutt_is_application_pgp(hdr->content) & PGPENCRYPT)
+    else if (mutt_is_application_pgp (hdr->content) & PGPENCRYPT)
     {
       chflags |= CH_MIME | CH_TXTPLAIN;
-      cmflags = M_CM_DECODE;
+      cmflags = M_CM_DECODE | M_CM_CHARCONV;
       pgp &= ~PGPENCRYPT;
     }
   }
@@ -1029,7 +1038,7 @@ BODY *mutt_make_file_attach (const char *path)
   char buf[SHORT_STRING];
   int n;
   
-  if ((info = mutt_get_content_info (path)) == NULL)
+  if ((info = mutt_get_content_info (path, NULL)) == NULL)
     return NULL;
 
   att = mutt_new_body ();
@@ -1054,8 +1063,7 @@ BODY *mutt_make_file_attach (const char *path)
        */
       att->type = TYPETEXT;
       att->subtype = safe_strdup ("plain");
-      
-      mutt_set_parameter("charset", set_text_charset(info), &att->parameter);
+      mutt_set_body_charset(att, get_text_charset(att, info));
     }
     else
     {