This is the mail archive of the libc-alpha@sources.redhat.com mailing list for the glibc project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]

Unicode 3.1 support (6)



Unicode 3.1 introduces special characters called "tag characters" which
serve to declare (in a stateful manner!) the language of the following
plain text. It is ugly, but it is a compromise. Without these characters,
Unicode is apparently not acceptable to some Japanese people. Fortunately,
the support for us is easy: Unicode recommends

   "If language codes are not relevant to the particular processing
    operation, then they should be ignored."

This means that all converters from Unicode to a non-Unicode encoding must
ignore these tag characters if they occur in the input stream. Only the
ISO-2022-JP-2 encoder can make use of them.


2001-06-04  Bruno Haible  <haible@clisp.cons.org>

	* iconv/loop.c (UNICODE_TAG_HANDLER): New macro.
	* iconv/gconv_simple.c (__gconv_transform_internal_ascii): Invoke
	UNICODE_TAG_HANDLER.
	(__gconv_transform_internal_ucs2): Likewise.
	(__gconv_transform_internal_ucs2reverse): Likewise.
	* iconvdata/8bit-gap.c (BODY for TO_LOOP): Invoke UNICODE_TAG_HANDLER.
	* iconvdata/8bit-generic.c (BODY for TO_LOOP): Likewise.
	* iconvdata/ansi_x3.110.c (BODY for TO_LOOP): Likewise.
	* iconvdata/big5.c (BODY for TO_LOOP): Likewise.
	* iconvdata/big5hkscs.c (BODY for TO_LOOP): Likewise.
	* iconvdata/cp1255.c (BODY for TO_LOOP): Likewise.
	* iconvdata/cp1258.c (BODY for TO_LOOP): Likewise.
	* iconvdata/euc-cn.c (BODY for TO_LOOP): Likewise.
	* iconvdata/euc-jp.c (BODY for TO_LOOP): Likewise.
	* iconvdata/euc-kr.c (BODY for TO_LOOP): Likewise.
	* iconvdata/euc-tw.c (BODY for TO_LOOP): Likewise.
	* iconvdata/gbk.c (BODY for TO_LOOP): Likewise.
	* iconvdata/ibm930.c (BODY for TO_LOOP): Likewise.
	* iconvdata/ibm932.c (BODY for TO_LOOP): Likewise.
	* iconvdata/ibm933.c (BODY for TO_LOOP): Likewise.
	* iconvdata/ibm935.c (BODY for TO_LOOP): Likewise.
	* iconvdata/ibm937.c (BODY for TO_LOOP): Likewise.
	* iconvdata/ibm939.c (BODY for TO_LOOP): Likewise.
	* iconvdata/ibm943.c (BODY for TO_LOOP): Likewise.
	* iconvdata/iso646.c (BODY for TO_LOOP): Likewise.
	* iconvdata/iso8859-1.c (BODY for TO_LOOP): Likewise.
	* iconvdata/iso_6937.c (BODY for TO_LOOP): Likewise.
	* iconvdata/iso_6937-2.c (BODY for TO_LOOP): Likewise.
	* iconvdata/iso-2022-cn.c (BODY for TO_LOOP): Likewise.
	* iconvdata/iso-2022-cn-ext.c (BODY for TO_LOOP): Likewise.
	* iconvdata/iso-2022-kr.c (BODY for TO_LOOP): Likewise.
	* iconvdata/johab.c (BODY for TO_LOOP): Likewise.
	* iconvdata/sjis.c (BODY for TO_LOOP): Likewise.
	* iconvdata/t.61.c (BODY for TO_LOOP): Likewise.
	* iconvdata/uhc.c (BODY for TO_LOOP): Likewise.
	* iconvdata/unicode.c (BODY for TO_LOOP): Likewise.
	* iconvdata/iso-2022-jp.c (TAG_none, TAG_language, TAG_language_j,
	TAG_language_ja, TAG_language_k, TAG_language_ko, TAG_language_z,
	TAG_language_zh, CURRENT_TAG_MASK): New enum values.
	(EMIT_SHIFT_TO_INIT): Don't emit an escape sequence if ASCII_set
	is already selected but set2 or tag are set.
	(conversion): New enum type.
	(cvlist_t): New type.
	(CVLIST, CVLIST_FIRST, CVLIST_REST): New macros.
	(conversion_lists): New array.
	(BODY for TO_LOOP): Keep track of Unicode 3.1 language tag. If "ja",
	prefer conversion to Japanese character sets. If "zh", prefer
	conversion to GB2312. If "ko", prefer conversion to KSC5601. Small
	optimizations.
	(INIT_PARAMS): Add tag.
	(UPDATE_PARAMS): Add tag.

--- glibc-20010430/iconv/loop.c.bak	Mon Nov 27 23:28:38 2000
+++ glibc-20010430/iconv/loop.c	Mon May 28 03:53:17 2001
@@ -1,5 +1,5 @@
 /* Conversion loop frame work.
-   Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
 
@@ -212,6 +233,21 @@
   }
 
 
+/* Handling of Unicode 3.1 TAG characters.  Unicode recommends
+   "If language codes are not relevant to the particular processing
+    operation, then they should be ignored."
+   This macro is usually called right before STANDARD_ERR_HANDLER (Incr).  */
+#define UNICODE_TAG_HANDLER(Character, Incr) \
+  {									      \
+    /* TAG characters are those in the range U+E0000..U+E007F.  */	      \
+    if (((Character) >> 7) == (0xe0000 >> 7))				      \
+      {									      \
+	inptr += Incr;							      \
+	continue;							      \
+      }									      \
+  }
+
+
 /* The function returns the status, as defined in gconv.h.  */
 static inline int
 FCTNAME (LOOPFCT) (struct __gconv_step *step,
--- glibc-20010430/iconv/gconv_simple.c.bak	Tue Apr 10 23:52:15 2001
+++ glibc-20010430/iconv/gconv_simple.c	Mon May 28 03:30:18 2001
@@ -807,6 +807,7 @@
   {									      \
     if (__builtin_expect (*((const uint32_t *) inptr), 0) > 0x7f)	      \
       {									      \
+	UNICODE_TAG_HANDLER (*((const uint32_t *) inptr), 4);		      \
 	STANDARD_ERR_HANDLER (4);					      \
       }									      \
     else								      \
@@ -1199,6 +1200,7 @@
 									      \
     if (__builtin_expect (val, 0) >= 0x10000)				      \
       {									      \
+	UNICODE_TAG_HANDLER (val, 4);					      \
 	STANDARD_ERR_HANDLER (4);					      \
       }									      \
     else if (__builtin_expect (val >= 0xd800 && val < 0xe000, 0))	      \
@@ -1288,6 +1290,7 @@
     uint32_t val = *((const uint32_t *) inptr);				      \
     if (__builtin_expect (val, 0) >= 0x10000)				      \
       {									      \
+	UNICODE_TAG_HANDLER (val, 4);					      \
 	STANDARD_ERR_HANDLER (4);					      \
       }									      \
     else if (__builtin_expect (val >= 0xd800 && val < 0xe000, 0))	      \
--- glibc-20010430/iconvdata/8bit-gap.c.bak	Mon Jul  3 16:39:26 2000
+++ glibc-20010430/iconvdata/8bit-gap.c	Mon May 28 03:33:33 2001
@@ -1,6 +1,6 @@
 /* Generic conversion to and from 8bit charsets,
    converting from UCS using gaps.
-   Copyright (C) 1997, 1998, 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1997, 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
 
@@ -83,7 +83,10 @@
     unsigned char res;							      \
 									      \
     if (__builtin_expect (ch, 0) >= 0xffff)				      \
-      rp = NULL;							      \
+      {									      \
+	UNICODE_TAG_HANDLER (ch, 4);					      \
+	rp = NULL;							      \
+      }									      \
     else								      \
       while (ch > rp->end)						      \
 	++rp;								      \
--- glibc-20010430/iconvdata/8bit-generic.c.bak	Mon Jul  3 16:39:26 2000
+++ glibc-20010430/iconvdata/8bit-generic.c	Mon May 28 03:34:15 2001
@@ -1,5 +1,5 @@
 /* Generic conversion to and from 8bit charsets.
-   Copyright (C) 1997, 1998, 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1997, 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
 
@@ -67,6 +67,8 @@
     if (__builtin_expect (ch >= sizeof (from_ucs4) / sizeof (from_ucs4[0]), 0)\
 	|| (__builtin_expect (from_ucs4[ch], '\1') == '\0' && ch != 0))	      \
       {									      \
+	UNICODE_TAG_HANDLER (ch, 4);					      \
+									      \
 	/* This is an illegal character.  */				      \
 	STANDARD_ERR_HANDLER (4);					      \
       }									      \
--- glibc-20010430/iconvdata/ansi_x3.110.c.bak	Mon Dec  4 19:53:41 2000
+++ glibc-20010430/iconvdata/ansi_x3.110.c	Mon May 28 03:34:56 2001
@@ -1,5 +1,5 @@
 /* Generic conversion to and from ANSI_X3.110-1983.
-   Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
 
@@ -549,6 +549,8 @@
 	  cp = "\xd5";							      \
 	else								      \
 	  {								      \
+	    UNICODE_TAG_HANDLER (ch, 4);				      \
+									      \
 	    /* Illegal characters.  */					      \
 	    STANDARD_ERR_HANDLER (4);					      \
 	  }								      \
--- glibc-20010430/iconvdata/big5.c.bak	Mon Dec  4 19:53:42 2000
+++ glibc-20010430/iconvdata/big5.c	Mon May 28 03:35:19 2001
@@ -1,5 +1,5 @@
 /* Mapping tables for Big5 handling.
-   Copyright (C) 1997, 1998, 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1997, 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
 
@@ -8564,6 +8564,7 @@
 	  cp = from_ucs4_tab15[ch - 0xff01];				      \
 	  break;							      \
 	default:							      \
+	  UNICODE_TAG_HANDLER (ch, 4);					      \
 	  /* Illegal character.  */					      \
 	  cp = "";							      \
 	  break;							      \
--- glibc-20010430/iconvdata/big5hkscs.c.bak	Mon Dec  4 19:53:42 2000
+++ glibc-20010430/iconvdata/big5hkscs.c	Mon May 28 03:35:42 2001
@@ -1,5 +1,5 @@
 /* Mapping tables for Big5-HKSCS handling.
-   Copyright (C) 1997, 1998, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1997, 1998, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
    Modified for Big5-HKSCS by Roger So <spacehunt@e-fever.org>, 2000.
@@ -11772,6 +11772,8 @@
 									      \
     if (cp[0] == '\0' && ch != 0)					      \
       {									      \
+	UNICODE_TAG_HANDLER (ch, 4);					      \
+									      \
 	/* Illegal character.  */					      \
 	STANDARD_ERR_HANDLER (4);					      \
       }									      \
--- glibc-20010430/iconvdata/cp1255.c.bak	Sun May 20 17:44:32 2001
+++ glibc-20010430/iconvdata/cp1255.c	Mon May 28 03:36:41 2001
@@ -451,7 +451,10 @@
 	else if (ch == 0x2122)						      \
 	  res = 0x99;							      \
 	else								      \
-	  res = 0;							      \
+	  {								      \
+	    UNICODE_TAG_HANDLER (ch, 4);				      \
+	    res = 0;							      \
+	  }								      \
 									      \
 	if (__builtin_expect (res != 0, 1))				      \
 	  {								      \
--- glibc-20010430/iconvdata/cp1258.c.bak	Sun May 20 03:53:53 2001
+++ glibc-20010430/iconvdata/cp1258.c	Mon May 28 03:37:17 2001
@@ -776,7 +776,10 @@
 	else if (ch == 0x2122)						      \
 	  res = 0x99;							      \
 	else								      \
-	  res = 0;							      \
+	  {								      \
+	    UNICODE_TAG_HANDLER (ch, 4);				      \
+	    res = 0;							      \
+	  }								      \
 									      \
 	if (__builtin_expect (res != 0, 1))				      \
 	  {								      \
--- glibc-20010430/iconvdata/euc-cn.c.bak	Mon Jul  3 16:39:27 2000
+++ glibc-20010430/iconvdata/euc-cn.c	Mon May 28 03:37:49 2001
@@ -1,5 +1,5 @@
 /* Mapping tables for EUC-CN handling.
-   Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
 
@@ -140,6 +140,8 @@
 	  {								      \
 	    if (__builtin_expect (found, 0) == __UNKNOWN_10646_CHAR)	      \
 	      {								      \
+		UNICODE_TAG_HANDLER (ch, 4);				      \
+									      \
 		/* Illegal character.  */				      \
 		STANDARD_ERR_HANDLER (4);				      \
 	      }								      \
--- glibc-20010430/iconvdata/euc-jp.c.bak	Wed Jul 12 18:11:43 2000
+++ glibc-20010430/iconvdata/euc-jp.c	Mon May 28 03:38:19 2001
@@ -1,5 +1,5 @@
 /* Mapping tables for EUC-JP handling.
-   Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
 
@@ -227,6 +227,8 @@
 		  }							      \
 		else							      \
 		  {							      \
+		    UNICODE_TAG_HANDLER (ch, 4);			      \
+									      \
 		    /* Illegal character.  */				      \
 		    STANDARD_ERR_HANDLER (4);				      \
 		  }							      \
--- glibc-20010430/iconvdata/euc-kr.c.bak	Wed Nov  8 22:01:58 2000
+++ glibc-20010430/iconvdata/euc-kr.c	Mon May 28 03:38:39 2001
@@ -1,5 +1,5 @@
 /* Mapping tables for EUC-KR handling.
-   Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Jungshik Shin <jshin@pantheon.yale.edu>
    and Ulrich Drepper <drepper@cygnus.com>, 1998.
@@ -142,10 +142,12 @@
 									      \
     /* Decomposing Hangul syllables not available in KS C 5601 into	      \
        Jamos should be considered either here or in euckr_from_ucs4() */      \
-    euckr_from_ucs4 (ch, cp) ;						      \
+    euckr_from_ucs4 (ch, cp);						      \
 									      \
     if (__builtin_expect (cp[0], '\1') == '\0' && ch != 0)		      \
       {									      \
+	UNICODE_TAG_HANDLER (ch, 4);					      \
+									      \
 	/* Illegal character.  */					      \
 	STANDARD_ERR_HANDLER (4);					      \
       }									      \
--- glibc-20010430/iconvdata/euc-tw.c.bak	Tue Sep  5 15:24:48 2000
+++ glibc-20010430/iconvdata/euc-tw.c	Mon May 28 03:38:54 2001
@@ -1,5 +1,5 @@
 /* Mapping tables for EUC-TW handling.
-   Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
 
@@ -61,14 +61,14 @@
       }									      \
     else								      \
       {									      \
-	/* Two or more byte character.  First test whether the next	      \
-	   character is also available.  */				      \
+	/* Two or more byte character.  First test whether the next byte      \
+	   is also available.  */					      \
 	uint32_t ch2;							      \
 									      \
 	if (inptr + 1 >= inend)						      \
 	  {								      \
-	    /* The second character is not available.  Store the	      \
-	       intermediate result.  */					      \
+	    /* The second byte is not available.  Store the intermediate      \
+	       result.  */						      \
 	    result = __GCONV_INCOMPLETE_INPUT;				      \
 	    break;							      \
 	  }								      \
@@ -99,7 +99,7 @@
 									      \
 	    if (ch == 0)						      \
 	      {								      \
-		/* The third or fourth character is not available.  Store     \
+		/* The third or fourth byte is not available.  Store	      \
 		   the intermediate result.  */				      \
 		result = __GCONV_INCOMPLETE_INPUT;			      \
 		break;							      \
@@ -170,7 +170,7 @@
       *outptr++ = ch;							      \
     else								      \
       {									      \
-	/* Try the JIS character sets.  */				      \
+	/* Try the CNS 11643 planes.  */				      \
 	size_t found;							      \
 									      \
 	found = ucs4_to_cns11643l1 (ch, outptr, outend - outptr);	      \
@@ -199,6 +199,8 @@
 	      }								      \
 	    if (__builtin_expect (found, 0) == __UNKNOWN_10646_CHAR)	      \
 	      {								      \
+		UNICODE_TAG_HANDLER (ch, 4);				      \
+									      \
 		/* Illegal character.  */				      \
 		STANDARD_ERR_HANDLER (4);				      \
 	      }								      \
--- glibc-20010430/iconvdata/gbk.c.bak	Tue Sep 26 14:39:33 2000
+++ glibc-20010430/iconvdata/gbk.c	Mon May 28 03:39:18 2001
@@ -1,5 +1,5 @@
 /* Mapping tables for GBK handling.
-   Copyright (C) 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Sean Chen <sean.chen@turbolinux.com>, 1999.
 
@@ -13471,6 +13471,7 @@
 	  cp = "\xa1\xe9\0\0\xa1\xea\0\0\xa9\x56\0\0\xa3\xfe\0\0\xa9\x57\0\0\xa3\xa4" + ((ch - 0xffe0) * 4); \
 	  break;							      \
 	default:							      \
+	  UNICODE_TAG_HANDLER (ch, 4);					      \
 	  cp = "";							      \
 	  break; 							      \
 	}								      \
--- glibc-20010430/iconvdata/ibm930.c.bak	Mon May 14 13:05:08 2001
+++ glibc-20010430/iconvdata/ibm930.c	Mon May 28 03:39:34 2001
@@ -203,6 +203,8 @@
 									      \
     if (__builtin_expect (ch, 0) >= 0xffff)				      \
       {									      \
+	UNICODE_TAG_HANDLER (ch, 4);					      \
+									      \
 	if (! ignore_errors_p ())					      \
 	  {								      \
 	    result = __GCONV_ILLEGAL_INPUT;				      \
--- glibc-20010430/iconvdata/ibm932.c.bak	Sat Jan 20 22:16:44 2001
+++ glibc-20010430/iconvdata/ibm932.c	Mon May 28 03:39:51 2001
@@ -1,5 +1,5 @@
 /* Conversion from and to IBM932.
-   Copyright (C) 2000 Free Software Foundation, Inc.
+   Copyright (C) 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Masahide Washizawa <washi@jp.ibm.com>, 2000.
 
@@ -148,7 +148,7 @@
   {									      \
     const struct gap *rp = __ucs4_to_ibm932sb_idx;			      \
     unsigned char sc;							      \
-    uint32_t ch = get32(inptr);						      \
+    uint32_t ch = get32 (inptr);					      \
     uint16_t found = TRUE;						      \
     uint32_t i;								      \
     uint32_t low;							      \
@@ -156,7 +156,10 @@
     uint16_t pccode;							      \
 									      \
     if (__builtin_expect (ch, 0) >= 0xffff)				      \
+      {									      \
+	UNICODE_TAG_HANDLER (ch, 4);					      \
 	rp = NULL;							      \
+      }									      \
     else								      \
       while (ch > rp->end)						      \
 	++rp;								      \
@@ -188,26 +191,26 @@
 		break;							      \
 	      }								      \
 	  }								      \
-	  if (found) 							      \
-	    {								      \
-	      if (__builtin_expect (outptr + 2 > outend, 0))		      \
-		{							      \
-		  result = __GCONV_FULL_OUTPUT;				      \
-		  break;						      \
-		}							      \
-	      *outptr++ = pccode >> 8 & 0xff;				      \
-	      *outptr++ = pccode & 0xff;				      \
-	    }								      \
-	  else								      \
-	    {								      \
-	      /* This is an illegal character.  */			      \
-	      if (! ignore_errors_p ())					      \
-		{							      \
-		  result = __GCONV_ILLEGAL_INPUT;			      \
-		  break;						      \
-		}							      \
-	      ++*irreversible;						      \
-	    }								      \
+	if (found) 							      \
+	  {								      \
+	    if (__builtin_expect (outptr + 2 > outend, 0))		      \
+	      {								      \
+		result = __GCONV_FULL_OUTPUT;				      \
+		break;							      \
+	      }								      \
+	    *outptr++ = pccode >> 8 & 0xff;				      \
+	    *outptr++ = pccode & 0xff;					      \
+	  }								      \
+	else								      \
+	  {								      \
+	    /* This is an illegal character.  */			      \
+	    if (! ignore_errors_p ())					      \
+	      {								      \
+		result = __GCONV_ILLEGAL_INPUT;				      \
+		break;							      \
+	      }								      \
+	    ++*irreversible;						      \
+	  }								      \
       }									      \
     else								      \
       {									      \
@@ -224,8 +227,8 @@
 	  *outptr++ = sc;						      \
       }									      \
 									      \
-      /* Now that we wrote the output increment the input pointer.  */	      \
-      inptr += 4;							      \
+    /* Now that we wrote the output increment the input pointer.  */	      \
+    inptr += 4;								      \
   }
 #define LOOP_NEED_FLAGS
 #include <iconv/loop.c>
--- glibc-20010430/iconvdata/ibm933.c.bak	Mon May 14 13:05:35 2001
+++ glibc-20010430/iconvdata/ibm933.c	Mon May 28 03:40:20 2001
@@ -205,6 +205,8 @@
 									      \
     if (__builtin_expect (ch, 0) >= 0xffff)				      \
       {									      \
+	UNICODE_TAG_HANDLER (ch, 4);					      \
+									      \
 	if (! ignore_errors_p ())					      \
 	  {								      \
 	    result = __GCONV_ILLEGAL_INPUT;				      \
--- glibc-20010430/iconvdata/ibm935.c.bak	Mon May 14 13:05:59 2001
+++ glibc-20010430/iconvdata/ibm935.c	Mon May 28 03:41:02 2001
@@ -203,6 +203,8 @@
 									      \
     if (__builtin_expect (ch, 0) >= 0xffff)				      \
       {									      \
+	UNICODE_TAG_HANDLER (ch, 4);					      \
+									      \
 	if (! ignore_errors_p ())					      \
 	  {								      \
 	    result = __GCONV_ILLEGAL_INPUT;				      \
--- glibc-20010430/iconvdata/ibm937.c.bak	Mon May 14 13:06:23 2001
+++ glibc-20010430/iconvdata/ibm937.c	Mon May 28 03:41:21 2001
@@ -195,6 +195,8 @@
 				     / sizeof (__ucs4_to_ibm937db[0])), 0)    \
 	    || __builtin_expect (cp[0], '\1') == '\0')			      \
 	  {								      \
+	    UNICODE_TAG_HANDLER (ch, 4);				      \
+									      \
 	    /* This is an illegal character.  */			      \
 	    if (! ignore_errors_p ())					      \
 	      {								      \
--- glibc-20010430/iconvdata/ibm939.c.bak	Mon May 14 13:06:43 2001
+++ glibc-20010430/iconvdata/ibm939.c	Mon May 28 03:41:40 2001
@@ -203,6 +203,7 @@
 									      \
     if (__builtin_expect (ch, 0) >= 0xffff)				      \
       {									      \
+	UNICODE_TAG_HANDLER (ch, 4);					      \
 	rp1 = NULL;							      \
 	rp2 = NULL;							      \
       }									      \
--- glibc-20010430/iconvdata/ibm943.c.bak	Sat Jan 20 22:16:54 2001
+++ glibc-20010430/iconvdata/ibm943.c	Mon May 28 03:41:56 2001
@@ -1,5 +1,5 @@
 /* Conversion from and to IBM943.
-   Copyright (C) 2000 Free Software Foundation, Inc.
+   Copyright (C) 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Masahide Washizawa <washi@jp.ibm.com>, 2000.
 
@@ -156,7 +156,10 @@
     uint16_t pccode;							      \
 									      \
     if (__builtin_expect (ch, 0) >= 0xffff)				      \
+      {									      \
+	UNICODE_TAG_HANDLER (ch, 4);					      \
 	rp = NULL;							      \
+      }									      \
     else								      \
       while (ch > rp->end)						      \
 	++rp;								      \
@@ -188,26 +191,26 @@
 		break;							      \
 	      }								      \
 	  }								      \
-	  if (found) 							      \
-	    {								      \
-	      if (__builtin_expect (outptr + 2 > outend, 0))		      \
-		{							      \
-		  result = __GCONV_FULL_OUTPUT;				      \
-		  break;						      \
-		}							      \
-	      *outptr++ = pccode >> 8 & 0xff;				      \
-	      *outptr++ = pccode & 0xff;				      \
-	    }								      \
-	  else								      \
-	    {								      \
-	      /* This is an illegal character.  */			      \
-	      if (! ignore_errors_p ())					      \
-		{							      \
-		  result = __GCONV_ILLEGAL_INPUT;			      \
-		  break;						      \
-		}							      \
-	      ++*irreversible;						      \
-	    }								      \
+	if (found) 							      \
+	  {								      \
+	    if (__builtin_expect (outptr + 2 > outend, 0))		      \
+	      {								      \
+		result = __GCONV_FULL_OUTPUT;				      \
+		break;							      \
+	      }								      \
+	    *outptr++ = pccode >> 8 & 0xff;				      \
+	    *outptr++ = pccode & 0xff;					      \
+	  }								      \
+	else								      \
+	  {								      \
+	    /* This is an illegal character.  */			      \
+	    if (! ignore_errors_p ())					      \
+	      {								      \
+		result = __GCONV_ILLEGAL_INPUT;				      \
+		break;							      \
+	      }								      \
+	    ++*irreversible;						      \
+	  }								      \
       }									      \
     else								      \
       {									      \
@@ -224,8 +227,8 @@
 	  *outptr++ = sc;						      \
       }									      \
 									      \
-      /* Now that we wrote the output increment the input pointer.  */	      \
-      inptr += 4;							      \
+    /* Now that we wrote the output increment the input pointer.  */	      \
+    inptr += 4;								      \
   }
 #define LOOP_NEED_FLAGS
 #include <iconv/loop.c>
--- glibc-20010430/iconvdata/iso646.c.bak	Mon Dec  4 19:53:43 2000
+++ glibc-20010430/iconvdata/iso646.c	Mon May 28 03:42:37 2001
@@ -1,5 +1,5 @@
 /* Conversion to and from the various ISO 646 CCS.
-   Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
 
@@ -879,7 +879,10 @@
 	break;								      \
       default:								      \
 	if (__builtin_expect (ch, 0) > 0x7f)				      \
-	  failure = __GCONV_ILLEGAL_INPUT;				      \
+	  {								      \
+	    UNICODE_TAG_HANDLER (ch, 4);				      \
+	    failure = __GCONV_ILLEGAL_INPUT;				      \
+	  }								      \
 	break;								      \
       }									      \
 									      \
--- glibc-20010430/iconvdata/iso8859-1.c.bak	Mon Nov 27 23:28:38 2000
+++ glibc-20010430/iconvdata/iso8859-1.c	Mon May 28 03:43:14 2001
@@ -1,5 +1,5 @@
 /* Conversion to and from ISO 8859-1.
-   Copyright (C) 1997, 1998, 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1997, 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
 
@@ -48,6 +48,8 @@
     uint32_t ch = *((const uint32_t *) inptr);				      \
     if (__builtin_expect (ch, 0) > 0xff)				      \
       {									      \
+	UNICODE_TAG_HANDLER (ch, 4);					      \
+									      \
 	/* We have an illegal character.  */				      \
 	STANDARD_ERR_HANDLER (4);					      \
       }									      \
--- glibc-20010430/iconvdata/iso_6937.c.bak	Mon Dec  4 19:53:44 2000
+++ glibc-20010430/iconvdata/iso_6937.c	Mon May 28 03:43:01 2001
@@ -1,5 +1,5 @@
 /* Generic conversion to and from ISO 6937.
-   Copyright (C) 1997, 1998, 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1997, 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
 
@@ -536,6 +536,7 @@
 	    cp = "\xd5";						      \
 	    break;							      \
 	  default:							      \
+	    UNICODE_TAG_HANDLER (ch, 4);				      \
 	    cp = NULL;							      \
 	    fail = 1;							      \
 	  }								      \
--- glibc-20010430/iconvdata/iso_6937-2.c.bak	Mon Dec  4 19:53:44 2000
+++ glibc-20010430/iconvdata/iso_6937-2.c	Mon May 28 03:42:52 2001
@@ -1,5 +1,5 @@
 /* Generic conversion to and from ISO 6937-2.
-   Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
 
@@ -563,6 +563,7 @@
 	    cp = "\xd5";						      \
 	    break;							      \
 	  default:							      \
+	    UNICODE_TAG_HANDLER (ch, 4);				      \
 	    /* Illegal characters.  */					      \
 	    cp = NULL;							      \
 	    break;							      \
--- glibc-20010430/iconvdata/iso-2022-cn.c.bak	Mon May 14 13:07:10 2001
+++ glibc-20010430/iconvdata/iso-2022-cn.c	Mon Jun  4 03:50:46 2001
@@ -1,5 +1,5 @@
 /* Conversion module for ISO-2022-CN.
-   Copyright (C) 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1999.
 
@@ -320,6 +320,8 @@
 		  used = GB2312_set + CNS11643_1_set - used;		      \
 		else							      \
 		  {							      \
+		    UNICODE_TAG_HANDLER (ch, 4);			      \
+									      \
 		    /* Even this does not work.  Error.  */		      \
 		    STANDARD_ERR_HANDLER (4);				      \
 		  }							      \
--- glibc-20010430/iconvdata/iso-2022-cn-ext.c.bak	Mon Jun  4 16:31:00 2001
+++ glibc-20010430/iconvdata/iso-2022-cn-ext.c	Mon Jun  4 03:52:41 2001
@@ -560,6 +560,7 @@
 		  }							      \
 		if (used == ASCII_set)					      \
 		  {							      \
+		    UNICODE_TAG_HANDLER (ch, 4);			      \
 		    STANDARD_ERR_HANDLER (4);				      \
 		  }							      \
 	      }								      \
--- glibc-20010430/iconvdata/iso-2022-kr.c.bak	Mon May 14 13:08:44 2001
+++ glibc-20010430/iconvdata/iso-2022-kr.c	Mon May 28 03:42:11 2001
@@ -1,5 +1,5 @@
 /* Conversion module for ISO-2022-KR.
-   Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
 
@@ -221,7 +221,6 @@
 #define BODY \
   {									      \
     uint32_t ch;							      \
-    size_t written = 0;							      \
 									      \
     ch = get32 (inptr);							      \
 									      \
@@ -241,16 +240,18 @@
 	  }								      \
  									      \
 	*outptr++ = ch;							      \
-	written = 1;							      \
       }									      \
     else								      \
       {									      \
 	char buf[2];							      \
+	size_t written;							      \
 									      \
 	written = ucs4_to_ksc5601 (ch, buf, 2);				      \
 									      \
 	if (__builtin_expect (written, 0) == __UNKNOWN_10646_CHAR)	      \
 	  {								      \
+	    UNICODE_TAG_HANDLER (ch, 4);				      \
+									      \
 	    /* Illegal character.  */					      \
 	    STANDARD_ERR_HANDLER (4);					      \
 	  }								      \
--- glibc-20010430/iconvdata/johab.c.bak	Tue Sep 26 14:39:33 2000
+++ glibc-20010430/iconvdata/johab.c	Mon May 28 03:43:33 2001
@@ -1,5 +1,5 @@
 /* Mapping tables for JOHAB handling.
-   Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Jungshik Shin <jshin@pantheon.yale.edu>
    and Ulrich Drepper <drepper@cygnus.com>, 1998.
@@ -426,6 +426,7 @@
 	    if (__builtin_expect (written, 1) == __UNKNOWN_10646_CHAR	      \
 		|| (outptr[0] == 0x22 && outptr[1] > 0x65))		      \
 	      {								      \
+		UNICODE_TAG_HANDLER (ch, 4);				      \
 		STANDARD_ERR_HANDLER (4);				      \
 	      }								      \
 									      \
--- glibc-20010430/iconvdata/sjis.c.bak	Mon Dec  4 19:53:44 2000
+++ glibc-20010430/iconvdata/sjis.c	Mon May 28 03:44:30 2001
@@ -1,5 +1,5 @@
 /* Mapping tables for SJIS handling.
-   Copyright (C) 1997, 1998, 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1997, 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
 
@@ -4471,8 +4471,11 @@
 		 && __builtin_expect (ch, 0xff01) <= 0xffef)		      \
 	  cp = from_ucs4_extra[ch - 0xff00];				      \
 	else								      \
-	  /* Illegal character.  */					      \
-	  cp = "";							      \
+	  {								      \
+	    UNICODE_TAG_HANDLER (ch, 4);				      \
+	    /* Illegal character.  */					      \
+	    cp = "";							      \
+	  }								      \
       }									      \
     else								      \
       cp = from_ucs4_lat1[ch];						      \
--- glibc-20010430/iconvdata/t.61.c.bak	Mon Dec  4 19:53:44 2000
+++ glibc-20010430/iconvdata/t.61.c	Mon May 28 03:44:44 2001
@@ -1,5 +1,5 @@
 /* Generic conversion to and from T.61.
-   Copyright (C) 1997, 1998, 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1997, 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
 
@@ -468,6 +468,8 @@
 		 || __builtin_expect (ch, 0x2d8) > 0x2dd		      \
 		 || __builtin_expect (ch, 0x2d8) == 0x02dc)		      \
 	  {								      \
+	    UNICODE_TAG_HANDLER (ch, 4);				      \
+									      \
 	    /* Illegal characters.  */					      \
 	    STANDARD_ERR_HANDLER (4);					      \
 	  }								      \
--- glibc-20010430/iconvdata/uhc.c.bak	Tue Aug  1 13:52:59 2000
+++ glibc-20010430/iconvdata/uhc.c	Mon May 28 03:44:59 2001
@@ -1,5 +1,5 @@
 /* Mapping tables for UHC handling.
-   Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Jungshik Shin <jshin@pantheon.yale.edu>, 1998.
 
@@ -3243,6 +3243,7 @@
 	  }								      \
 	if (__builtin_expect (written, 0) == __UNKNOWN_10646_CHAR)	      \
 	  {								      \
+	    UNICODE_TAG_HANDLER (ch, 4);				      \
 	    STANDARD_ERR_HANDLER (4);					      \
 	  }								      \
 									      \
--- glibc-20010430/iconvdata/unicode.c.bak	Tue Apr 17 19:07:33 2001
+++ glibc-20010430/iconvdata/unicode.c	Mon May 28 03:45:19 2001
@@ -1,5 +1,5 @@
 /* Conversion module for Unicode
-   Copyright (C) 1999, 2000 Free Software Foundation, Inc.
+   Copyright (C) 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1999.
 
@@ -154,6 +154,7 @@
 									      \
     if (__builtin_expect (c, 0) >= 0x10000)				      \
       {									      \
+	UNICODE_TAG_HANDLER (c, 4);					      \
 	STANDARD_ERR_HANDLER (4);					      \
       }									      \
     else if (__builtin_expect (c >= 0xd800 && c < 0xe000, 0))		      \
--- glibc-20010430/iconvdata/iso-2022-jp.c.bak	Mon May 14 13:59:53 2001
+++ glibc-20010430/iconvdata/iso-2022-jp.c	Mon Jun  4 15:53:21 2001
@@ -1,5 +1,5 @@
-/* Conversion module for ISO-2022-JP.
-   Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+/* Conversion module for ISO-2022-JP and ISO-2022-JP-2.
+   Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
 
@@ -90,14 +90,14 @@
 enum
 {
   ASCII_set = 0,
-  JISX0208_1978_set = 8,
-  JISX0208_1983_set = 16,
-  JISX0201_Roman_set = 24,
-  JISX0201_Kana_set = 32,
-  GB2312_set = 40,
-  KSC5601_set = 48,
-  JISX0212_set = 56,
-  CURRENT_SEL_MASK = 56
+  JISX0208_1978_set = 1 << 3,
+  JISX0208_1983_set = 2 << 3,
+  JISX0201_Roman_set = 3 << 3,
+  JISX0201_Kana_set = 4 << 3,
+  GB2312_set = 5 << 3,
+  KSC5601_set = 6 << 3,
+  JISX0212_set = 7 << 3,
+  CURRENT_SEL_MASK = 7 << 3
 };
 
 /* The second value stored is the designation of the G2 set.  The following
@@ -105,9 +105,25 @@
 enum
 {
   UNSPECIFIED_set = 0,
-  ISO88591_set = 64,
-  ISO88597_set = 128,
-  CURRENT_ASSIGN_MASK = 192
+  ISO88591_set = 1 << 6,
+  ISO88597_set = 2 << 6,
+  CURRENT_ASSIGN_MASK = 3 << 6
+};
+
+/* The third value, only used during conversion from Unicode to ISO-2022-JP-2,
+   describes the language tag parsing status.  The possible values are as
+   follows.  Values >= TAG_language are temporary tag parsing states.  */
+enum
+{
+  TAG_none = 0,
+  TAG_language = 4 << 8,
+  TAG_language_j = 5 << 8,
+  TAG_language_ja = 1 << 8,
+  TAG_language_k = 6 << 8,
+  TAG_language_ko = 2 << 8,
+  TAG_language_z = 7 << 8,
+  TAG_language_zh = 3 << 8,
+  CURRENT_TAG_MASK = 7 << 8
 };
 
 
@@ -198,7 +214,8 @@
 									      \
   if ((data->__statep->__count & ~7) != ASCII_set)			      \
     {									      \
-      if (dir == from_iso2022jp)					      \
+      if (dir == from_iso2022jp						      \
+	  || (data->__statep->__count & CURRENT_SEL_MASK) == ASCII_set)	      \
 	{								      \
 	  /* It's easy, we don't have to emit anything, we just reset the     \
 	     state for the input.  Note that this also clears the G2	      \
@@ -503,6 +520,23 @@
 
 
 /* Next, define the other direction.  */
+
+enum conversion { none = 0, european, japanese, chinese, korean, other };
+
+/* A datatype for conversion lists.  */
+typedef unsigned int cvlist_t;
+#define CVLIST(cv1, cv2, cv3, cv4, cv5) \
+  ((cv1) + ((cv2) << 3) + ((cv3) << 6) + ((cv4) << 9) + ((cv5) << 12))
+#define CVLIST_FIRST(cvl) ((cvl) & ((1 << 3) - 1))
+#define CVLIST_REST(cvl) ((cvl) >> 3)
+static const cvlist_t conversion_lists[4] =
+  {
+    /* TAG_none */        CVLIST (japanese, european, chinese, korean, other),
+    /* TAG_language_ja */ CVLIST (japanese, european, chinese, korean, other),
+    /* TAG_language_ko */ CVLIST (korean, european, japanese, chinese, other),
+    /* TAG_language_zh */ CVLIST (chinese, european, japanese, korean, other)
+  };
+
 #define MIN_NEEDED_INPUT	MIN_NEEDED_TO
 #define MIN_NEEDED_OUTPUT	MIN_NEEDED_FROM
 #define MAX_NEEDED_OUTPUT	(MAX_NEEDED_FROM + 2)
@@ -510,12 +544,56 @@
 #define BODY \
   {									      \
     uint32_t ch;							      \
-    size_t written = 0;							      \
+    size_t written;							      \
 									      \
     ch = get32 (inptr);							      \
 									      \
+    if (var == iso2022jp2)						      \
+      {									      \
+	/* Handle Unicode tag characters (range U+E0000..U+E007F).  */	      \
+	if (__builtin_expect ((ch >> 7) == (0xe0000 >> 7), 0))		      \
+	  {								      \
+	    ch &= 0x7f;							      \
+	    if (ch >= 'A' && ch <= 'Z')					      \
+	      ch += 'a' - 'A';						      \
+	    if (ch == 0x01)						      \
+	      tag = TAG_language;					      \
+	    else if (ch == 'j' && tag == TAG_language)			      \
+	      tag = TAG_language_j;					      \
+	    else if (ch == 'a' && tag == TAG_language_j)		      \
+	      tag = TAG_language_ja;					      \
+	    else if (ch == 'k' && tag == TAG_language)			      \
+	      tag = TAG_language_k;					      \
+	    else if (ch == 'o' && tag == TAG_language_k)		      \
+	      tag = TAG_language_ko;					      \
+	    else if (ch == 'z' && tag == TAG_language)			      \
+	      tag = TAG_language_z;					      \
+	    else if (ch == 'h' && tag == TAG_language_z)		      \
+	      tag = TAG_language_zh;					      \
+	    else if (ch == 0x7f)					      \
+	      tag = TAG_none;						      \
+	    else							      \
+	      {								      \
+		/* Other tag characters reset the tag parsing state (if the   \
+		   current state is a temporary state) or are ignored (if     \
+		   the current state is a stable one).  */		      \
+		if (tag >= TAG_language)				      \
+		  tag = TAG_none;					      \
+	      }								      \
+									      \
+	    inptr += 4;							      \
+	    continue;							      \
+	  }								      \
+									      \
+	/* Non-tag characters reset the tag parsing state, if the current     \
+	   state is a temporary state.  */				      \
+	if (__builtin_expect (tag >= TAG_language, 0))			      \
+	  tag = TAG_none;						      \
+      }									      \
+									      \
     /* First see whether we can write the character using the currently	      \
-       selected character set.  */					      \
+       selected character set.  But ignore the selected character set if      \
+       the current language tag shows different preferences.  */	      \
     if (set == ASCII_set)						      \
       {									      \
 	/* Please note that the NUL byte is *not* matched if we are not	      \
@@ -525,58 +603,75 @@
 	  {								      \
 	    *outptr++ = ch;						      \
 	    written = 1;						      \
+									      \
+	    /* At the beginning of a line, G2 designation is cleared.  */     \
+	    if (var == iso2022jp2 && ch == 0x0a)			      \
+	      set2 = UNSPECIFIED_set;					      \
 	  }								      \
-	/* At the beginning of a line, G2 designation is cleared.  */	      \
-	if (var == iso2022jp2 && ch == 0x0a)				      \
-	  set2 = UNSPECIFIED_set;					      \
+	else								      \
+	  written = __UNKNOWN_10646_CHAR;				      \
       }									      \
     /* ISO-2022-JP recommends to encode the newline character always in	      \
        ASCII since this allows a context-free interpretation of the	      \
        characters at the beginning of the next line.  Otherwise it would      \
        have to be known whether the last line ended using ASCII or	      \
        JIS X 0201.  */							      \
-    else if (set == JISX0201_Roman_set)					      \
+    else if (set == JISX0201_Roman_set					      \
+	     && (__builtin_expect (tag == TAG_none, 1)			      \
+		 || tag == TAG_language_ja))				      \
       {									      \
-	unsigned char buf[2];						      \
+	unsigned char buf[1];						      \
 	written = ucs4_to_jisx0201 (ch, buf);				      \
-	if (written != __UNKNOWN_10646_CHAR && buf[0] > 0x20		      \
-	    && buf[0] < 0x80)						      \
+	if (written != __UNKNOWN_10646_CHAR)				      \
 	  {								      \
-	    *outptr++ = buf[0];						      \
-	    written = 1;						      \
+	    if (buf[0] > 0x20 && buf[0] < 0x80)				      \
+	      {								      \
+		*outptr++ = buf[0];					      \
+		written = 1;						      \
+	      }								      \
+	    else							      \
+	      written = __UNKNOWN_10646_CHAR;				      \
 	  }								      \
-	else								      \
-	  written = __UNKNOWN_10646_CHAR;				      \
       }									      \
-    else if (set == JISX0201_Kana_set)					      \
+    else if (set == JISX0201_Kana_set					      \
+	     && (__builtin_expect (tag == TAG_none, 1)			      \
+		 || tag == TAG_language_ja))				      \
       {									      \
-	unsigned char buf[2];						      \
+	unsigned char buf[1];						      \
 	written = ucs4_to_jisx0201 (ch, buf);				      \
-	if (written != __UNKNOWN_10646_CHAR && buf[0] > 0xa0		      \
-	    && buf[0] < 0xe0)						      \
+	if (written != __UNKNOWN_10646_CHAR)				      \
 	  {								      \
-	    *outptr++ = buf[0] - 0x80;					      \
-	    written = 1;						      \
+	    if (buf[0] > 0xa0 && buf[0] < 0xe0)				      \
+	      {								      \
+		*outptr++ = buf[0] - 0x80;				      \
+		written = 1;						      \
+	      }								      \
+	    else							      \
+	      written = __UNKNOWN_10646_CHAR;				      \
 	  }								      \
-	else								      \
-	  written = __UNKNOWN_10646_CHAR;				      \
       }									      \
     else								      \
       {									      \
-	if (set == JISX0208_1978_set || set == JISX0208_1983_set)	      \
+	if ((set == JISX0208_1978_set || set == JISX0208_1983_set)	      \
+	    && (__builtin_expect (tag == TAG_none, 1)			      \
+		|| tag == TAG_language_ja))				      \
 	  written = ucs4_to_jisx0208 (ch, outptr, outend - outptr);	      \
-	else if (set == JISX0212_set)					      \
+	else if (set == JISX0212_set					      \
+		 && (__builtin_expect (tag == TAG_none, 1)		      \
+		     || tag == TAG_language_ja))			      \
 	  written = ucs4_to_jisx0212 (ch, outptr, outend - outptr);	      \
-	else if (set == GB2312_set)					      \
+	else if (set == GB2312_set					      \
+		 && (__builtin_expect (tag == TAG_none, 1)		      \
+		     || tag == TAG_language_zh))			      \
 	  written = ucs4_to_gb2312 (ch, outptr, outend - outptr);	      \
+	else if (set == KSC5601_set					      \
+		 && (__builtin_expect (tag == TAG_none, 1)		      \
+		     || tag == TAG_language_ko))			      \
+	  written = ucs4_to_ksc5601 (ch, outptr, outend - outptr);	      \
 	else								      \
-	  {								      \
-	    assert (set == KSC5601_set);				      \
-									      \
-	    written = ucs4_to_ksc5601 (ch, outptr, outend - outptr);	      \
-	  }								      \
+	  written = __UNKNOWN_10646_CHAR;				      \
 									      \
-	if (__builtin_expect (written, 1) == 0)				      \
+	if (__builtin_expect (written == 0, 0))				      \
 	  {								      \
 	    result = __GCONV_FULL_OUTPUT;				      \
 	    break;							      \
@@ -585,18 +680,19 @@
 	  outptr += written;						      \
       }									      \
 									      \
-    if (written == __UNKNOWN_10646_CHAR || written == 0)		      \
+    if (written == __UNKNOWN_10646_CHAR					      \
+	&& __builtin_expect (tag == TAG_none, 1))			      \
       {									      \
 	if (set2 == ISO88591_set)					      \
 	  {								      \
-	    if (__builtin_expect (outptr + 3 > outend, 0))		      \
+	    if (ch >= 0x80 && ch <= 0xff)				      \
 	      {								      \
-		result = __GCONV_FULL_OUTPUT;				      \
-		break;							      \
-	      }								      \
+		if (__builtin_expect (outptr + 3 > outend, 0))		      \
+		  {							      \
+		    result = __GCONV_FULL_OUTPUT;			      \
+		    break;						      \
+		  }							      \
 									      \
-	    if (ch >= 0x80 && ch <= 0xff) 				      \
-	      {								      \
 		*outptr++ = ESC;					      \
 		*outptr++ = 'N';					      \
 		*outptr++ = ch & 0x7f;					      \
@@ -629,18 +725,15 @@
 	  }								      \
       }									      \
 									      \
-    if (written == __UNKNOWN_10646_CHAR || written == 0)		      \
+    if (written == __UNKNOWN_10646_CHAR)				      \
       {									      \
-	/* Either this is an unknown character or we have to switch	      \
-	   the currently selected character set.  The character sets	      \
-	   do not code entirely separate parts of ISO 10646 and		      \
-	   therefore there is no single correct result.  If we choose	      \
-	   the character set to use wrong we might be end up with	      \
-	   using yet another character set for the next character	      \
-	   though the current and the next could be encoded with one	      \
-	   character set.  We leave this kind of optimization for	      \
-	   later and now simply use a fixed order in which we test for	      \
-	   availability  */						      \
+	/* The attempts to use the currently selected character set	      \
+	   failed, either because the language tag changed, or because	      \
+	   the character requires a different character set, or because	      \
+	   the character is unknown.					      \
+	   The CJK character sets partially overlap when seen as subsets      \
+	   of ISO 10646; therefore there is no single correct result.	      \
+	   We use a preferrence order which depends on the language tag.  */  \
 									      \
 	if (ch <= 0x7f)							      \
 	  {								      \
@@ -671,128 +764,76 @@
 	else								      \
 	  {								      \
 	    /* Now it becomes difficult.  We must search the other	      \
-	       character sets one by one and we cannot use simple	      \
-	       arithmetic to determine whether the character can be	      \
-	       encoded using this set.  */				      \
-	    size_t written;						      \
+	       character sets one by one.  Use an ordered conversion	      \
+	       list that depends on the current language tag.  */	      \
+	    cvlist_t conversion_list;					      \
 	    unsigned char buf[2];					      \
 									      \
-	    written = ucs4_to_jisx0201 (ch, buf);			      \
-	    if (written != __UNKNOWN_10646_CHAR && buf[0] < 0x80)	      \
-	      {								      \
-		/* We use JIS X 0201.  */				      \
-		if (__builtin_expect (outptr + 3 > outend, 0))		      \
-		  {							      \
-		    result = __GCONV_FULL_OUTPUT;			      \
-		    break;						      \
-		  }							      \
-									      \
-		*outptr++ = ESC;					      \
-		*outptr++ = '(';					      \
-		*outptr++ = 'J';					      \
-		set = JISX0201_Roman_set;				      \
+	    result = __GCONV_ILLEGAL_INPUT;				      \
 									      \
-		if (__builtin_expect (outptr + 1 > outend, 0))		      \
-		  {							      \
-		    result = __GCONV_FULL_OUTPUT;			      \
-		    break;						      \
-		  }							      \
-		*outptr++ = buf[0];					      \
-	      }								      \
+	    if (var == iso2022jp2)					      \
+	      conversion_list = conversion_lists[tag >> 8];		      \
 	    else							      \
-	      {								      \
-		written = ucs4_to_jisx0208 (ch, buf, 2);		      \
-		if (written != __UNKNOWN_10646_CHAR)			      \
-		  {							      \
-		    /* We use JIS X 0208.  */				      \
-		    if (__builtin_expect (outptr + 3 > outend, 0))	      \
-		      {							      \
-			result = __GCONV_FULL_OUTPUT;			      \
-			break;						      \
-		      }							      \
+	      conversion_list = CVLIST (japanese, 0, 0, 0, 0);		      \
 									      \
-		    *outptr++ = ESC;					      \
-		    *outptr++ = '$';					      \
-		    *outptr++ = 'B';					      \
-		    set = JISX0208_1983_set;				      \
+	    do								      \
+	      switch (CVLIST_FIRST (conversion_list))			      \
+		{							      \
+		case european:						      \
+									      \
+		  /* Try ISO 8859-1 upper half.   */			      \
+		  if (ch >= 0x80 && ch <= 0xff)				      \
+		    {							      \
+		      if (set2 != ISO88591_set)				      \
+			{						      \
+			  if (__builtin_expect (outptr + 3 > outend, 0))      \
+			    {						      \
+			      result = __GCONV_FULL_OUTPUT;		      \
+			      break;					      \
+			    }						      \
+			  *outptr++ = ESC;				      \
+			  *outptr++ = '.';				      \
+			  *outptr++ = 'A';				      \
+			  set2 = ISO88591_set;				      \
+			}						      \
+									      \
+		      if (__builtin_expect (outptr + 3 > outend, 0))	      \
+			{						      \
+			  result = __GCONV_FULL_OUTPUT;			      \
+			  break;					      \
+			}						      \
+		      *outptr++ = ESC;					      \
+		      *outptr++ = 'N';					      \
+		      *outptr++ = ch - 0x80;				      \
+		      result = __GCONV_OK;				      \
+		      break;						      \
+		    }							      \
 									      \
-		    if (__builtin_expect (outptr + 2 > outend, 0))	      \
-		      {							      \
-			result = __GCONV_FULL_OUTPUT;			      \
-			break;						      \
-		      }							      \
-		    *outptr++ = buf[0];					      \
-		    *outptr++ = buf[1];					      \
-		  }							      \
-		else if (__builtin_expect (var, iso2022jp2) == iso2022jp)     \
+		  /* Try ISO 8859-7 upper half.  */			      \
 		  {							      \
-		    /* We have no other choice.  */			      \
-		    STANDARD_ERR_HANDLER (4);				      \
-		  }							      \
-		else							      \
-		  {							      \
-		    written = ucs4_to_jisx0212 (ch, buf, 2);		      \
-		    if (written != __UNKNOWN_10646_CHAR)		      \
-		      {							      \
-			/* We use JIS X 0212.  */			      \
-			if (__builtin_expect (outptr + 4 > outend, 0))	      \
-			  {						      \
-			    result = __GCONV_FULL_OUTPUT;		      \
-			    break;					      \
-			  }						      \
-			*outptr++ = ESC;				      \
-			*outptr++ = '$';				      \
-			*outptr++ = '(';				      \
-			*outptr++ = 'D';				      \
-			set = JISX0212_set;				      \
+		    const struct gap *rp = from_idx;			      \
 									      \
-			if (__builtin_expect (outptr + 2 > outend, 0))	      \
-			  {						      \
-			    result = __GCONV_FULL_OUTPUT;		      \
-			    break;					      \
-			  }						      \
-			*outptr++ = buf[0];				      \
-			*outptr++ = buf[1];				      \
-		      }							      \
-		    else						      \
+		    while (ch > rp->end)				      \
+		      ++rp;						      \
+		    if (ch >= rp->start)				      \
 		      {							      \
-			written = ucs4_to_jisx0201 (ch, buf);		      \
-			if (written != __UNKNOWN_10646_CHAR		      \
-			    && buf[0] >= 0x80)				      \
+			unsigned char res =				      \
+			  iso88597_from_ucs4[ch - 0xa0 + rp->idx];	      \
+			if (res != '\0')				      \
 			  {						      \
-			    /* We use JIS X 0201.  */			      \
-			    if (__builtin_expect (outptr + 3 > outend, 0))    \
-			      {						      \
-			        result = __GCONV_FULL_OUTPUT;		      \
-			        break;					      \
-			      }						      \
-									      \
-			    *outptr++ = ESC;				      \
-			    *outptr++ = '(';				      \
-			    *outptr++ = 'I';				      \
-			    set = JISX0201_Kana_set;			      \
-									      \
-			    if (__builtin_expect (outptr + 1 > outend, 0))    \
+			    if (set2 != ISO88597_set)			      \
 			      {						      \
-			        result = __GCONV_FULL_OUTPUT;		      \
-			        break;					      \
-			      }						      \
-			    *outptr++ = buf[0] - 0x80;			      \
-			  }						      \
-			else if (ch != 0xa5 && ch >= 0x80 && ch <= 0xff)      \
-			  {						      \
-			    /* ISO 8859-1 upper half.   */		      \
-			    if (__builtin_expect (outptr + 3 > outend, 0))    \
-			      {						      \
-				result = __GCONV_FULL_OUTPUT;		      \
-				break;					      \
+				if (__builtin_expect (outptr + 3 > outend, 0))\
+				  {					      \
+				    result = __GCONV_FULL_OUTPUT;	      \
+				    break;				      \
+				  }					      \
+				*outptr++ = ESC;			      \
+				*outptr++ = '.';			      \
+				*outptr++ = 'F';			      \
+				set2 = ISO88597_set;			      \
 			      }						      \
 									      \
-			    *outptr++ = ESC;				      \
-			    *outptr++ = '.';				      \
-			    *outptr++ = 'A';				      \
-			    set2 = ISO88591_set;			      \
-									      \
 			    if (__builtin_expect (outptr + 3 > outend, 0))    \
 			      {						      \
 				result = __GCONV_FULL_OUTPUT;		      \
@@ -800,106 +841,221 @@
 			      }						      \
 			    *outptr++ = ESC;				      \
 			    *outptr++ = 'N';				      \
-			    *outptr++ = ch - 0x80;			      \
+			    *outptr++ = res;				      \
+			    result = __GCONV_OK;			      \
+			    break;					      \
 			  }						      \
-			else						      \
-			  {						      \
-			    written = ucs4_to_gb2312 (ch, buf, 2);	      \
-			    if (written != __UNKNOWN_10646_CHAR)	      \
-			      {						      \
-				/* We use GB 2312.  */			      \
-				if (__builtin_expect (outptr + 3 > outend, 0))\
-				  {					      \
-				    result = __GCONV_FULL_OUTPUT;	      \
-				    break;				      \
-				  }					      \
+		      }							      \
+		  }							      \
 									      \
-				*outptr++ = ESC;			      \
-				*outptr++ = '$';			      \
-				*outptr++ = 'A';			      \
-				set = GB2312_set;			      \
+		  break;						      \
 									      \
-				if (__builtin_expect (outptr + 2 > outend, 0))\
-				  {					      \
-				    result = __GCONV_FULL_OUTPUT;	      \
-				    break;				      \
-				  }					      \
-				*outptr++ = buf[0];			      \
-				*outptr++ = buf[1];			      \
-			      }						      \
-			    else					      \
-			      {						      \
-				written = ucs4_to_ksc5601 (ch, buf, 2);	      \
-				if (written != __UNKNOWN_10646_CHAR)	      \
-				  {					      \
-				    /* We use KSC 5601.  */		      \
-				    if (__builtin_expect (outptr + 4 > outend,\
-							  0))		      \
-				      {					      \
-					result = __GCONV_FULL_OUTPUT;	      \
-					break;				      \
-				      }					      \
-				    *outptr++ = ESC;			      \
-				    *outptr++ = '$';			      \
-				    *outptr++ = '(';			      \
-				    *outptr++ = 'C';			      \
-				    set = KSC5601_set;			      \
-									      \
-				    if (__builtin_expect (outptr + 2 > outend,\
-							  0))		      \
-				      {					      \
-					result = __GCONV_FULL_OUTPUT;	      \
-					break;				      \
-				      }					      \
-				    *outptr++ = buf[0];			      \
-				    *outptr++ = buf[1];			      \
-				  }					      \
-				else					      \
-				  {					      \
-				    const struct gap *rp = from_idx;	      \
-				    unsigned char gch = 0;		      \
+		case japanese:						      \
 									      \
-				    while (ch > rp->end)		      \
-				      ++rp;				      \
-				    if (ch >= rp->start)		      \
-				      {					      \
-					ch = ch - 0xa0 + rp->idx;	      \
-					gch = iso88597_from_ucs4[ch];	      \
-				      }					      \
-									      \
-				    if (__builtin_expect (gch, 1) != 0)	      \
-				      {					      \
-					/* We use ISO 8859-7 greek.  */	      \
-					if (__builtin_expect (outptr + 3      \
-							      > outend, 0))   \
-					  {				      \
-					    result = __GCONV_FULL_OUTPUT;     \
-					    break;			      \
-					  }				      \
-					*outptr++ = ESC;		      \
-					*outptr++ = '.';		      \
-					*outptr++ = 'F';		      \
-					set2 = ISO88597_set;		      \
-									      \
-					if (__builtin_expect (outptr + 3      \
-							      > outend, 0))   \
-					  {				      \
-					    result = __GCONV_FULL_OUTPUT;     \
-					    break;			      \
-					  }				      \
-					*outptr++ = ESC;		      \
-					*outptr++ = 'N';		      \
-					*outptr++ = gch;		      \
-				      }					      \
-				    else				      \
-				      {					      \
-					STANDARD_ERR_HANDLER (4);	      \
-				      }					      \
-				  }					      \
-			      }						      \
-			  }						      \
-		      }							      \
-		  }							      \
+		  /* Try JIS X 0201 Roman.  */				      \
+		  written = ucs4_to_jisx0201 (ch, buf);			      \
+		  if (written != __UNKNOWN_10646_CHAR			      \
+		      && buf[0] > 0x20 && buf[0] < 0x80)		      \
+		    {							      \
+		      if (set != JISX0201_Roman_set)			      \
+			{						      \
+			  if (__builtin_expect (outptr + 3 > outend, 0))      \
+			    {						      \
+			      result = __GCONV_FULL_OUTPUT;		      \
+			      break;					      \
+			    }						      \
+			  *outptr++ = ESC;				      \
+			  *outptr++ = '(';				      \
+			  *outptr++ = 'J';				      \
+			  set = JISX0201_Roman_set;			      \
+			}						      \
+									      \
+		      if (__builtin_expect (outptr + 1 > outend, 0))	      \
+			{						      \
+			  result = __GCONV_FULL_OUTPUT;			      \
+			  break;					      \
+			}						      \
+		      *outptr++ = buf[0];				      \
+		      result = __GCONV_OK;				      \
+		      break;						      \
+		    }							      \
+									      \
+		  /* Try JIS X 0208.  */				      \
+		  written = ucs4_to_jisx0208 (ch, buf, 2);		      \
+		  if (written != __UNKNOWN_10646_CHAR)			      \
+		    {							      \
+		      if (set != JISX0208_1983_set)			      \
+			{						      \
+			  if (__builtin_expect (outptr + 3 > outend, 0))      \
+			    {						      \
+			      result = __GCONV_FULL_OUTPUT;		      \
+			      break;					      \
+			    }						      \
+			  *outptr++ = ESC;				      \
+			  *outptr++ = '$';				      \
+			  *outptr++ = 'B';				      \
+			  set = JISX0208_1983_set;			      \
+			}						      \
+									      \
+		      if (__builtin_expect (outptr + 2 > outend, 0))	      \
+			{						      \
+			  result = __GCONV_FULL_OUTPUT;			      \
+			  break;					      \
+			}						      \
+		      *outptr++ = buf[0];				      \
+		      *outptr++ = buf[1];				      \
+		      result = __GCONV_OK;				      \
+		      break;						      \
+		    }							      \
+									      \
+		  if (__builtin_expect (var == iso2022jp, 0))		      \
+		    /* Don't use the other Japanese character sets.  */	      \
+		    break;						      \
+									      \
+		  /* Try JIS X 0212.  */				      \
+		  written = ucs4_to_jisx0212 (ch, buf, 2);		      \
+		  if (written != __UNKNOWN_10646_CHAR)			      \
+		    {							      \
+		      if (set != JISX0212_set)				      \
+			{						      \
+			  if (__builtin_expect (outptr + 4 > outend, 0))      \
+			    {						      \
+			      result = __GCONV_FULL_OUTPUT;		      \
+			      break;					      \
+			    }						      \
+			  *outptr++ = ESC;				      \
+			  *outptr++ = '$';				      \
+			  *outptr++ = '(';				      \
+			  *outptr++ = 'D';				      \
+			  set = JISX0212_set;				      \
+			}						      \
+									      \
+		      if (__builtin_expect (outptr + 2 > outend, 0))	      \
+			{						      \
+			  result = __GCONV_FULL_OUTPUT;			      \
+			  break;					      \
+			}						      \
+		      *outptr++ = buf[0];				      \
+		      *outptr++ = buf[1];				      \
+		      result = __GCONV_OK;				      \
+		      break;						      \
+		    }							      \
+									      \
+		  break;						      \
+									      \
+		case chinese:						      \
+		  assert (var == iso2022jp2);				      \
+									      \
+		  /* Try GB 2312.  */					      \
+		  written = ucs4_to_gb2312 (ch, buf, 2);		      \
+		  if (written != __UNKNOWN_10646_CHAR)			      \
+		    {							      \
+		      if (set != GB2312_set)				      \
+			{						      \
+			  if (__builtin_expect (outptr + 3 > outend, 0))      \
+			    {						      \
+			      result = __GCONV_FULL_OUTPUT;		      \
+			      break;					      \
+			    }						      \
+			  *outptr++ = ESC;				      \
+			  *outptr++ = '$';				      \
+			  *outptr++ = 'A';				      \
+			  set = GB2312_set;				      \
+			}						      \
+									      \
+		      if (__builtin_expect (outptr + 2 > outend, 0))	      \
+			{						      \
+			  result = __GCONV_FULL_OUTPUT;			      \
+			  break;					      \
+			}						      \
+		      *outptr++ = buf[0];				      \
+		      *outptr++ = buf[1];				      \
+		      result = __GCONV_OK;				      \
+		      break;						      \
+		    }							      \
+									      \
+		  break;						      \
+									      \
+		case korean:						      \
+		  assert (var == iso2022jp2);				      \
+									      \
+		  /* Try KSC 5601.  */					      \
+		  written = ucs4_to_ksc5601 (ch, buf, 2);		      \
+		  if (written != __UNKNOWN_10646_CHAR)			      \
+		    {							      \
+		      if (set != KSC5601_set)				      \
+			{						      \
+			  if (__builtin_expect (outptr + 4 > outend, 0))      \
+			    {						      \
+			      result = __GCONV_FULL_OUTPUT;		      \
+			      break;					      \
+			    }						      \
+			  *outptr++ = ESC;				      \
+			  *outptr++ = '$';				      \
+			  *outptr++ = '(';				      \
+			  *outptr++ = 'C';				      \
+			  set = KSC5601_set;				      \
+			}						      \
+									      \
+		      if (__builtin_expect (outptr + 2 > outend, 0))	      \
+			{						      \
+			  result = __GCONV_FULL_OUTPUT;			      \
+			  break;					      \
+			}						      \
+		      *outptr++ = buf[0];				      \
+		      *outptr++ = buf[1];				      \
+		      result = __GCONV_OK;				      \
+		      break;						      \
+		    }							      \
+									      \
+		  break;						      \
+									      \
+		case other:						      \
+		  assert (var == iso2022jp2);				      \
+									      \
+		  /* Try JIS X 0201 Kana.  This is not officially part	      \
+		     of ISO-2022-JP-2, according to RFC 1554.  Therefore      \
+		     we try this only after all other attempts.  */	      \
+		  written = ucs4_to_jisx0201 (ch, buf);			      \
+		  if (written != __UNKNOWN_10646_CHAR && buf[0] >= 0x80)      \
+		    {							      \
+		      if (set != JISX0201_Kana_set)			      \
+			{						      \
+			  if (__builtin_expect (outptr + 3 > outend, 0))      \
+			    {						      \
+			      result = __GCONV_FULL_OUTPUT;		      \
+			      break;					      \
+			    }						      \
+			  *outptr++ = ESC;				      \
+			  *outptr++ = '(';				      \
+			  *outptr++ = 'I';				      \
+			  set = JISX0201_Kana_set;			      \
+			}						      \
+									      \
+		      if (__builtin_expect (outptr + 1 > outend, 0))	      \
+			{						      \
+			  result = __GCONV_FULL_OUTPUT;			      \
+			  break;					      \
+			}						      \
+		      *outptr++ = buf[0] - 0x80;			      \
+		      result = __GCONV_OK;				      \
+		      break;						      \
+		    }							      \
+									      \
+		  break;						      \
+									      \
+		default:						      \
+		  abort ();						      \
+		}							      \
+	    while (result == __GCONV_ILLEGAL_INPUT			      \
+		   && (conversion_list = CVLIST_REST (conversion_list)) != 0);\
+									      \
+	    if (result == __GCONV_FULL_OUTPUT)				      \
+	      break;							      \
+									      \
+	    if (result == __GCONV_ILLEGAL_INPUT)			      \
+	      {								      \
+		STANDARD_ERR_HANDLER (4);				      \
 	      }								      \
 	  }								      \
       }									      \
@@ -910,8 +1066,9 @@
 #define LOOP_NEED_FLAGS
 #define EXTRA_LOOP_DECLS	, enum variant var, int *setp
 #define INIT_PARAMS		int set = *setp & CURRENT_SEL_MASK;	      \
-				int set2 = *setp & CURRENT_ASSIGN_MASK
-#define UPDATE_PARAMS		*setp = set | set2
+				int set2 = *setp & CURRENT_ASSIGN_MASK;	      \
+				int tag = *setp & CURRENT_TAG_MASK;
+#define UPDATE_PARAMS		*setp = set | set2 | tag
 #include <iconv/loop.c>
 
 


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]