emacs: src/coding.c comparison

comparison src/coding.c @ 87676:2aeceff24280

(detect_coding_iso2022): New arg latin_extra_code_state. Allow Latin extra codes only when *latin_extra_code_state is nonzero. (detect_coding_mask): If there is a NULL byte, detect the encoding as UTF-16 or binary. If there is a Latin extra code, detect the encoding as ISO-2022 only when no other proper encoding is found.

author	Kenichi Handa <handa@m17n.org>
date	Wed, 09 Jan 2008 06:05:23 +0000
parents	107ccd98fa12
children	df9e1c663162

comparison

equal deleted inserted replaced

-:e4a11c2d5016
+:2aeceff24280
 	CODING_CATEGORY_MASK_ISO_8_1
 	CODING_CATEGORY_MASK_ISO_8_2
 	CODING_CATEGORY_MASK_ISO_7_ELSE
 	CODING_CATEGORY_MASK_ISO_8_ELSE
 are set.  If a code which should never appear in ISO2022 is found,
-returns 0.  */
+returns 0.
+If *latin_extra_code_state is zero and Latin extra codes are found,
+set *latin_extra_code_state to 1 and return 0.  If it is nonzero,
+accept Latin extra codes.  */
 static int
-detect_coding_iso2022 (src, src_end, multibytep)
+detect_coding_iso2022 (src, src_end, multibytep, latin_extra_code_state)
 unsigned char *src, *src_end;
 int multibytep;
+int *latin_extra_code_state;
 {
 int mask = CODING_CATEGORY_MASK_ISO;
 int mask_found = 0;
 int reg[4], shift_out = 0, single_shifting = 0;
 int c, c1, charset;
 		single_shifting = 1;
 	      }
 	    if (VECTORP (Vlatin_extra_code_table)
 		&& !NILP (XVECTOR (Vlatin_extra_code_table)->contents[c]))
 	      {
+		if (! *latin_extra_code_state)
+		  {
+		    *latin_extra_code_state = 1;
+		    return 0;
+		  }
 		if (coding_system_table[CODING_CATEGORY_IDX_ISO_8_1]->flags
 		    & CODING_FLAG_ISO_LATIN_EXTRA)
 		  newmask |= CODING_CATEGORY_MASK_ISO_8_1;
 		if (coding_system_table[CODING_CATEGORY_IDX_ISO_8_2]->flags
 		    & CODING_FLAG_ISO_LATIN_EXTRA)
 	      if (VECTORP (Vlatin_extra_code_table)
 		  && !NILP (XVECTOR (Vlatin_extra_code_table)->contents[c]))
 		{
 		  int newmask = 0;
+		  if (! *latin_extra_code_state)
+		    {
+		      *latin_extra_code_state = 1;
+		      return 0;
+		    }
 		  if (coding_system_table[CODING_CATEGORY_IDX_ISO_8_1]->flags
 		      & CODING_FLAG_ISO_LATIN_EXTRA)
 		    newmask |= CODING_CATEGORY_MASK_ISO_8_1;
 		  if (coding_system_table[CODING_CATEGORY_IDX_ISO_8_2]->flags
 		      & CODING_FLAG_ISO_LATIN_EXTRA)
 {
 register unsigned char c;
 unsigned char *src = source, *src_end = source + src_bytes;
 unsigned int mask, utf16_examined_p, iso2022_examined_p;
 int i;
+int null_byte_found;
+int latin_extra_code_state = 1;
 /* At first, skip all ASCII characters and control characters except
 for three ISO2022 specific control characters.  */
 ascii_skip_code[ISO_CODE_SO] = 0;
 ascii_skip_code[ISO_CODE_SI] = 0;
 ascii_skip_code[ISO_CODE_ESC] = 0;
 label_loop_detect_coding:
-while (src < src_end && ascii_skip_code[*src]) src++;
+null_byte_found = 0;
+while (src < src_end && ascii_skip_code[*src])
+null_byte_found |= (! *src++);
+if (! null_byte_found)
+{
+unsigned char *p = src + 1;
+while (p < src_end)
+	null_byte_found |= (! *p++);
+}
 *skip = src - source;
 if (src >= src_end)
-/* We found nothing other than ASCII.  There's nothing to do.  */
+/* We found nothing other than ASCII (and NULL byte).  There's
+nothing to do.  */
 return 0;
 c = *src;
 /* The text seems to be encoded in some multilingual coding system.
 Now, try to find in which coding system the text is encoded.  */
-if (c < 0x80)
+if (! null_byte_found && c < 0x80)
 {
 /* i.e. (c == ISO_CODE_ESC || c == ISO_CODE_SI || c == ISO_CODE_SO) */
 /* C is an ISO2022 specific control code of C0.  */
-mask = detect_coding_iso2022 (src, src_end, multibytep);
+latin_extra_code_state = 1;
+mask = detect_coding_iso2022 (src, src_end, multibytep,
+				    &latin_extra_code_state);
 if (mask == 0)
 	{
 	  /* No valid ISO2022 code follows C.  Try again.  */
 	  src++;
 	  if (c == ISO_CODE_ESC)
 int try;
 if (multibytep && c == LEADING_CODE_8_BIT_CONTROL)
 	c = src[1] - 0x20;
-if (c < 0xA0)
+if (null_byte_found)
+	{
+	  try = (CODING_CATEGORY_MASK_UTF_16_BE
+		 | CODING_CATEGORY_MASK_UTF_16_LE);
+	}
+else if (c < 0xA0)
 	{
 	  /* C is the first byte of SJIS character code,
 	     or a leading-code of Emacs' internal format (emacs-mule),
 	     or the first byte of UTF-16.  */
 	  try = (CODING_CATEGORY_MASK_SJIS
-		  | CODING_CATEGORY_MASK_EMACS_MULE
+		 | CODING_CATEGORY_MASK_EMACS_MULE
-		  | CODING_CATEGORY_MASK_UTF_16_BE
+		 | CODING_CATEGORY_MASK_UTF_16_BE
-		  | CODING_CATEGORY_MASK_UTF_16_LE);
+		 | CODING_CATEGORY_MASK_UTF_16_LE);
 	  /* Or, if C is a special latin extra code,
 	     or is an ISO2022 specific control code of C1 (SS2 or SS3),
 	     or is an ISO2022 control-sequence-introducer (CSI),
 	     we should also consider the possibility of ISO2022 codings.  */
-	  if ((VECTORP (Vlatin_extra_code_table)
+	  if ((latin_extra_code_state
+	       && VECTORP (Vlatin_extra_code_table)
 	       && !NILP (XVECTOR (Vlatin_extra_code_table)->contents[c]))
 	      || (c == ISO_CODE_SS2 || c == ISO_CODE_SS3)
 	      || (c == ISO_CODE_CSI
 		  && (src < src_end
 		      && (*src == ']'
 			  || ((*src == '0' || *src == '1' || *src == '2')
 			      && src + 1 < src_end
 			      && src[1] == ']')))))
 	    try |= (CODING_CATEGORY_MASK_ISO_8_ELSE
-		     | CODING_CATEGORY_MASK_ISO_8BIT);
+		    | CODING_CATEGORY_MASK_ISO_8BIT);
 	}
 else
 	/* C is a character of ISO2022 in graphic plane right,
 	   or a SJIS's 1-byte character code (i.e. JISX0201),
 	   or the first byte of BIG5's 2-byte code,
 	   or the first byte of UTF-8/16.  */
 	try = (CODING_CATEGORY_MASK_ISO_8_ELSE
-		| CODING_CATEGORY_MASK_ISO_8BIT
+	       | CODING_CATEGORY_MASK_ISO_8BIT
-		| CODING_CATEGORY_MASK_SJIS
+	       | CODING_CATEGORY_MASK_SJIS
-		| CODING_CATEGORY_MASK_BIG5
+	       | CODING_CATEGORY_MASK_BIG5
-	        | CODING_CATEGORY_MASK_UTF_8
+	       | CODING_CATEGORY_MASK_UTF_8
-	        | CODING_CATEGORY_MASK_UTF_16_BE
+	       | CODING_CATEGORY_MASK_UTF_16_BE
-	        | CODING_CATEGORY_MASK_UTF_16_LE);
+	       | CODING_CATEGORY_MASK_UTF_16_LE);
 /* Or, we may have to consider the possibility of CCL.  */
-if (coding_system_table[CODING_CATEGORY_IDX_CCL]
+if (! null_byte_found
+	  && coding_system_table[CODING_CATEGORY_IDX_CCL]
 	  && (coding_system_table[CODING_CATEGORY_IDX_CCL]
 	      ->spec.ccl.valid_codes)[c])
 	try |= CODING_CATEGORY_MASK_CCL;
 mask = 0;
-utf16_examined_p = iso2022_examined_p = 0;
 if (priorities)
 	{
+	  /* At first try detection with Latin extra codes not-allowed.
+	     If no proper coding system is found because of Latin extra
+	     codes, try detection with Latin extra codes allowed.  */
+	  latin_extra_code_state = 0;
+	label_retry:
+	  utf16_examined_p = iso2022_examined_p = 0;
 	  for (i = 0; i < CODING_CATEGORY_IDX_MAX; i++)
 	    {
 	      if (!iso2022_examined_p
 		  && (priorities[i] & try & CODING_CATEGORY_MASK_ISO))
 		{
-		  mask |= detect_coding_iso2022 (src, src_end, multibytep);
+		  mask |= detect_coding_iso2022 (src, src_end, multibytep,
+						 &latin_extra_code_state);
 		  iso2022_examined_p = 1;
 		}
 	      else if (priorities[i] & try & CODING_CATEGORY_MASK_SJIS)
 		mask |= detect_coding_sjis (src, src_end, multibytep);
 	      else if (priorities[i] & try & CODING_CATEGORY_MASK_UTF_8)
 	      else if (priorities[i] & try & CODING_CATEGORY_MASK_EMACS_MULE)
 		mask |= detect_coding_emacs_mule (src, src_end, multibytep);
 	      else if (priorities[i] & try & CODING_CATEGORY_MASK_CCL)
 		mask |= detect_coding_ccl (src, src_end, multibytep);
 	      else if (priorities[i] & CODING_CATEGORY_MASK_RAW_TEXT)
-		mask |= CODING_CATEGORY_MASK_RAW_TEXT;
+		{
+		  if (latin_extra_code_state == 1)
+		    {
+		      /* Detection of ISO-2022 based coding system
+			 failed because of Latin extra codes.  Before
+			 falling back to raw-text, try again with
+			 Latin extra codes allowed.  */
+		      latin_extra_code_state = 2;
+		      try = (mask | CODING_CATEGORY_MASK_ISO_8_ELSE
+			     | CODING_CATEGORY_MASK_ISO_8BIT);
+		      goto label_retry;
+		    }
+		  mask |= CODING_CATEGORY_MASK_RAW_TEXT;
+		}
 	      else if (priorities[i] & CODING_CATEGORY_MASK_BINARY)
-		mask |= CODING_CATEGORY_MASK_BINARY;
+		{
+		  if (latin_extra_code_state == 1)
+		    {
+		      /* See the above comment.  */
+		      latin_extra_code_state = 2;
+		      try = (mask | CODING_CATEGORY_MASK_ISO_8_ELSE
+			     | CODING_CATEGORY_MASK_ISO_8BIT);
+		      goto label_retry;
+		    }
+		  mask |= CODING_CATEGORY_MASK_BINARY;
+		}
 	      if (mask & priorities[i])
 		return priorities[i];
 	    }
 	  return CODING_CATEGORY_MASK_RAW_TEXT;
 	}
 if (try & CODING_CATEGORY_MASK_ISO)
-	mask |= detect_coding_iso2022 (src, src_end, multibytep);
+	mask |= detect_coding_iso2022 (src, src_end, multibytep,
+				       &latin_extra_code_state);
 if (try & CODING_CATEGORY_MASK_SJIS)
 	mask |= detect_coding_sjis (src, src_end, multibytep);
 if (try & CODING_CATEGORY_MASK_BIG5)
 	mask |= detect_coding_big5 (src, src_end, multibytep);
 if (try & CODING_CATEGORY_MASK_UTF_8)

Mercurial > emacs

comparison src/coding.c @ 87676:2aeceff24280