emacs: src/coding.c comparison

comparison src/coding.c @ 93595:ac4d127a841a

(CATEGORY_MASK_ANY): Add CATEGORY_MASK_UTF_16_AUTO. (CATEGORY_MASK_UTF_16): Likewise. (detect_coding_utf_16): Add heuristics to reject utf-16 for a binary file. (detect_coding): Add null-byte detection for a binary file. (detect_coding_system): Likewise.

author	Kenichi Handa <handa@m17n.org>
date	Thu, 03 Apr 2008 12:30:02 +0000
parents	06e93ffa2e9f
children	62d97ebb13a9

comparison

equal deleted inserted replaced

-:46b8fe649bbc
+:ac4d127a841a
 | CATEGORY_MASK_ISO_8_1		\
 | CATEGORY_MASK_ISO_8_2		\
 | CATEGORY_MASK_ISO_7_ELSE		\
 | CATEGORY_MASK_ISO_8_ELSE		\
 | CATEGORY_MASK_UTF_8		\
+| CATEGORY_MASK_UTF_16_AUTO		\
 | CATEGORY_MASK_UTF_16_BE		\
 | CATEGORY_MASK_UTF_16_LE		\
 | CATEGORY_MASK_UTF_16_BE_NOSIG	\
 | CATEGORY_MASK_UTF_16_LE_NOSIG	\
 | CATEGORY_MASK_CHARSET		\
 (  CATEGORY_MASK_ISO_7BIT	\
 | CATEGORY_MASK_ISO_8BIT	\
 | CATEGORY_MASK_ISO_ELSE)
 #define CATEGORY_MASK_UTF_16		\
-(CATEGORY_MASK_UTF_16_BE		\
+(CATEGORY_MASK_UTF_16_AUTO		\
+| CATEGORY_MASK_UTF_16_BE		\
 | CATEGORY_MASK_UTF_16_LE		\
 | CATEGORY_MASK_UTF_16_BE_NOSIG	\
 | CATEGORY_MASK_UTF_16_LE_NOSIG)
 			     | CATEGORY_MASK_UTF_16_AUTO);
 detect_info->rejected |= (CATEGORY_MASK_UTF_16_LE
 				| CATEGORY_MASK_UTF_16_BE_NOSIG
 				| CATEGORY_MASK_UTF_16_LE_NOSIG);
 }
-else if (c1 >= 0 && c2 >= 0)
+else
 {
+/* We check the dispersion of Eth and Oth bytes where E is even and
+	 O is odd.  If both are high, we assume binary data.*/
+unsigned char e[256], o[256];
+unsigned e_num = 1, o_num = 1;
+memset (e, 0, 256);
+memset (o, 0, 256);
+e[c1] = 1;
+o[c2] = 1;
 detect_info->rejected
 	|= (CATEGORY_MASK_UTF_16_BE | CATEGORY_MASK_UTF_16_LE);
-}
+while (1)
+	{
+	  ONE_MORE_BYTE (c1);
+	  ONE_MORE_BYTE (c2);
+	  if (! e[c1])
+	    {
+	      e[c1] = 1;
+	      e_num++;
+	      if (e_num >= 128)
+		break;
+	    }
+	  if (! o[c2])
+	    {
+	      o[c1] = 1;
+	      o_num++;
+	      if (o_num >= 128)
+		break;
+	    }
+	}
+detect_info->rejected |= CATEGORY_MASK_UTF_16;
+return 0;
+}
 no_more_source:
 return 1;
 }
 static void
 now.  */
 if (EQ (CODING_ATTR_TYPE (CODING_ID_ATTRS (coding->id)), Qundecided))
 {
 int c, i;
 struct coding_detection_info detect_info;
+int null_byte_found = 0, eight_bit_found = 0;
 detect_info.checked = detect_info.found = detect_info.rejected = 0;
-for (i = 0, src = coding->source; src < src_end; i++, src++)
+coding->head_ascii = -1;
+for (src = coding->source; src < src_end; src++)
 	{
 	  c = *src;
 	  if (c & 0x80)
-	    break;
-	  if (c < 0x20
-	      && (c == ISO_CODE_ESC || c == ISO_CODE_SI || c == ISO_CODE_SO)
-	      && ! inhibit_iso_escape_detection
-	      && ! detect_info.checked)
 	    {
-	      coding->head_ascii = src - (coding->source + coding->consumed);
+	      eight_bit_found = 1;
-	      if (detect_coding_iso_2022 (coding, &detect_info))
+	      if (coding->head_ascii < 0)
+		coding->head_ascii = src - coding->source;
+	      if (null_byte_found)
+		break;
+	    }
+	  else if (c < 0x20)
+	    {
+	      if ((c == ISO_CODE_ESC || c == ISO_CODE_SI || c == ISO_CODE_SO)
+		  && ! inhibit_iso_escape_detection
+		  && ! detect_info.checked)
 		{
-		  /* We have scanned the whole data.  */
+		  if (coding->head_ascii < 0)
-		  if (! (detect_info.rejected & CATEGORY_MASK_ISO_7_ELSE))
+		    coding->head_ascii = src - coding->source;
-		    /* We didn't find an 8-bit code.  */
+		  if (detect_coding_iso_2022 (coding, &detect_info))
-		    src = src_end;
+		    {
-		  break;
+		      /* We have scanned the whole data.  */
+		      if (! (detect_info.rejected & CATEGORY_MASK_ISO_7_ELSE))
+			/* We didn't find an 8-bit code.  We may have
+			   found a null-byte, but it's very rare that
+			   a binary file confirm to ISO-2022.  */
+			src = src_end;
+		      break;
+		    }
+		}
+	      else if (! c)
+		{
+		  null_byte_found = 1;
+		  if (eight_bit_found)
+		    break;
 		}
 	    }
 	}
-coding->head_ascii = src - (coding->source + coding->consumed);
+if (coding->head_ascii < 0)
+	coding->head_ascii = src - coding->source;
-if (coding->head_ascii < coding->src_bytes
+if (null_byte_found || eight_bit_found
+	  || coding->head_ascii < coding->src_bytes
 	  || detect_info.found)
 	{
 	  enum coding_category category;
 	  struct coding_system *this;
 		this = coding_categories + category;
 		if (detect_info.found & (1 << category))
 		  break;
 	      }
 	  else
-	    for (i = 0; i < coding_category_raw_text; i++)
+	    {
-	      {
+	      if (null_byte_found)
-		category = coding_priorities[i];
+		{
-		this = coding_categories + category;
+		  detect_info.checked |= ~CATEGORY_MASK_UTF_16;
-		if (this->id < 0)
+		  detect_info.rejected |= ~CATEGORY_MASK_UTF_16;
-		  {
+		}
-		    /* No coding system of this category is defined.  */
+	      for (i = 0; i < coding_category_raw_text; i++)
-		    detect_info.rejected |= (1 << category);
+		{
-		  }
+		  category = coding_priorities[i];
-		else if (category >= coding_category_raw_text)
+		  this = coding_categories + category;
-		  continue;
+		  if (this->id < 0)
-		else if (detect_info.checked & (1 << category))
+		    {
-		  {
+		      /* No coding system of this category is defined.  */
-		    if (detect_info.found & (1 << category))
+		      detect_info.rejected |= (1 << category);
+		    }
+		  else if (category >= coding_category_raw_text)
+		    continue;
+		  else if (detect_info.checked & (1 << category))
+		    {
+		      if (detect_info.found & (1 << category))
+			break;
+		    }
+		  else if ((*(this->detector)) (coding, &detect_info)
+			   && detect_info.found & (1 << category))
+		    {
+		      if (category == coding_category_utf_16_auto)
+			{
+			  if (detect_info.found & CATEGORY_MASK_UTF_16_LE)
+			    category = coding_category_utf_16_le;
+			  else
+			    category = coding_category_utf_16_be;
+			}
 		      break;
-		  }
+		    }
-		else if ((*(this->detector)) (coding, &detect_info)
+		}
-			 && detect_info.found & (1 << category))
-		  {
-		    if (category == coding_category_utf_16_auto)
-		      {
-			if (detect_info.found & CATEGORY_MASK_UTF_16_LE)
-			  category = coding_category_utf_16_le;
-			else
-			  category = coding_category_utf_16_be;
-		      }
-		    break;
-		  }
-	      }
-	  if (i < coding_category_raw_text)
+	      if (i < coding_category_raw_text)
-	    setup_coding_system (CODING_ID_NAME (this->id), coding);
+		setup_coding_system (CODING_ID_NAME (this->id), coding);
-	  else if (detect_info.rejected == CATEGORY_MASK_ANY)
+	      else if (null_byte_found)
-	    setup_coding_system (Qraw_text, coding);
+		setup_coding_system (Qno_conversion, coding);
-	  else if (detect_info.rejected)
+	      else if ((detect_info.rejected & CATEGORY_MASK_ANY)
-	    for (i = 0; i < coding_category_raw_text; i++)
+		       == CATEGORY_MASK_ANY)
-	      if (! (detect_info.rejected & (1 << coding_priorities[i])))
+		setup_coding_system (Qraw_text, coding);
-		{
+	      else if (detect_info.rejected)
-		  this = coding_categories + coding_priorities[i];
+		for (i = 0; i < coding_category_raw_text; i++)
-		  setup_coding_system (CODING_ID_NAME (this->id), coding);
+		  if (! (detect_info.rejected & (1 << coding_priorities[i])))
-		  break;
+		    {
-		}
+		      this = coding_categories + coding_priorities[i];
+		      setup_coding_system (CODING_ID_NAME (this->id), coding);
+		      break;
+		    }
+	    }
 	}
 }
 else if (XINT (CODING_ATTR_CATEGORY (CODING_ID_ATTRS (coding->id)))
 	   == coding_category_utf_16_auto)
 {
 Lisp_Object val;
 struct coding_system coding;
 int id;
 struct coding_detection_info detect_info;
 enum coding_category base_category;
+int null_byte_found = 0, eight_bit_found = 0;
 if (NILP (coding_system))
 coding_system = Qundecided;
 setup_coding_system (coding_system, &coding);
 attrs = CODING_ID_ATTRS (coding.id);
 {
 enum coding_category category;
 struct coding_system *this;
 int c, i;
+coding.head_ascii = -1;
 /* Skip all ASCII bytes except for a few ISO2022 controls.  */
-for (i = 0; src < src_end; i++, src++)
+for (; src < src_end; src++)
 	{
 	  c = *src;
 	  if (c & 0x80)
-	    break;
-	  if (c < 0x20
-	      && (c == ISO_CODE_ESC || c == ISO_CODE_SI || c == ISO_CODE_SO)
-	      && ! inhibit_iso_escape_detection)
 	    {
-	      coding.head_ascii = src - coding.source;
+	      eight_bit_found = 1;
-	      if (detect_coding_iso_2022 (&coding, &detect_info))
+	      if (coding.head_ascii < 0)
+		coding.head_ascii = src - coding.source;
+	      if (null_byte_found)
+		break;
+	    }
+	  if (c < 0x20)
+	    {
+	      if ((c == ISO_CODE_ESC || c == ISO_CODE_SI || c == ISO_CODE_SO)
+		  && ! inhibit_iso_escape_detection
+		  && ! detect_info.checked)
 		{
-		  /* We have scanned the whole data.  */
+		  if (coding.head_ascii < 0)
-		  if (! (detect_info.rejected & CATEGORY_MASK_ISO_7_ELSE))
+		    coding.head_ascii = src - coding.source;
-		    /* We didn't find an 8-bit code.  */
+		  if (detect_coding_iso_2022 (&coding, &detect_info))
-		    src = src_end;
+		    {
-		  break;
+		      /* We have scanned the whole data.  */
+		      if (! (detect_info.rejected & CATEGORY_MASK_ISO_7_ELSE))
+			/* We didn't find an 8-bit code.  We may have
+			   found a null-byte, but it's very rare that
+			   a binary file confirm to ISO-2022.  */
+			src = src_end;
+		      break;
+		    }
+		}
+	      else if (! c)
+		{
+		  null_byte_found = 1;
+		  if (eight_bit_found)
+		    break;
 		}
 	    }
 	}
-coding.head_ascii = src - coding.source;
+if (coding.head_ascii < 0)
+	coding.head_ascii = src - coding.source;
-if (src < src_end
+if (null_byte_found || eight_bit_found
+	  || coding.head_ascii < coding.src_bytes
 	  || detect_info.found)
 	{
-	  if (src == src_end)
+	  if (coding.head_ascii == coding.src_bytes)
 	    /* As all bytes are 7-bit, we can ignore non-ISO-2022 codings.  */
 	    for (i = 0; i < coding_category_raw_text; i++)
 	      {
 		category = coding_priorities[i];
 		this = coding_categories + category;
 		if (detect_info.found & (1 << category))
 		  break;
 	      }
 	  else
-	    for (i = 0; i < coding_category_raw_text; i++)
+	    {
-	      {
+	      if (null_byte_found)
-		category = coding_priorities[i];
+		{
-		this = coding_categories + category;
+		  detect_info.checked |= ~CATEGORY_MASK_UTF_16;
+		  detect_info.rejected |= ~CATEGORY_MASK_UTF_16;
-		if (this->id < 0)
+		}
-		  {
+	      for (i = 0; i < coding_category_raw_text; i++)
-		    /* No coding system of this category is defined.  */
+		{
-		    detect_info.rejected |= (1 << category);
+		  category = coding_priorities[i];
-		  }
+		  this = coding_categories + category;
-		else if (category >= coding_category_raw_text)
-		  continue;
+		  if (this->id < 0)
-		else if (detect_info.checked & (1 << category))
+		    {
-		  {
+		      /* No coding system of this category is defined.  */
-		    if (highest
+		      detect_info.rejected |= (1 << category);
-			&& (detect_info.found & (1 << category)))
+		    }
+		  else if (category >= coding_category_raw_text)
+		    continue;
+		  else if (detect_info.checked & (1 << category))
+		    {
+		      if (highest
+			  && (detect_info.found & (1 << category)))
+			break;
+		    }
+		  else if ((*(this->detector)) (&coding, &detect_info)
+			   && highest
+			   && (detect_info.found & (1 << category)))
+		    {
+		      if (category == coding_category_utf_16_auto)
+			{
+			  if (detect_info.found & CATEGORY_MASK_UTF_16_LE)
+			    category = coding_category_utf_16_le;
+			  else
+			    category = coding_category_utf_16_be;
+			}
 		      break;
-		  }
+		    }
-		else
+		}
-		  {
+	    }
-		    if ((*(this->detector)) (&coding, &detect_info)
+	}
-			&& highest
-			&& (detect_info.found & (1 << category)))
+if ((detect_info.rejected & CATEGORY_MASK_ANY) == CATEGORY_MASK_ANY)
-		      {
-			if (category == coding_category_utf_16_auto)
-			  {
-			    if (detect_info.found & CATEGORY_MASK_UTF_16_LE)
-			      category = coding_category_utf_16_le;
-			    else
-			      category = coding_category_utf_16_be;
-			  }
-			break;
-		      }
-		  }
-	      }
-	}
-if (detect_info.rejected == CATEGORY_MASK_ANY)
 	{
 	  detect_info.found = CATEGORY_MASK_RAW_TEXT;
 	  id = coding_categories[coding_category_raw_text].id;
 	  val = Fcons (make_number (id), Qnil);
 	}
 Lisp_Object tail;
 if (VECTORP (eol_type))
 {
 	if (detect_info.found & ~CATEGORY_MASK_UTF_16)
-	  normal_eol = detect_eol (coding.source, src_bytes,
+	  {
-				   coding_category_raw_text);
+	    if (null_byte_found)
+	      normal_eol = EOL_SEEN_LF;
+	    else
+	      normal_eol = detect_eol (coding.source, src_bytes,
+				       coding_category_raw_text);
+	  }
 	if (detect_info.found & (CATEGORY_MASK_UTF_16_BE
 				 | CATEGORY_MASK_UTF_16_BE_NOSIG))
 	  utf_16_be_eol = detect_eol (coding.source, src_bytes,
 				      coding_category_utf_16_be);
 	if (detect_info.found & (CATEGORY_MASK_UTF_16_LE

Mercurial > emacs

comparison src/coding.c @ 93595:ac4d127a841a