git Service of IN-Ulm e.V. Explore Help
Sign In
cbiedl
/
file
1
0
Fork 0
Files Pull Requests 0 Wiki
Tree: 9a00e188e5
Branches Tags
file
/
debian
/
patches
/
cherry-pick.FILE5_35-56-gf0a26da7.pr-61-tmc-add-ucs-32-built-in-detection.patch

cherry-pick.FILE5_35-56-gf0a26da7.pr-61-tmc-add-ucs-32-built-in-detection.patch 3.2 KB
History Raw

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117 
  1. Subject: PR/61: tmc: Add UCS-32 built-in detection
    2. 

  2. Origin: FILE5_35-56-gf0a26da7 <https://github.com/file/file/commit/FILE5_35-56-gf0a26da7>
    3. 

  3. Upstream-Author: Christos Zoulas <christos@zoulas.com>
    4. 

  4. Date: Tue Feb 19 20:30:35 2019 +0000
    5. 

  5. Comment: Prerequisite for FILE5_36-1-gecca6e54
    6. 

  6. 

  7. --- a/src/encoding.c
    8. 

  8. +++ b/src/encoding.c
    9. 

  9. @@ -49,6 +49,7 @@
    10. 

  10.      size_t *);
    11. 

  11.  private int looks_utf7(const unsigned char *, size_t, unichar *, size_t *);
    12. 

  12.  private int looks_ucs16(const unsigned char *, size_t, unichar *, size_t *);
    13. 

  13. +private int looks_ucs32(const unsigned char *, size_t, unichar *, size_t *);
    14. 

  14.  private int looks_latin1(const unsigned char *, size_t, unichar *, size_t *);
    15. 

  15.  private int looks_extended(const unsigned char *, size_t, unichar *, size_t *);
    16. 

  16.  private void from_ebcdic(const unsigned char *, size_t, unsigned char *);
    17. 

  17. @@ -116,6 +117,15 @@
    18. 

  18.  		DPRINTF(("utf8 %" SIZE_T_FORMAT "u\n", *ulen));
    19. 

  19.  		*code = "UTF-8 Unicode";
    20. 

  20.  		*code_mime = "utf-8";
    21. 

  21. +	} else if ((ucs_type = looks_ucs32(buf, nbytes, *ubuf, ulen)) != 0) {
    22. 

  22. +		if (ucs_type == 1) {
    23. 

  23. +			*code = "Little-endian UTF-32 Unicode";
    24. 

  24. +			*code_mime = "utf-32le";
    25. 

  25. +		} else {
    26. 

  26. +			*code = "Big-endian UTF-32 Unicode";
    27. 

  27. +			*code_mime = "utf-32be";
    28. 

  28. +		}
    29. 

  29. +		DPRINTF(("ucs32 %" SIZE_T_FORMAT "u\n", *ulen));
    30. 

  30.  	} else if ((ucs_type = looks_ucs16(buf, nbytes, *ubuf, ulen)) != 0) {
    31. 

  31.  		if (ucs_type == 1) {
    32. 

  32.  			*code = "Little-endian UTF-16 Unicode";
    33. 

  33. @@ -410,7 +420,7 @@
    34. 

  34.  }
    35. 

  35.  
    36. 

  36.  private int
    37. 

  37. -looks_ucs16(const unsigned char *buf, size_t nbytes, unichar *ubuf,
    38. 

  38. +looks_ucs16(const unsigned char *bf, size_t nbytes, unichar *ubf,
    39. 

  39.      size_t *ulen)
    40. 

  40.  {
    41. 

  41.  	int bigend;
    42. 

  42. @@ -419,9 +429,9 @@
    43. 

  43.  	if (nbytes < 2)
    44. 

  44.  		return 0;
    45. 

  45.  
    46. 

  46. -	if (buf[0] == 0xff && buf[1] == 0xfe)
    47. 

  47. +	if (bf[0] == 0xff && bf[1] == 0xfe)
    48. 

  48.  		bigend = 0;
    49. 

  49. -	else if (buf[0] == 0xfe && buf[1] == 0xff)
    50. 

  50. +	else if (bf[0] == 0xfe && bf[1] == 0xff)
    51. 

  51.  		bigend = 1;
    52. 

  52.  	else
    53. 

  53.  		return 0;
    54. 

  54. @@ -432,20 +442,58 @@
    55. 

  55.  		/* XXX fix to properly handle chars > 65536 */
    56. 

  56.  
    57. 

  57.  		if (bigend)
    58. 

  58. -			ubuf[(*ulen)++] = buf[i + 1] + 256 * buf[i];
    59. 

  59. +			ubf[(*ulen)++] = bf[i + 1] + 256 * bf[i];
    60. 

  60.  		else
    61. 

  61. -			ubuf[(*ulen)++] = buf[i] + 256 * buf[i + 1];
    62. 

  62. +			ubf[(*ulen)++] = bf[i] + 256 * bf[i + 1];
    63. 

  63.  
    64. 

  64. -		if (ubuf[*ulen - 1] == 0xfffe)
    65. 

  65. +		if (ubf[*ulen - 1] == 0xfffe)
    66. 

  66.  			return 0;
    67. 

  67. -		if (ubuf[*ulen - 1] < 128 &&
    68. 

  68. -		    text_chars[(size_t)ubuf[*ulen - 1]] != T)
    69. 

  69. +		if (ubf[*ulen - 1] < 128 &&
    70. 

  70. +		    text_chars[(size_t)ubf[*ulen - 1]] != T)
    71. 

  71.  			return 0;
    72. 

  72.  	}
    73. 

  73.  
    74. 

  74.  	return 1 + bigend;
    75. 

  75.  }
    76. 

  76.  
    77. 

  77. +private int
    78. 

  78. +looks_ucs32(const unsigned char *bf, size_t nbytes, unichar *ubf,
    79. 

  79. +    size_t *ulen)
    80. 

  80. +{
    81. 

  81. +	int bigend;
    82. 

  82. +	size_t i;
    83. 

  83. +
    84. 

  84. +	if (nbytes < 4)
    85. 

  85. +		return 0;
    86. 

  86. +
    87. 

  87. +	if (bf[0] == 0xff && bf[1] == 0xfe && bf[2] == 0 && bf[3] == 0)
    88. 

  88. +		bigend = 0;
    89. 

  89. +	else if (bf[0] == 0 && bf[1] == 0 && bf[2] == 0xfe && bf[3] == 0xff)
    90. 

  90. +		bigend = 1;
    91. 

  91. +	else
    92. 

  92. +		return 0;
    93. 

  93. +
    94. 

  94. +	*ulen = 0;
    95. 

  95. +
    96. 

  96. +	for (i = 4; i + 1 < nbytes; i += 4) {
    97. 

  97. +		/* XXX fix to properly handle chars > 65536 */
    98. 

  98. +
    99. 

  99. +		if (bigend)
    100. 

  100. +			ubf[(*ulen)++] = bf[i + 3] | (bf[i + 2] << 8)
    101. 

  101. +			    | (bf[i + 1] << 16) | bf[i] << 24;
    102. 

  102. +		else
    103. 

  103. +			ubf[(*ulen)++] = bf[i] | (bf[i + 1] << 8) 
    104. 

  104. +			    | (bf[i + 2] << 16) | (bf[i + 3] << 24);
    105. 

  105. +
    106. 

  106. +		if (ubf[*ulen - 1] == 0xfffe)
    107. 

  107. +			return 0;
    108. 

  108. +		if (ubf[*ulen - 1] < 128 &&
    109. 

  109. +		    text_chars[(size_t)ubf[*ulen - 1]] != T)
    110. 

  110. +			return 0;
    111. 

  111. +	}
    112. 

  112. +
    113. 

  113. +	return 1 + bigend;
    114. 

  114. +}
    115. 

  115.  #undef F
    116. 

  116.  #undef T
    117. 

  117.  #undef I
    118.