encoding.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658
  1. /*
  2. * Copyright (c) Ian F. Darwin 1986-1995.
  3. * Software written by Ian F. Darwin and others;
  4. * maintained 1995-present by Christos Zoulas and others.
  5. *
  6. * Redistribution and use in source and binary forms, with or without
  7. * modification, are permitted provided that the following conditions
  8. * are met:
  9. * 1. Redistributions of source code must retain the above copyright
  10. * notice immediately at the beginning of the file, without modification,
  11. * this list of conditions, and the following disclaimer.
  12. * 2. Redistributions in binary form must reproduce the above copyright
  13. * notice, this list of conditions and the following disclaimer in the
  14. * documentation and/or other materials provided with the distribution.
  15. *
  16. * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17. * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18. * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19. * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
  20. * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21. * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22. * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23. * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24. * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25. * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26. * SUCH DAMAGE.
  27. */
  28. /*
  29. * Encoding -- determine the character encoding of a text file.
  30. *
  31. * Joerg Wunsch <joerg@freebsd.org> wrote the original support for 8-bit
  32. * international characters.
  33. */
  34. #include "file.h"
  35. #ifndef lint
  36. FILE_RCSID("@(#)$File: encoding.c,v 1.39 2022/09/13 18:46:07 christos Exp $")
  37. #endif /* lint */
  38. #include "magic.h"
  39. #include <string.h>
  40. #include <stdlib.h>
  41. private int looks_ascii(const unsigned char *, size_t, file_unichar_t *,
  42. size_t *);
  43. private int looks_utf8_with_BOM(const unsigned char *, size_t, file_unichar_t *,
  44. size_t *);
  45. private int looks_utf7(const unsigned char *, size_t, file_unichar_t *,
  46. size_t *);
  47. private int looks_ucs16(const unsigned char *, size_t, file_unichar_t *,
  48. size_t *);
  49. private int looks_ucs32(const unsigned char *, size_t, file_unichar_t *,
  50. size_t *);
  51. private int looks_latin1(const unsigned char *, size_t, file_unichar_t *,
  52. size_t *);
  53. private int looks_extended(const unsigned char *, size_t, file_unichar_t *,
  54. size_t *);
  55. private void from_ebcdic(const unsigned char *, size_t, unsigned char *);
  56. #ifdef DEBUG_ENCODING
  57. #define DPRINTF(a) printf a
  58. #else
  59. #define DPRINTF(a)
  60. #endif
  61. /*
  62. * Try to determine whether text is in some character code we can
  63. * identify. Each of these tests, if it succeeds, will leave
  64. * the text converted into one-file_unichar_t-per-character Unicode in
  65. * ubuf, and the number of characters converted in ulen.
  66. */
  67. protected int
  68. file_encoding(struct magic_set *ms, const struct buffer *b,
  69. file_unichar_t **ubuf, size_t *ulen, const char **code,
  70. const char **code_mime, const char **type)
  71. {
  72. const unsigned char *buf = CAST(const unsigned char *, b->fbuf);
  73. size_t nbytes = b->flen;
  74. size_t mlen;
  75. int rv = 1, ucs_type;
  76. file_unichar_t *udefbuf;
  77. size_t udeflen;
  78. if (ubuf == NULL)
  79. ubuf = &udefbuf;
  80. if (ulen == NULL)
  81. ulen = &udeflen;
  82. *type = "text";
  83. *ulen = 0;
  84. *code = "unknown";
  85. *code_mime = "binary";
  86. if (nbytes > ms->encoding_max)
  87. nbytes = ms->encoding_max;
  88. mlen = (nbytes + 1) * sizeof((*ubuf)[0]);
  89. *ubuf = CAST(file_unichar_t *, calloc(CAST(size_t, 1), mlen));
  90. if (*ubuf == NULL) {
  91. file_oomem(ms, mlen);
  92. goto done;
  93. }
  94. if (looks_ascii(buf, nbytes, *ubuf, ulen)) {
  95. if (looks_utf7(buf, nbytes, *ubuf, ulen) > 0) {
  96. DPRINTF(("utf-7 %" SIZE_T_FORMAT "u\n", *ulen));
  97. *code = "Unicode text, UTF-7";
  98. *code_mime = "utf-7";
  99. } else {
  100. DPRINTF(("ascii %" SIZE_T_FORMAT "u\n", *ulen));
  101. *code = "ASCII";
  102. *code_mime = "us-ascii";
  103. }
  104. } else if (looks_utf8_with_BOM(buf, nbytes, *ubuf, ulen) > 0) {
  105. DPRINTF(("utf8/bom %" SIZE_T_FORMAT "u\n", *ulen));
  106. *code = "Unicode text, UTF-8 (with BOM)";
  107. *code_mime = "utf-8";
  108. } else if (file_looks_utf8(buf, nbytes, *ubuf, ulen) > 1) {
  109. DPRINTF(("utf8 %" SIZE_T_FORMAT "u\n", *ulen));
  110. *code = "Unicode text, UTF-8";
  111. *code_mime = "utf-8";
  112. } else if ((ucs_type = looks_ucs32(buf, nbytes, *ubuf, ulen)) != 0) {
  113. if (ucs_type == 1) {
  114. *code = "Unicode text, UTF-32, little-endian";
  115. *code_mime = "utf-32le";
  116. } else {
  117. *code = "Unicode text, UTF-32, big-endian";
  118. *code_mime = "utf-32be";
  119. }
  120. DPRINTF(("ucs32 %" SIZE_T_FORMAT "u\n", *ulen));
  121. } else if ((ucs_type = looks_ucs16(buf, nbytes, *ubuf, ulen)) != 0) {
  122. if (ucs_type == 1) {
  123. *code = "Unicode text, UTF-16, little-endian";
  124. *code_mime = "utf-16le";
  125. } else {
  126. *code = "Unicode text, UTF-16, big-endian";
  127. *code_mime = "utf-16be";
  128. }
  129. DPRINTF(("ucs16 %" SIZE_T_FORMAT "u\n", *ulen));
  130. } else if (looks_latin1(buf, nbytes, *ubuf, ulen)) {
  131. DPRINTF(("latin1 %" SIZE_T_FORMAT "u\n", *ulen));
  132. *code = "ISO-8859";
  133. *code_mime = "iso-8859-1";
  134. } else if (looks_extended(buf, nbytes, *ubuf, ulen)) {
  135. DPRINTF(("extended %" SIZE_T_FORMAT "u\n", *ulen));
  136. *code = "Non-ISO extended-ASCII";
  137. *code_mime = "unknown-8bit";
  138. } else {
  139. unsigned char *nbuf;
  140. mlen = (nbytes + 1) * sizeof(nbuf[0]);
  141. if ((nbuf = CAST(unsigned char *, malloc(mlen))) == NULL) {
  142. file_oomem(ms, mlen);
  143. goto done;
  144. }
  145. from_ebcdic(buf, nbytes, nbuf);
  146. if (looks_ascii(nbuf, nbytes, *ubuf, ulen)) {
  147. DPRINTF(("ebcdic %" SIZE_T_FORMAT "u\n", *ulen));
  148. *code = "EBCDIC";
  149. *code_mime = "ebcdic";
  150. } else if (looks_latin1(nbuf, nbytes, *ubuf, ulen)) {
  151. DPRINTF(("ebcdic/international %" SIZE_T_FORMAT "u\n",
  152. *ulen));
  153. *code = "International EBCDIC";
  154. *code_mime = "ebcdic";
  155. } else { /* Doesn't look like text at all */
  156. DPRINTF(("binary\n"));
  157. rv = 0;
  158. *type = "binary";
  159. }
  160. free(nbuf);
  161. }
  162. done:
  163. if (ubuf == &udefbuf)
  164. free(udefbuf);
  165. return rv;
  166. }
  167. /*
  168. * This table reflects a particular philosophy about what constitutes
  169. * "text," and there is room for disagreement about it.
  170. *
  171. * Version 3.31 of the file command considered a file to be ASCII if
  172. * each of its characters was approved by either the isascii() or
  173. * isalpha() function. On most systems, this would mean that any
  174. * file consisting only of characters in the range 0x00 ... 0x7F
  175. * would be called ASCII text, but many systems might reasonably
  176. * consider some characters outside this range to be alphabetic,
  177. * so the file command would call such characters ASCII. It might
  178. * have been more accurate to call this "considered textual on the
  179. * local system" than "ASCII."
  180. *
  181. * It considered a file to be "International language text" if each
  182. * of its characters was either an ASCII printing character (according
  183. * to the real ASCII standard, not the above test), a character in
  184. * the range 0x80 ... 0xFF, or one of the following control characters:
  185. * backspace, tab, line feed, vertical tab, form feed, carriage return,
  186. * escape. No attempt was made to determine the language in which files
  187. * of this type were written.
  188. *
  189. *
  190. * The table below considers a file to be ASCII if all of its characters
  191. * are either ASCII printing characters (again, according to the X3.4
  192. * standard, not isascii()) or any of the following controls: bell,
  193. * backspace, tab, line feed, form feed, carriage return, esc, nextline.
  194. *
  195. * I include bell because some programs (particularly shell scripts)
  196. * use it literally, even though it is rare in normal text. I exclude
  197. * vertical tab because it never seems to be used in real text. I also
  198. * include, with hesitation, the X3.64/ECMA-43 control nextline (0x85),
  199. * because that's what the dd EBCDIC->ASCII table maps the EBCDIC newline
  200. * character to. It might be more appropriate to include it in the 8859
  201. * set instead of the ASCII set, but it's got to be included in *something*
  202. * we recognize or EBCDIC files aren't going to be considered textual.
  203. * Some old Unix source files use SO/SI (^N/^O) to shift between Greek
  204. * and Latin characters, so these should possibly be allowed. But they
  205. * make a real mess on VT100-style displays if they're not paired properly,
  206. * so we are probably better off not calling them text.
  207. *
  208. * A file is considered to be ISO-8859 text if its characters are all
  209. * either ASCII, according to the above definition, or printing characters
  210. * from the ISO-8859 8-bit extension, characters 0xA0 ... 0xFF.
  211. *
  212. * Finally, a file is considered to be international text from some other
  213. * character code if its characters are all either ISO-8859 (according to
  214. * the above definition) or characters in the range 0x80 ... 0x9F, which
  215. * ISO-8859 considers to be control characters but the IBM PC and Macintosh
  216. * consider to be printing characters.
  217. */
  218. #define F 0 /* character never appears in text */
  219. #define T 1 /* character appears in plain ASCII text */
  220. #define I 2 /* character appears in ISO-8859 text */
  221. #define X 3 /* character appears in non-ISO extended ASCII (Mac, IBM PC) */
  222. private char text_chars[256] = {
  223. /* BEL BS HT LF VT FF CR */
  224. F, F, F, F, F, F, F, T, T, T, T, T, T, T, F, F, /* 0x0X */
  225. /* ESC */
  226. F, F, F, F, F, F, F, F, F, F, F, T, F, F, F, F, /* 0x1X */
  227. T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x2X */
  228. T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x3X */
  229. T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x4X */
  230. T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x5X */
  231. T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x6X */
  232. T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, F, /* 0x7X */
  233. /* NEL */
  234. X, X, X, X, X, T, X, X, X, X, X, X, X, X, X, X, /* 0x8X */
  235. X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, /* 0x9X */
  236. I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xaX */
  237. I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xbX */
  238. I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xcX */
  239. I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xdX */
  240. I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xeX */
  241. I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I /* 0xfX */
  242. };
  243. #define LOOKS(NAME, COND) \
  244. private int \
  245. looks_ ## NAME(const unsigned char *buf, size_t nbytes, file_unichar_t *ubuf, \
  246. size_t *ulen) \
  247. { \
  248. size_t i; \
  249. \
  250. *ulen = 0; \
  251. \
  252. for (i = 0; i < nbytes; i++) { \
  253. int t = text_chars[buf[i]]; \
  254. \
  255. if (COND) \
  256. return 0; \
  257. \
  258. ubuf[(*ulen)++] = buf[i]; \
  259. } \
  260. return 1; \
  261. }
  262. LOOKS(ascii, t != T)
  263. LOOKS(latin1, t != T && t != I)
  264. LOOKS(extended, t != T && t != I && t != X)
  265. /*
  266. * Decide whether some text looks like UTF-8. Returns:
  267. *
  268. * -1: invalid UTF-8
  269. * 0: uses odd control characters, so doesn't look like text
  270. * 1: 7-bit text
  271. * 2: definitely UTF-8 text (valid high-bit set bytes)
  272. *
  273. * If ubuf is non-NULL on entry, text is decoded into ubuf, *ulen;
  274. * ubuf must be big enough!
  275. */
  276. // from: https://golang.org/src/unicode/utf8/utf8.go
  277. #define XX 0xF1 // invalid: size 1
  278. #define AS 0xF0 // ASCII: size 1
  279. #define S1 0x02 // accept 0, size 2
  280. #define S2 0x13 // accept 1, size 3
  281. #define S3 0x03 // accept 0, size 3
  282. #define S4 0x23 // accept 2, size 3
  283. #define S5 0x34 // accept 3, size 4
  284. #define S6 0x04 // accept 0, size 4
  285. #define S7 0x44 // accept 4, size 4
  286. #define LOCB 0x80
  287. #define HICB 0xBF
  288. // first is information about the first byte in a UTF-8 sequence.
  289. static const uint8_t first[] = {
  290. // 1 2 3 4 5 6 7 8 9 A B C D E F
  291. AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x00-0x0F
  292. AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x10-0x1F
  293. AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x20-0x2F
  294. AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x30-0x3F
  295. AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x40-0x4F
  296. AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x50-0x5F
  297. AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x60-0x6F
  298. AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, AS, // 0x70-0x7F
  299. // 1 2 3 4 5 6 7 8 9 A B C D E F
  300. XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0x80-0x8F
  301. XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0x90-0x9F
  302. XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0xA0-0xAF
  303. XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0xB0-0xBF
  304. XX, XX, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, // 0xC0-0xCF
  305. S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, S1, // 0xD0-0xDF
  306. S2, S3, S3, S3, S3, S3, S3, S3, S3, S3, S3, S3, S3, S4, S3, S3, // 0xE0-0xEF
  307. S5, S6, S6, S6, S7, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, XX, // 0xF0-0xFF
  308. };
  309. // acceptRange gives the range of valid values for the second byte in a UTF-8
  310. // sequence.
  311. struct accept_range {
  312. uint8_t lo; // lowest value for second byte.
  313. uint8_t hi; // highest value for second byte.
  314. } accept_ranges[16] = {
  315. // acceptRanges has size 16 to avoid bounds checks in the code that uses it.
  316. { LOCB, HICB },
  317. { 0xA0, HICB },
  318. { LOCB, 0x9F },
  319. { 0x90, HICB },
  320. { LOCB, 0x8F },
  321. };
  322. protected int
  323. file_looks_utf8(const unsigned char *buf, size_t nbytes, file_unichar_t *ubuf,
  324. size_t *ulen)
  325. {
  326. size_t i;
  327. int n;
  328. file_unichar_t c;
  329. int gotone = 0, ctrl = 0;
  330. if (ubuf)
  331. *ulen = 0;
  332. for (i = 0; i < nbytes; i++) {
  333. if ((buf[i] & 0x80) == 0) { /* 0xxxxxxx is plain ASCII */
  334. /*
  335. * Even if the whole file is valid UTF-8 sequences,
  336. * still reject it if it uses weird control characters.
  337. */
  338. if (text_chars[buf[i]] != T)
  339. ctrl = 1;
  340. if (ubuf)
  341. ubuf[(*ulen)++] = buf[i];
  342. } else if ((buf[i] & 0x40) == 0) { /* 10xxxxxx never 1st byte */
  343. return -1;
  344. } else { /* 11xxxxxx begins UTF-8 */
  345. int following;
  346. uint8_t x = first[buf[i]];
  347. const struct accept_range *ar =
  348. &accept_ranges[(unsigned int)x >> 4];
  349. if (x == XX)
  350. return -1;
  351. if ((buf[i] & 0x20) == 0) { /* 110xxxxx */
  352. c = buf[i] & 0x1f;
  353. following = 1;
  354. } else if ((buf[i] & 0x10) == 0) { /* 1110xxxx */
  355. c = buf[i] & 0x0f;
  356. following = 2;
  357. } else if ((buf[i] & 0x08) == 0) { /* 11110xxx */
  358. c = buf[i] & 0x07;
  359. following = 3;
  360. } else if ((buf[i] & 0x04) == 0) { /* 111110xx */
  361. c = buf[i] & 0x03;
  362. following = 4;
  363. } else if ((buf[i] & 0x02) == 0) { /* 1111110x */
  364. c = buf[i] & 0x01;
  365. following = 5;
  366. } else
  367. return -1;
  368. for (n = 0; n < following; n++) {
  369. i++;
  370. if (i >= nbytes)
  371. goto done;
  372. if (n == 0 &&
  373. (buf[i] < ar->lo || buf[i] > ar->hi))
  374. return -1;
  375. if ((buf[i] & 0x80) == 0 || (buf[i] & 0x40))
  376. return -1;
  377. c = (c << 6) + (buf[i] & 0x3f);
  378. }
  379. if (ubuf)
  380. ubuf[(*ulen)++] = c;
  381. gotone = 1;
  382. }
  383. }
  384. done:
  385. return ctrl ? 0 : (gotone ? 2 : 1);
  386. }
  387. /*
  388. * Decide whether some text looks like UTF-8 with BOM. If there is no
  389. * BOM, return -1; otherwise return the result of looks_utf8 on the
  390. * rest of the text.
  391. */
  392. private int
  393. looks_utf8_with_BOM(const unsigned char *buf, size_t nbytes,
  394. file_unichar_t *ubuf, size_t *ulen)
  395. {
  396. if (nbytes > 3 && buf[0] == 0xef && buf[1] == 0xbb && buf[2] == 0xbf)
  397. return file_looks_utf8(buf + 3, nbytes - 3, ubuf, ulen);
  398. else
  399. return -1;
  400. }
  401. private int
  402. looks_utf7(const unsigned char *buf, size_t nbytes, file_unichar_t *ubuf,
  403. size_t *ulen)
  404. {
  405. if (nbytes > 4 && buf[0] == '+' && buf[1] == '/' && buf[2] == 'v')
  406. switch (buf[3]) {
  407. case '8':
  408. case '9':
  409. case '+':
  410. case '/':
  411. if (ubuf)
  412. *ulen = 0;
  413. return 1;
  414. default:
  415. return -1;
  416. }
  417. else
  418. return -1;
  419. }
  420. #define UCS16_NOCHAR(c) ((c) >= 0xfdd0 && (c) <= 0xfdef)
  421. #define UCS16_HISURR(c) ((c) >= 0xd800 && (c) <= 0xdbff)
  422. #define UCS16_LOSURR(c) ((c) >= 0xdc00 && (c) <= 0xdfff)
  423. private int
  424. looks_ucs16(const unsigned char *bf, size_t nbytes, file_unichar_t *ubf,
  425. size_t *ulen)
  426. {
  427. int bigend;
  428. uint32_t hi;
  429. size_t i;
  430. if (nbytes < 2)
  431. return 0;
  432. if (bf[0] == 0xff && bf[1] == 0xfe)
  433. bigend = 0;
  434. else if (bf[0] == 0xfe && bf[1] == 0xff)
  435. bigend = 1;
  436. else
  437. return 0;
  438. *ulen = 0;
  439. hi = 0;
  440. for (i = 2; i + 1 < nbytes; i += 2) {
  441. uint32_t uc;
  442. if (bigend)
  443. uc = CAST(uint32_t,
  444. bf[i + 1] | (CAST(file_unichar_t, bf[i]) << 8));
  445. else
  446. uc = CAST(uint32_t,
  447. bf[i] | (CAST(file_unichar_t, bf[i + 1]) << 8));
  448. uc &= 0xffff;
  449. switch (uc) {
  450. case 0xfffe:
  451. case 0xffff:
  452. return 0;
  453. default:
  454. if (UCS16_NOCHAR(uc))
  455. return 0;
  456. break;
  457. }
  458. if (hi) {
  459. if (!UCS16_LOSURR(uc))
  460. return 0;
  461. uc = 0x10000 + 0x400 * (hi - 1) + (uc - 0xdc00);
  462. hi = 0;
  463. }
  464. if (uc < 128 && text_chars[CAST(size_t, uc)] != T)
  465. return 0;
  466. ubf[(*ulen)++] = uc;
  467. if (UCS16_HISURR(uc))
  468. hi = uc - 0xd800 + 1;
  469. if (UCS16_LOSURR(uc))
  470. return 0;
  471. }
  472. return 1 + bigend;
  473. }
  474. private int
  475. looks_ucs32(const unsigned char *bf, size_t nbytes, file_unichar_t *ubf,
  476. size_t *ulen)
  477. {
  478. int bigend;
  479. size_t i;
  480. if (nbytes < 4)
  481. return 0;
  482. if (bf[0] == 0xff && bf[1] == 0xfe && bf[2] == 0 && bf[3] == 0)
  483. bigend = 0;
  484. else if (bf[0] == 0 && bf[1] == 0 && bf[2] == 0xfe && bf[3] == 0xff)
  485. bigend = 1;
  486. else
  487. return 0;
  488. *ulen = 0;
  489. for (i = 4; i + 3 < nbytes; i += 4) {
  490. /* XXX fix to properly handle chars > 65536 */
  491. if (bigend)
  492. ubf[(*ulen)++] = CAST(file_unichar_t, bf[i + 3])
  493. | (CAST(file_unichar_t, bf[i + 2]) << 8)
  494. | (CAST(file_unichar_t, bf[i + 1]) << 16)
  495. | (CAST(file_unichar_t, bf[i]) << 24);
  496. else
  497. ubf[(*ulen)++] = CAST(file_unichar_t, bf[i + 0])
  498. | (CAST(file_unichar_t, bf[i + 1]) << 8)
  499. | (CAST(file_unichar_t, bf[i + 2]) << 16)
  500. | (CAST(file_unichar_t, bf[i + 3]) << 24);
  501. if (ubf[*ulen - 1] == 0xfffe)
  502. return 0;
  503. if (ubf[*ulen - 1] < 128 &&
  504. text_chars[CAST(size_t, ubf[*ulen - 1])] != T)
  505. return 0;
  506. }
  507. return 1 + bigend;
  508. }
  509. #undef F
  510. #undef T
  511. #undef I
  512. #undef X
  513. /*
  514. * This table maps each EBCDIC character to an (8-bit extended) ASCII
  515. * character, as specified in the rationale for the dd(1) command in
  516. * draft 11.2 (September, 1991) of the POSIX P1003.2 standard.
  517. *
  518. * Unfortunately it does not seem to correspond exactly to any of the
  519. * five variants of EBCDIC documented in IBM's _Enterprise Systems
  520. * Architecture/390: Principles of Operation_, SA22-7201-06, Seventh
  521. * Edition, July, 1999, pp. I-1 - I-4.
  522. *
  523. * Fortunately, though, all versions of EBCDIC, including this one, agree
  524. * on most of the printing characters that also appear in (7-bit) ASCII.
  525. * Of these, only '|', '!', '~', '^', '[', and ']' are in question at all.
  526. *
  527. * Fortunately too, there is general agreement that codes 0x00 through
  528. * 0x3F represent control characters, 0x41 a nonbreaking space, and the
  529. * remainder printing characters.
  530. *
  531. * This is sufficient to allow us to identify EBCDIC text and to distinguish
  532. * between old-style and internationalized examples of text.
  533. */
  534. private unsigned char ebcdic_to_ascii[] = {
  535. 0, 1, 2, 3, 156, 9, 134, 127, 151, 141, 142, 11, 12, 13, 14, 15,
  536. 16, 17, 18, 19, 157, 133, 8, 135, 24, 25, 146, 143, 28, 29, 30, 31,
  537. 128, 129, 130, 131, 132, 10, 23, 27, 136, 137, 138, 139, 140, 5, 6, 7,
  538. 144, 145, 22, 147, 148, 149, 150, 4, 152, 153, 154, 155, 20, 21, 158, 26,
  539. ' ', 160, 161, 162, 163, 164, 165, 166, 167, 168, 213, '.', '<', '(', '+', '|',
  540. '&', 169, 170, 171, 172, 173, 174, 175, 176, 177, '!', '$', '*', ')', ';', '~',
  541. '-', '/', 178, 179, 180, 181, 182, 183, 184, 185, 203, ',', '%', '_', '>', '?',
  542. 186, 187, 188, 189, 190, 191, 192, 193, 194, '`', ':', '#', '@', '\'','=', '"',
  543. 195, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 196, 197, 198, 199, 200, 201,
  544. 202, 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', '^', 204, 205, 206, 207, 208,
  545. 209, 229, 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 210, 211, 212, '[', 214, 215,
  546. 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, ']', 230, 231,
  547. '{', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 232, 233, 234, 235, 236, 237,
  548. '}', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 238, 239, 240, 241, 242, 243,
  549. '\\',159, 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 244, 245, 246, 247, 248, 249,
  550. '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 250, 251, 252, 253, 254, 255
  551. };
  552. #ifdef notdef
  553. /*
  554. * The following EBCDIC-to-ASCII table may relate more closely to reality,
  555. * or at least to modern reality. It comes from
  556. *
  557. * http://ftp.s390.ibm.com/products/oe/bpxqp9.html
  558. *
  559. * and maps the characters of EBCDIC code page 1047 (the code used for
  560. * Unix-derived software on IBM's 390 systems) to the corresponding
  561. * characters from ISO 8859-1.
  562. *
  563. * If this table is used instead of the above one, some of the special
  564. * cases for the NEL character can be taken out of the code.
  565. */
  566. private unsigned char ebcdic_1047_to_8859[] = {
  567. 0x00,0x01,0x02,0x03,0x9C,0x09,0x86,0x7F,0x97,0x8D,0x8E,0x0B,0x0C,0x0D,0x0E,0x0F,
  568. 0x10,0x11,0x12,0x13,0x9D,0x0A,0x08,0x87,0x18,0x19,0x92,0x8F,0x1C,0x1D,0x1E,0x1F,
  569. 0x80,0x81,0x82,0x83,0x84,0x85,0x17,0x1B,0x88,0x89,0x8A,0x8B,0x8C,0x05,0x06,0x07,
  570. 0x90,0x91,0x16,0x93,0x94,0x95,0x96,0x04,0x98,0x99,0x9A,0x9B,0x14,0x15,0x9E,0x1A,
  571. 0x20,0xA0,0xE2,0xE4,0xE0,0xE1,0xE3,0xE5,0xE7,0xF1,0xA2,0x2E,0x3C,0x28,0x2B,0x7C,
  572. 0x26,0xE9,0xEA,0xEB,0xE8,0xED,0xEE,0xEF,0xEC,0xDF,0x21,0x24,0x2A,0x29,0x3B,0x5E,
  573. 0x2D,0x2F,0xC2,0xC4,0xC0,0xC1,0xC3,0xC5,0xC7,0xD1,0xA6,0x2C,0x25,0x5F,0x3E,0x3F,
  574. 0xF8,0xC9,0xCA,0xCB,0xC8,0xCD,0xCE,0xCF,0xCC,0x60,0x3A,0x23,0x40,0x27,0x3D,0x22,
  575. 0xD8,0x61,0x62,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0xAB,0xBB,0xF0,0xFD,0xFE,0xB1,
  576. 0xB0,0x6A,0x6B,0x6C,0x6D,0x6E,0x6F,0x70,0x71,0x72,0xAA,0xBA,0xE6,0xB8,0xC6,0xA4,
  577. 0xB5,0x7E,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7A,0xA1,0xBF,0xD0,0x5B,0xDE,0xAE,
  578. 0xAC,0xA3,0xA5,0xB7,0xA9,0xA7,0xB6,0xBC,0xBD,0xBE,0xDD,0xA8,0xAF,0x5D,0xB4,0xD7,
  579. 0x7B,0x41,0x42,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0xAD,0xF4,0xF6,0xF2,0xF3,0xF5,
  580. 0x7D,0x4A,0x4B,0x4C,0x4D,0x4E,0x4F,0x50,0x51,0x52,0xB9,0xFB,0xFC,0xF9,0xFA,0xFF,
  581. 0x5C,0xF7,0x53,0x54,0x55,0x56,0x57,0x58,0x59,0x5A,0xB2,0xD4,0xD6,0xD2,0xD3,0xD5,
  582. 0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39,0xB3,0xDB,0xDC,0xD9,0xDA,0x9F
  583. };
  584. #endif
  585. /*
  586. * Copy buf[0 ... nbytes-1] into out[], translating EBCDIC to ASCII.
  587. */
  588. private void
  589. from_ebcdic(const unsigned char *buf, size_t nbytes, unsigned char *out)
  590. {
  591. size_t i;
  592. for (i = 0; i < nbytes; i++) {
  593. out[i] = ebcdic_to_ascii[buf[i]];
  594. }
  595. }