tokenize.c 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322
  1. /** \file tokenize.c
  2. *
  3. * Tokenize a string, accommodating quoted strings.
  4. *
  5. * @addtogroup autoopts
  6. * @{
  7. */
  8. /*
  9. * This file defines the string_tokenize interface
  10. * This file is part of AutoOpts, a companion to AutoGen.
  11. * AutoOpts is free software.
  12. * AutoOpts is Copyright (C) 1992-2018 by Bruce Korb - all rights reserved
  13. *
  14. * AutoOpts is available under any one of two licenses. The license
  15. * in use must be one of these two and the choice is under the control
  16. * of the user of the license.
  17. *
  18. * The GNU Lesser General Public License, version 3 or later
  19. * See the files "COPYING.lgplv3" and "COPYING.gplv3"
  20. *
  21. * The Modified Berkeley Software Distribution License
  22. * See the file "COPYING.mbsd"
  23. *
  24. * These files have the following sha256 sums:
  25. *
  26. * 8584710e9b04216a394078dc156b781d0b47e1729104d666658aecef8ee32e95 COPYING.gplv3
  27. * 4379e7444a0e2ce2b12dd6f5a52a27a4d02d39d247901d3285c88cf0d37f477b COPYING.lgplv3
  28. * 13aa749a5b0a454917a944ed8fffc530b784f5ead522b1aacaf4ec8aa55a6239 COPYING.mbsd
  29. */
  30. static void
  31. copy_cooked(ch_t ** ppDest, char const ** ppSrc)
  32. {
  33. ch_t * pDest = (ch_t *)*ppDest;
  34. const ch_t * pSrc = (const ch_t *)(*ppSrc + 1);
  35. for (;;) {
  36. ch_t ch = *(pSrc++);
  37. switch (ch) {
  38. case NUL: *ppSrc = NULL; return;
  39. case '"': goto done;
  40. case '\\':
  41. pSrc += ao_string_cook_escape_char((char *)pSrc, (char *)&ch, 0x7F);
  42. if (ch == 0x7F)
  43. break;
  44. /* FALLTHROUGH */
  45. default:
  46. *(pDest++) = ch;
  47. }
  48. }
  49. done:
  50. *ppDest = (ch_t *)pDest; /* next spot for storing character */
  51. *ppSrc = (char const *)pSrc; /* char following closing quote */
  52. }
  53. static void
  54. copy_raw(ch_t ** ppDest, char const ** ppSrc)
  55. {
  56. ch_t * pDest = *ppDest;
  57. cc_t * pSrc = (cc_t *) (*ppSrc + 1);
  58. for (;;) {
  59. ch_t ch = *(pSrc++);
  60. switch (ch) {
  61. case NUL: *ppSrc = NULL; return;
  62. case '\'': goto done;
  63. case '\\':
  64. /*
  65. * *Four* escapes are handled: newline removal, escape char
  66. * quoting and apostrophe quoting
  67. */
  68. switch (*pSrc) {
  69. case NUL: *ppSrc = NULL; return;
  70. case '\r':
  71. if (*(++pSrc) == NL)
  72. ++pSrc;
  73. continue;
  74. case NL:
  75. ++pSrc;
  76. continue;
  77. case '\'':
  78. ch = '\'';
  79. /* FALLTHROUGH */
  80. case '\\':
  81. ++pSrc;
  82. break;
  83. }
  84. /* FALLTHROUGH */
  85. default:
  86. *(pDest++) = ch;
  87. }
  88. }
  89. done:
  90. *ppDest = pDest; /* next spot for storing character */
  91. *ppSrc = (char const *) pSrc; /* char following closing quote */
  92. }
  93. static token_list_t *
  94. alloc_token_list(char const * str)
  95. {
  96. token_list_t * res;
  97. int max_token_ct = 2; /* allow for trailing NULL pointer & NUL on string */
  98. if (str == NULL) goto enoent_res;
  99. /*
  100. * Trim leading white space. Use "ENOENT" and a NULL return to indicate
  101. * an empty string was passed.
  102. */
  103. str = SPN_WHITESPACE_CHARS(str);
  104. if (*str == NUL) goto enoent_res;
  105. /*
  106. * Take an approximate count of tokens. If no quoted strings are used,
  107. * it will be accurate. If quoted strings are used, it will be a little
  108. * high and we'll squander the space for a few extra pointers.
  109. */
  110. {
  111. char const * pz = str;
  112. do {
  113. max_token_ct++;
  114. pz = BRK_WHITESPACE_CHARS(pz+1);
  115. pz = SPN_WHITESPACE_CHARS(pz);
  116. } while (*pz != NUL);
  117. res = malloc(sizeof(*res) + (size_t)(pz - str)
  118. + ((size_t)max_token_ct * sizeof(ch_t *)));
  119. }
  120. if (res == NULL)
  121. errno = ENOMEM;
  122. else res->tkn_list[0] = (ch_t *)(res->tkn_list + (max_token_ct - 1));
  123. return res;
  124. enoent_res:
  125. errno = ENOENT;
  126. return NULL;
  127. }
  128. /*=export_func ao_string_tokenize
  129. *
  130. * what: tokenize an input string
  131. *
  132. * arg: + char const * + string + string to be tokenized +
  133. *
  134. * ret_type: token_list_t *
  135. * ret_desc: pointer to a structure that lists each token
  136. *
  137. * doc:
  138. *
  139. * This function will convert one input string into a list of strings.
  140. * The list of strings is derived by separating the input based on
  141. * white space separation. However, if the input contains either single
  142. * or double quote characters, then the text after that character up to
  143. * a matching quote will become the string in the list.
  144. *
  145. * The returned pointer should be deallocated with @code{free(3C)} when
  146. * are done using the data. The data are placed in a single block of
  147. * allocated memory. Do not deallocate individual token/strings.
  148. *
  149. * The structure pointed to will contain at least these two fields:
  150. * @table @samp
  151. * @item tkn_ct
  152. * The number of tokens found in the input string.
  153. * @item tok_list
  154. * An array of @code{tkn_ct + 1} pointers to substring tokens, with
  155. * the last pointer set to NULL.
  156. * @end table
  157. *
  158. * There are two types of quoted strings: single quoted (@code{'}) and
  159. * double quoted (@code{"}). Singly quoted strings are fairly raw in that
  160. * escape characters (@code{\\}) are simply another character, except when
  161. * preceding the following characters:
  162. * @example
  163. * @code{\\} double backslashes reduce to one
  164. * @code{'} incorporates the single quote into the string
  165. * @code{\n} suppresses both the backslash and newline character
  166. * @end example
  167. *
  168. * Double quote strings are formed according to the rules of string
  169. * constants in ANSI-C programs.
  170. *
  171. * example:
  172. * @example
  173. * #include <stdlib.h>
  174. * int ix;
  175. * token_list_t * ptl = ao_string_tokenize(some_string)
  176. * for (ix = 0; ix < ptl->tkn_ct; ix++)
  177. * do_something_with_tkn(ptl->tkn_list[ix]);
  178. * free(ptl);
  179. * @end example
  180. * Note that everything is freed with the one call to @code{free(3C)}.
  181. *
  182. * err:
  183. * NULL is returned and @code{errno} will be set to indicate the problem:
  184. * @itemize @bullet
  185. * @item
  186. * @code{EINVAL} - There was an unterminated quoted string.
  187. * @item
  188. * @code{ENOENT} - The input string was empty.
  189. * @item
  190. * @code{ENOMEM} - There is not enough memory.
  191. * @end itemize
  192. =*/
  193. token_list_t *
  194. ao_string_tokenize(char const * str)
  195. {
  196. token_list_t * res = alloc_token_list(str);
  197. ch_t * pzDest;
  198. /*
  199. * Now copy each token into the output buffer.
  200. */
  201. if (res == NULL)
  202. return res;
  203. pzDest = (ch_t *)(res->tkn_list[0]);
  204. res->tkn_ct = 0;
  205. do {
  206. res->tkn_list[ res->tkn_ct++ ] = pzDest;
  207. for (;;) {
  208. int ch = (ch_t)*str;
  209. if (IS_WHITESPACE_CHAR(ch)) {
  210. found_white_space:
  211. str = SPN_WHITESPACE_CHARS(str+1);
  212. break;
  213. }
  214. switch (ch) {
  215. case '"':
  216. copy_cooked(&pzDest, &str);
  217. if (str == NULL) {
  218. free(res);
  219. errno = EINVAL;
  220. return NULL;
  221. }
  222. if (IS_WHITESPACE_CHAR(*str))
  223. goto found_white_space;
  224. break;
  225. case '\'':
  226. copy_raw(&pzDest, &str);
  227. if (str == NULL) {
  228. free(res);
  229. errno = EINVAL;
  230. return NULL;
  231. }
  232. if (IS_WHITESPACE_CHAR(*str))
  233. goto found_white_space;
  234. break;
  235. case NUL:
  236. goto copy_done;
  237. default:
  238. str++;
  239. *(pzDest++) = (unsigned char)ch;
  240. }
  241. } copy_done:;
  242. /*
  243. * NUL terminate the last token and see if we have any more tokens.
  244. */
  245. *(pzDest++) = NUL;
  246. } while (*str != NUL);
  247. res->tkn_list[ res->tkn_ct ] = NULL;
  248. return res;
  249. }
  250. #ifdef TEST
  251. #include <stdio.h>
  252. #include <string.h>
  253. int
  254. main(int argc, char ** argv)
  255. {
  256. if (argc == 1) {
  257. printf("USAGE: %s arg [ ... ]\n", *argv);
  258. return 1;
  259. }
  260. while (--argc > 0) {
  261. char * arg = *(++argv);
  262. token_list_t * p = ao_string_tokenize(arg);
  263. if (p == NULL) {
  264. printf("Parsing string ``%s'' failed:\n\terrno %d (%s)\n",
  265. arg, errno, strerror(errno));
  266. } else {
  267. int ix = 0;
  268. printf("Parsed string ``%s''\ninto %d tokens:\n", arg, p->tkn_ct);
  269. do {
  270. printf(" %3d: ``%s''\n", ix+1, p->tkn_list[ix]);
  271. } while (++ix < p->tkn_ct);
  272. free(p);
  273. }
  274. }
  275. return 0;
  276. }
  277. #endif
  278. /** @}
  279. *
  280. * Local Variables:
  281. * mode: C
  282. * c-file-style: "stroustrup"
  283. * indent-tabs-mode: nil
  284. * End:
  285. * end of autoopts/tokenize.c */