gen_wctype.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869
  1. #define _GNU_SOURCE
  2. #include <stdio.h>
  3. #include <stdlib.h>
  4. #include <string.h>
  5. #include <locale.h>
  6. #include <wctype.h>
  7. #include <limits.h>
  8. #include <stdint.h>
  9. #include <wchar.h>
  10. #include <ctype.h>
  11. #ifndef __UCLIBC__
  12. #ifndef _WCTYPE_H
  13. #define _WCTYPE_H
  14. #endif
  15. #include "../../libc/sysdeps/linux/common/bits/uClibc_ctype.h"
  16. #endif
  17. /* 0x9 : space blank */
  18. /* 0xa : space */
  19. /* 0xb : space */
  20. /* 0xc : space */
  21. /* 0xd : space */
  22. /* 0x20 : space blank */
  23. /* 0x1680 : space blank */
  24. /* 0x2000 : space blank */
  25. /* 0x2001 : space blank */
  26. /* 0x2002 : space blank */
  27. /* 0x2003 : space blank */
  28. /* 0x2004 : space blank */
  29. /* 0x2005 : space blank */
  30. /* 0x2006 : space blank */
  31. /* 0x2008 : space blank */
  32. /* 0x2009 : space blank */
  33. /* 0x200a : space blank */
  34. /* 0x200b : space blank */
  35. /* 0x2028 : space */
  36. /* 0x2029 : space */
  37. /* 0x3000 : space blank */
  38. /* typecount[ 0] = 88670 C_alpha_nonupper_nonlower */
  39. /* typecount[ 1] = 742 C_alpha_lower */
  40. /* typecount[ 2] = 4 C_alpha_upper_lower */
  41. /* typecount[ 3] = 731 C_alpha_upper */
  42. /* typecount[ 4] = 10 C_digit */
  43. /* typecount[ 5] = 10270 C_punct */
  44. /* typecount[ 6] = 0 C_graph */
  45. /* typecount[ 7] = 0 C_print_space_nonblank */
  46. /* typecount[ 8] = 14 C_print_space_blank */
  47. /* typecount[ 9] = 0 C_space_nonblank_noncntrl */
  48. /* typecount[10] = 0 C_space_blank_noncntrl */
  49. /* typecount[11] = 6 C_cntrl_space_nonblank */
  50. /* typecount[12] = 1 C_cntrl_space_blank */
  51. /* typecount[13] = 60 C_cntrl_nonspace */
  52. /* typecount[14] = 96100 C_unclassified */
  53. /* typecount[15] = 0 empty_slot */
  54. /* Set to #if 0 to restrict wchars to 16 bits. */
  55. #if 1
  56. #define RANGE 0x2ffffUL
  57. #elif 0
  58. #define RANGE 0x1ffffUL
  59. #else
  60. #define RANGE 0xffffUL /* Restrict for 16-bit wchar_t... */
  61. #endif
  62. #if 0
  63. /* Classification codes. */
  64. static const char *typename[] = {
  65. "C_unclassified",
  66. "C_alpha_nonupper_nonlower",
  67. "C_alpha_lower",
  68. "C_alpha_upper_lower",
  69. "C_alpha_upper",
  70. "C_digit",
  71. "C_punct",
  72. "C_graph",
  73. "C_print_space_nonblank",
  74. "C_print_space_blank",
  75. "C_space_nonblank_noncntrl",
  76. "C_space_blank_noncntrl",
  77. "C_cntrl_space_nonblank",
  78. "C_cntrl_space_blank",
  79. "C_cntrl_nonspace",
  80. "empty_slot"
  81. };
  82. #endif
  83. #if 0
  84. /* Taking advantage of the C99 mutual-exclusion guarantees for the various
  85. * (w)ctype classes, including the descriptions of printing and control
  86. * (w)chars, we can place each in one of the following mutually-exlusive
  87. * subsets. Since there are less than 16, we can store the data for
  88. * each (w)chars in a nibble. In contrast, glibc uses an unsigned int
  89. * per (w)char, with one bit flag for each is* type. While this allows
  90. * a simple '&' operation to determine the type vs. a range test and a
  91. * little special handling for the "blank" and "xdigit" types in my
  92. * approach, it also uses 8 times the space for the tables on the typical
  93. * 32-bit archs we supported.*/
  94. enum {
  95. __CTYPE_unclassified = 0,
  96. __CTYPE_alpha_nonupper_nonlower,
  97. __CTYPE_alpha_lower,
  98. __CTYPE_alpha_upper_lower,
  99. __CTYPE_alpha_upper,
  100. __CTYPE_digit,
  101. __CTYPE_punct,
  102. __CTYPE_graph,
  103. __CTYPE_print_space_nonblank,
  104. __CTYPE_print_space_blank,
  105. __CTYPE_space_nonblank_noncntrl,
  106. __CTYPE_space_blank_noncntrl,
  107. __CTYPE_cntrl_space_nonblank,
  108. __CTYPE_cntrl_space_blank,
  109. __CTYPE_cntrl_nonspace,
  110. };
  111. #endif
  112. #define __CTYPE_isxdigit(D,X) \
  113. (__CTYPE_isdigit(D) || (((unsigned int)(((X)|0x20) - 'a')) <= 5))
  114. #define mywalnum(x) __CTYPE_isalnum(d)
  115. #define mywalpha(x) __CTYPE_isalpha(d)
  116. #define mywblank(x) __CTYPE_isblank(d)
  117. #define mywcntrl(x) __CTYPE_iscntrl(d)
  118. #define mywdigit(x) __CTYPE_isdigit(d)
  119. #define mywgraph(x) __CTYPE_isgraph(d)
  120. #define mywlower(x) __CTYPE_islower(d)
  121. #define mywprint(x) __CTYPE_isprint(d)
  122. #define mywpunct(x) __CTYPE_ispunct(d)
  123. #define mywspace(x) __CTYPE_isspace(d)
  124. #define mywupper(x) __CTYPE_isupper(d)
  125. #define mywxdigit(x) __CTYPE_isxdigit(d,x)
  126. typedef struct {
  127. short l;
  128. short u;
  129. } uldiff_entry;
  130. typedef struct {
  131. uint16_t ii_len;
  132. uint16_t ti_len;
  133. uint16_t ut_len;
  134. unsigned char ii_shift;
  135. unsigned char ti_shift;
  136. unsigned char *ii;
  137. unsigned char *ti;
  138. unsigned char *ut;
  139. } table_data;
  140. void output_table(FILE *fp, const char *name, table_data *tbl)
  141. {
  142. size_t i;
  143. fprintf(fp, "#define WC%s_II_LEN %7u\n", name, tbl->ii_len);
  144. fprintf(fp, "#define WC%s_TI_LEN %7u\n", name, tbl->ti_len);
  145. fprintf(fp, "#define WC%s_UT_LEN %7u\n", name, tbl->ut_len);
  146. fprintf(fp, "#define WC%s_II_SHIFT %7u\n", name, tbl->ii_shift);
  147. fprintf(fp, "#define WC%s_TI_SHIFT %7u\n", name, tbl->ti_shift);
  148. fprintf(fp, "\n#ifdef WANT_WC%s_data\n", name);
  149. i = tbl->ii_len + tbl->ti_len + tbl->ut_len;
  150. fprintf(fp, "\nstatic const unsigned char WC%s_data[%zu] = {", name, i);
  151. for (i=0 ; i < tbl->ii_len ; i++) {
  152. if (i % 12 == 0) {
  153. fprintf(fp, "\n");
  154. }
  155. fprintf(fp, " %#04x,", tbl->ii[i]);
  156. }
  157. for (i=0 ; i < tbl->ti_len ; i++) {
  158. if (i % 12 == 0) {
  159. fprintf(fp, "\n");
  160. }
  161. fprintf(fp, " %#04x,", tbl->ti[i]);
  162. }
  163. for (i=0 ; i < tbl->ut_len ; i++) {
  164. if (i % 12 == 0) {
  165. fprintf(fp, "\n");
  166. }
  167. fprintf(fp, " %#04x,", tbl->ut[i]);
  168. }
  169. fprintf(fp, "\n};\n\n");
  170. fprintf(fp, "#endif /* WANT_WC%s_data */\n\n", name);
  171. }
  172. static void dump_table_data(table_data *tbl)
  173. {
  174. printf("ii_shift = %d ti_shift = %d\n"
  175. "ii_len = %d ti_len = %d ut_len = %d\n"
  176. "total = %d\n",
  177. tbl->ii_shift, tbl->ti_shift,
  178. tbl->ii_len, tbl->ti_len, tbl->ut_len,
  179. (int) tbl->ii_len + (int) tbl->ti_len + (int) tbl->ut_len);
  180. }
  181. /* For sorting the blocks of unsigned chars. */
  182. static size_t nu_val;
  183. int nu_memcmp(const void *a, const void *b)
  184. {
  185. return memcmp(*(unsigned char**)a, *(unsigned char**)b, nu_val);
  186. }
  187. static size_t newopt(unsigned char *ut, size_t usize, int shift, table_data *tbl);
  188. #define MAXTO 255 /* Restrict to minimal unsigned char max. */
  189. int main(int argc, char **argv)
  190. {
  191. long int u, l, tt;
  192. size_t smallest, t;
  193. unsigned int c;
  194. unsigned int d;
  195. int i, n;
  196. int ul_count = 0;
  197. uldiff_entry uldiff[MAXTO];
  198. table_data cttable;
  199. table_data ultable;
  200. table_data combtable;
  201. table_data widthtable;
  202. long int last_comb = 0;
  203. unsigned char wct[(RANGE/2)+1]; /* wctype table (nibble per wchar) */
  204. unsigned char ult[RANGE+1]; /* upper/lower table */
  205. unsigned char combt[(RANGE/4)+1]; /* combining */
  206. unsigned char widtht[(RANGE/4)+1]; /* width */
  207. wctrans_t totitle;
  208. wctype_t is_comb, is_comb3;
  209. long int typecount[16];
  210. const char *typename[16];
  211. static const char empty_slot[] = "empty_slot";
  212. int built = 0;
  213. #define INIT_TYPENAME(X) typename[__CTYPE_##X] = "C_" #X
  214. for (i=0 ; i < 16 ; i++) {
  215. typename[i] = empty_slot;
  216. }
  217. INIT_TYPENAME(unclassified);
  218. INIT_TYPENAME(alpha_nonupper_nonlower);
  219. INIT_TYPENAME(alpha_lower);
  220. INIT_TYPENAME(alpha_upper_lower);
  221. INIT_TYPENAME(alpha_upper);
  222. INIT_TYPENAME(digit);
  223. INIT_TYPENAME(punct);
  224. INIT_TYPENAME(graph);
  225. INIT_TYPENAME(print_space_nonblank);
  226. INIT_TYPENAME(print_space_blank);
  227. INIT_TYPENAME(space_nonblank_noncntrl);
  228. INIT_TYPENAME(space_blank_noncntrl);
  229. INIT_TYPENAME(cntrl_space_nonblank);
  230. INIT_TYPENAME(cntrl_space_blank);
  231. INIT_TYPENAME(cntrl_nonspace);
  232. setvbuf(stdout, NULL, _IONBF, 0);
  233. while (--argc) {
  234. if (!setlocale(LC_CTYPE, *++argv)) {
  235. printf("setlocale(LC_CTYPE,%s) failed!\n", *argv);
  236. continue;
  237. }
  238. if (!(totitle = wctrans("totitle"))) {
  239. printf("no totitle transformation.\n");
  240. }
  241. if (!(is_comb = wctype("combining"))) {
  242. printf("no combining wctype.\n");
  243. }
  244. if (!(is_comb3 = wctype("combining_level3"))) {
  245. printf("no combining_level3 wctype.\n");
  246. }
  247. if (!built) {
  248. built = 1;
  249. ul_count = 1;
  250. uldiff[0].u = uldiff[0].l = 0;
  251. memset(wct, 0, sizeof(wct));
  252. memset(combt, 0, sizeof(combt));
  253. memset(widtht, 0, sizeof(widtht));
  254. for (i = 0 ; i < 16 ; i++) {
  255. typecount[i] = 0;
  256. }
  257. for (c=0 ; c <= RANGE ; c++) {
  258. if (iswdigit(c)) {
  259. d = __CTYPE_digit;
  260. } else if (iswalpha(c)) {
  261. d = __CTYPE_alpha_nonupper_nonlower;
  262. if (iswlower(c)) {
  263. d = __CTYPE_alpha_lower;
  264. if (iswupper(c)) {
  265. d = __CTYPE_alpha_upper_lower;
  266. }
  267. } else if (iswupper(c)) {
  268. d = __CTYPE_alpha_upper;
  269. }
  270. } else if (iswpunct(c)) {
  271. d = __CTYPE_punct;
  272. } else if (iswgraph(c)) {
  273. d = __CTYPE_graph;
  274. } else if (iswprint(c)) {
  275. d = __CTYPE_print_space_nonblank;
  276. if (iswblank(c)) {
  277. d = __CTYPE_print_space_blank;
  278. }
  279. } else if (iswspace(c) && !iswcntrl(c)) {
  280. d = __CTYPE_space_nonblank_noncntrl;
  281. if (iswblank(c)) {
  282. d = __CTYPE_space_blank_noncntrl;
  283. }
  284. } else if (iswcntrl(c)) {
  285. d = __CTYPE_cntrl_nonspace;
  286. if (iswspace(c)) {
  287. d = __CTYPE_cntrl_space_nonblank;
  288. if (iswblank(c)) {
  289. d = __CTYPE_cntrl_space_blank;
  290. }
  291. }
  292. } else {
  293. d = __CTYPE_unclassified;
  294. }
  295. ++typecount[d];
  296. #if 0
  297. if (iswspace(c)) {
  298. if (iswblank(c)) {
  299. printf("%#8x : space blank\n", c);
  300. } else {
  301. printf("%#8x : space\n", c);
  302. }
  303. }
  304. #endif
  305. #if 0
  306. if (c < 256) {
  307. unsigned int glibc;
  308. glibc = 0;
  309. if (isalnum(c)) ++glibc; glibc <<= 1;
  310. if (isalpha(c)) ++glibc; glibc <<= 1;
  311. if (isblank(c)) ++glibc; glibc <<= 1;
  312. if (iscntrl(c)) ++glibc; glibc <<= 1;
  313. if (isdigit(c)) ++glibc; glibc <<= 1;
  314. if (isgraph(c)) ++glibc; glibc <<= 1;
  315. if (islower(c)) ++glibc; glibc <<= 1;
  316. if (isprint(c)) ++glibc; glibc <<= 1;
  317. if (ispunct(c)) ++glibc; glibc <<= 1;
  318. if (isspace(c)) ++glibc; glibc <<= 1;
  319. if (isupper(c)) ++glibc; glibc <<= 1;
  320. if (isxdigit(c)) ++glibc;
  321. printf("%#8x : ctype %#4x\n", c, glibc);
  322. }
  323. #endif
  324. #if 1
  325. /* Paranoid checking... */
  326. {
  327. unsigned int glibc;
  328. unsigned int mine;
  329. glibc = 0;
  330. if (iswalnum(c)) ++glibc; glibc <<= 1;
  331. if (iswalpha(c)) ++glibc; glibc <<= 1;
  332. if (iswblank(c)) ++glibc; glibc <<= 1;
  333. if (iswcntrl(c)) ++glibc; glibc <<= 1;
  334. if (iswdigit(c)) ++glibc; glibc <<= 1;
  335. if (iswgraph(c)) ++glibc; glibc <<= 1;
  336. if (iswlower(c)) ++glibc; glibc <<= 1;
  337. if (iswprint(c)) ++glibc; glibc <<= 1;
  338. if (iswpunct(c)) ++glibc; glibc <<= 1;
  339. if (iswspace(c)) ++glibc; glibc <<= 1;
  340. if (iswupper(c)) ++glibc; glibc <<= 1;
  341. if (iswxdigit(c)) ++glibc;
  342. mine = 0;
  343. if (mywalnum(c)) ++mine; mine <<= 1;
  344. if (mywalpha(c)) ++mine; mine <<= 1;
  345. if (mywblank(c)) ++mine; mine <<= 1;
  346. if (mywcntrl(c)) ++mine; mine <<= 1;
  347. if (mywdigit(c)) ++mine; mine <<= 1;
  348. if (mywgraph(c)) ++mine; mine <<= 1;
  349. if (mywlower(c)) ++mine; mine <<= 1;
  350. if (mywprint(c)) ++mine; mine <<= 1;
  351. if (mywpunct(c)) ++mine; mine <<= 1;
  352. if (mywspace(c)) ++mine; mine <<= 1;
  353. if (mywupper(c)) ++mine; mine <<= 1;
  354. if (mywxdigit(c)) ++mine;
  355. if (glibc != mine) {
  356. printf("%#8x : glibc %#4x != %#4x mine %u\n", c, glibc, mine, d);
  357. return EXIT_FAILURE;
  358. }
  359. #if 0
  360. if (iswctype(c,is_comb) || iswctype(c,is_comb3)) {
  361. /* if (!iswpunct(c)) { */
  362. printf("%#8x : %d %d %#4x\n",
  363. c, iswctype(c,is_comb),iswctype(c,is_comb3), glibc);
  364. /* } */
  365. }
  366. #endif
  367. #if 0
  368. if (iswctype(c,is_comb) || iswctype(c,is_comb3)) {
  369. if (!last_comb) {
  370. printf("%#8x - ", c);
  371. last_comb = c;
  372. } else if (last_comb + 1 < c) {
  373. printf("%#8x\n%#8x - ", last_comb, c);
  374. last_comb = c;
  375. } else {
  376. last_comb = c;
  377. }
  378. }
  379. #endif
  380. }
  381. #endif
  382. combt[c/4] |= ((((!!iswctype(c,is_comb)) << 1) | !!iswctype(c,is_comb3))
  383. << ((c & 3) << 1));
  384. /* comb3t[c/8] |= ((!!iswctype(c,is_comb3)) << (c & 7)); */
  385. /* widtht[c/4] |= (wcwidth(c) << ((c & 3) << 1)); */
  386. if (c & 1) { /* Use the high nibble for odd numbered wchars. */
  387. d <<= 4;
  388. }
  389. wct[c/2] |= d;
  390. l = towlower(c) - c;
  391. u = towupper(c) - c;
  392. ult[c] = 0;
  393. if (l || u) {
  394. if ((l != (short)l) || (u != (short)u)) {
  395. printf("range assumption error! %x %ld %ld\n", c, l, u);
  396. return EXIT_FAILURE;
  397. }
  398. for (i=0 ; i < ul_count ; i++) {
  399. if ((l == uldiff[i].l) && (u == uldiff[i].u)) {
  400. goto found;
  401. }
  402. }
  403. uldiff[ul_count].l = l;
  404. uldiff[ul_count].u = u;
  405. ++ul_count;
  406. if (ul_count > MAXTO) {
  407. printf("too many touppers/tolowers!\n");
  408. return EXIT_FAILURE;
  409. }
  410. found:
  411. ult[c] = i;
  412. }
  413. }
  414. for (i = 0 ; i < 16 ; i++) {
  415. printf("typecount[%2d] = %8ld %s\n", i, typecount[i], typename[i]);
  416. }
  417. printf("optimizing is* table..\n");
  418. n = -1;
  419. smallest = SIZE_MAX;
  420. cttable.ii = NULL;
  421. for (i=0 ; i < 14 ; i++) {
  422. t = newopt(wct, (RANGE/2)+1, i, &cttable);
  423. if (smallest >= t) {
  424. n = i;
  425. smallest = t;
  426. /* } else { */
  427. /* break; */
  428. }
  429. }
  430. printf("smallest = %zu\n", smallest);
  431. if (!(cttable.ii = malloc(smallest))) {
  432. printf("couldn't allocate space!\n");
  433. return EXIT_FAILURE;
  434. }
  435. smallest = SIZE_MAX;
  436. newopt(wct, (RANGE/2)+1, n, &cttable);
  437. ++cttable.ti_shift; /* correct for nibble mode */
  438. printf("optimizing u/l-to table..\n");
  439. smallest = SIZE_MAX;
  440. ultable.ii = NULL;
  441. for (i=0 ; i < 14 ; i++) {
  442. t = newopt(ult, RANGE+1, i, &ultable);
  443. if (smallest >= t) {
  444. n = i;
  445. smallest = t;
  446. /* } else { */
  447. /* break; */
  448. }
  449. }
  450. printf("%zu (smallest) + %zu (u/l diffs) = %zu\n",
  451. smallest, 4 * ul_count, smallest + 4 * ul_count);
  452. printf("smallest = %zu\n", smallest);
  453. if (!(ultable.ii = malloc(smallest))) {
  454. printf("couldn't allocate space!\n");
  455. return EXIT_FAILURE;
  456. }
  457. smallest = SIZE_MAX;
  458. newopt(ult, RANGE+1, n, &ultable);
  459. #if 0
  460. printf("optimizing comb table..\n");
  461. smallest = SIZE_MAX;
  462. combtable.ii = NULL;
  463. for (i=0 ; i < 14 ; i++) {
  464. t = newopt(combt, sizeof(combt), i, &combtable);
  465. if (smallest >= t) {
  466. n = i;
  467. smallest = t;
  468. /* } else { */
  469. /* break; */
  470. }
  471. }
  472. printf("smallest = %zu\n", smallest);
  473. if (!(combtable.ii = malloc(smallest))) {
  474. printf("couldn't allocate space!\n");
  475. return EXIT_FAILURE;
  476. }
  477. smallest = SIZE_MAX;
  478. newopt(combt, sizeof(combt), n, &combtable);
  479. combtable.ti_shift += 4; /* correct for 4 entries per */
  480. #endif
  481. #if 0
  482. printf("optimizing width table..\n");
  483. smallest = SIZE_MAX;
  484. widthtable.ii = NULL;
  485. for (i=0 ; i < 14 ; i++) {
  486. t = newopt(widtht, sizeof(widtht), i, &widthtable);
  487. if (smallest >= t) {
  488. n = i;
  489. smallest = t;
  490. /* } else { */
  491. /* break; */
  492. }
  493. }
  494. printf("smallest = %zu\n", smallest);
  495. if (!(widthtable.ii = malloc(smallest))) {
  496. printf("couldn't allocate space!\n");
  497. return EXIT_FAILURE;
  498. }
  499. smallest = SIZE_MAX;
  500. newopt(widtht, sizeof(widtht), n, &widthtable);
  501. widthtable.ti_shift += 4; /* correct for 4 entries per */
  502. #endif
  503. #if 0
  504. printf("optimizing comb3 table..\n");
  505. smallest = SIZE_MAX;
  506. comb3table.ii = NULL;
  507. for (i=0 ; i < 14 ; i++) {
  508. t = newopt(comb3t, sizeof(comb3t), i, &comb3table);
  509. if (smallest >= t) {
  510. n = i;
  511. smallest = t;
  512. /* } else { */
  513. /* break; */
  514. }
  515. }
  516. printf("smallest = %zu\n", smallest);
  517. if (!(comb3table.ii = malloc(smallest))) {
  518. printf("couldn't allocate space!\n");
  519. return EXIT_FAILURE;
  520. }
  521. smallest = SIZE_MAX;
  522. newopt(comb3t, sizeof(comb3t), n, &comb3table);
  523. comb3table.ti_shift += 8; /* correct for 4 entries per */
  524. #endif
  525. dump_table_data(&cttable);
  526. dump_table_data(&ultable);
  527. dump_table_data(&combtable);
  528. }
  529. printf("verifying for %s...\n", *argv);
  530. #if RANGE == 0xffffU
  531. for (c=0 ; c <= 0xffffUL ; c++)
  532. #else
  533. for (c=0 ; c <= 0x10ffffUL ; c++)
  534. #endif
  535. {
  536. unsigned int glibc;
  537. unsigned int mine;
  538. unsigned int upper, lower;
  539. #if 0
  540. #if RANGE < 0x10000UL
  541. if (c == 0x10000UL) {
  542. c = 0x30000UL; /* skip 1st and 2nd sup planes */
  543. }
  544. #elif RANGE < 0x20000UL
  545. if (c == 0x20000UL) {
  546. c = 0x30000UL; /* skip 2nd sup planes */
  547. }
  548. #endif
  549. #endif
  550. glibc = 0;
  551. if (iswalnum(c)) ++glibc; glibc <<= 1;
  552. if (iswalpha(c)) ++glibc; glibc <<= 1;
  553. if (iswblank(c)) ++glibc; glibc <<= 1;
  554. if (iswcntrl(c)) ++glibc; glibc <<= 1;
  555. if (iswdigit(c)) ++glibc; glibc <<= 1;
  556. if (iswgraph(c)) ++glibc; glibc <<= 1;
  557. if (iswlower(c)) ++glibc; glibc <<= 1;
  558. if (iswprint(c)) ++glibc; glibc <<= 1;
  559. if (iswpunct(c)) ++glibc; glibc <<= 1;
  560. if (iswspace(c)) ++glibc; glibc <<= 1;
  561. if (iswupper(c)) ++glibc; glibc <<= 1;
  562. if (iswxdigit(c)) ++glibc;
  563. {
  564. unsigned int u;
  565. int n, sc;
  566. int i0, i1;
  567. u = c;
  568. if (u <= RANGE) {
  569. sc = u & ((1 << cttable.ti_shift) - 1);
  570. u >>= cttable.ti_shift;
  571. n = u & ((1 << cttable.ii_shift) - 1);
  572. u >>= cttable.ii_shift;
  573. i0 = cttable.ii[u];
  574. i0 <<= cttable.ii_shift;
  575. i1 = cttable.ti[i0 + n];
  576. i1 <<= (cttable.ti_shift-1);
  577. d = cttable.ut[i1 + (sc >> 1)];
  578. if (sc & 1) {
  579. d >>= 4;
  580. }
  581. d &= 0x0f;
  582. } else if ((((unsigned int)(c - 0xe0020UL)) <= 0x5f) || (c == 0xe0001UL)){
  583. d = __CTYPE_punct;
  584. } else if (((unsigned int)(c - 0xf0000UL)) < 0x20000UL) {
  585. if ((c & 0xffffU) <= 0xfffdU) {
  586. d = __CTYPE_punct;
  587. } else {
  588. d = __CTYPE_unclassified;
  589. }
  590. } else {
  591. d = __CTYPE_unclassified;
  592. }
  593. mine = 0;
  594. if (mywalnum(c)) ++mine; mine <<= 1;
  595. if (mywalpha(c)) ++mine; mine <<= 1;
  596. if (mywblank(c)) ++mine; mine <<= 1;
  597. if (mywcntrl(c)) ++mine; mine <<= 1;
  598. if (mywdigit(c)) ++mine; mine <<= 1;
  599. if (mywgraph(c)) ++mine; mine <<= 1;
  600. if (mywlower(c)) ++mine; mine <<= 1;
  601. if (mywprint(c)) ++mine; mine <<= 1;
  602. if (mywpunct(c)) ++mine; mine <<= 1;
  603. if (mywspace(c)) ++mine; mine <<= 1;
  604. if (mywupper(c)) ++mine; mine <<= 1;
  605. if (mywxdigit(c)) ++mine;
  606. if (glibc != mine) {
  607. printf("%#8x : glibc %#4x != %#4x mine %d\n", c, glibc, mine, d);
  608. if (c < 0x30000UL) {
  609. printf("sc=%#x u=%#x n=%#x i0=%#x i1=%#x\n", sc, u, n, i0, i1);
  610. }
  611. }
  612. upper = lower = u = c;
  613. if (u <= RANGE) {
  614. sc = u & ((1 << ultable.ti_shift) - 1);
  615. u >>= ultable.ti_shift;
  616. n = u & ((1 << ultable.ii_shift) - 1);
  617. u >>= ultable.ii_shift;
  618. i0 = ultable.ii[u];
  619. i0 <<= ultable.ii_shift;
  620. i1 = ultable.ti[i0 + n];
  621. i1 <<= (ultable.ti_shift);
  622. i1 += sc;
  623. i0 = ultable.ut[i1];
  624. upper = c + uldiff[i0].u;
  625. lower = c + uldiff[i0].l;
  626. }
  627. if (towupper(c) != upper) {
  628. printf("%#8x : towupper glibc %#4x != %#4x mine\n",
  629. c, towupper(c), upper);
  630. }
  631. if (towlower(c) != lower) {
  632. printf("%#8x : towlower glibc %#4x != %#4x mine i0 = %d\n",
  633. c, towlower(c), lower, i0);
  634. }
  635. if (totitle && ((tt = towctrans(c, totitle)) != upper)) {
  636. printf("%#8x : totitle glibc %#4lx != %#4x mine i0 = %d\n",
  637. c, tt, upper, i0);
  638. }
  639. }
  640. if ((c & 0xfff) == 0xfff) printf(".");
  641. }
  642. printf("done\n");
  643. }
  644. if (1) {
  645. FILE *fp;
  646. if (!(fp = fopen("wctables.h", "w"))) {
  647. printf("couldn't open wctables.h!\n");
  648. return EXIT_FAILURE;
  649. }
  650. fprintf(fp, "#define WC_TABLE_DOMAIN_MAX %#8lx\n\n",
  651. (unsigned long) RANGE);
  652. output_table(fp, "ctype", &cttable);
  653. output_table(fp, "uplow", &ultable);
  654. #warning fix the upper bound on the upper/lower tables... save 200 bytes or so
  655. fprintf(fp, "#define WCuplow_diffs %7u\n", ul_count);
  656. fprintf(fp, "\n#ifdef WANT_WCuplow_diff_data\n\n");
  657. fprintf(fp, "\nstatic const short WCuplow_diff_data[%zu] = {",
  658. 2 * (size_t) ul_count);
  659. for (i=0 ; i < ul_count ; i++) {
  660. if (i % 4 == 0) {
  661. fprintf(fp, "\n");
  662. }
  663. fprintf(fp, " %6d, %6d,", uldiff[i].u, uldiff[i].l);
  664. }
  665. fprintf(fp, "\n};\n\n");
  666. fprintf(fp, "#endif /* WANT_WCuplow_diff_data */\n\n");
  667. /* output_table(fp, "comb", &combtable); */
  668. /* output_table(fp, "width", &widthtable); */
  669. fclose(fp);
  670. }
  671. return EXIT_SUCCESS;
  672. }
  673. size_t newopt(unsigned char *ut, size_t usize, int shift, table_data *tbl)
  674. {
  675. static int recurse = 0;
  676. unsigned char *ti[RANGE+1]; /* table index */
  677. size_t numblocks;
  678. size_t blocksize;
  679. size_t uniq;
  680. size_t i, j;
  681. size_t smallest, t;
  682. unsigned char *ii_save;
  683. int uniqblock[256];
  684. unsigned char uit[RANGE+1];
  685. int shift2;
  686. ii_save = NULL;
  687. blocksize = 1 << shift;
  688. numblocks = usize >> shift;
  689. /* init table index */
  690. for (i=j=0 ; i < numblocks ; i++) {
  691. ti[i] = ut + j;
  692. j += blocksize;
  693. }
  694. /* sort */
  695. nu_val = blocksize;
  696. qsort(ti, numblocks, sizeof(unsigned char *), nu_memcmp);
  697. uniq = 1;
  698. uit[(ti[0]-ut)/blocksize] = 0;
  699. for (i=1 ; i < numblocks ; i++) {
  700. if (memcmp(ti[i-1], ti[i], blocksize) < 0) {
  701. if (++uniq > 255) {
  702. break;
  703. }
  704. uniqblock[uniq - 1] = i;
  705. }
  706. #if 1
  707. else if (memcmp(ti[i-1], ti[i], blocksize) > 0) {
  708. printf("bad sort %i!\n", i);
  709. abort();
  710. }
  711. #endif
  712. uit[(ti[i]-ut)/blocksize] = uniq - 1;
  713. }
  714. smallest = SIZE_MAX;
  715. shift2 = -1;
  716. if (uniq <= 255) {
  717. smallest = numblocks + uniq * blocksize;
  718. if (!recurse) {
  719. ++recurse;
  720. for (j=1 ; j < 14 ; j++) {
  721. if ((numblocks >> j) < 2) break;
  722. if (tbl) {
  723. ii_save = tbl->ii;
  724. tbl->ii = NULL;
  725. }
  726. if ((t = newopt(uit, numblocks, j, tbl)) < SIZE_MAX) {
  727. t += uniq * blocksize;
  728. }
  729. if (tbl) {
  730. tbl->ii = ii_save;
  731. }
  732. if (smallest >= t) {
  733. shift2 = j;
  734. smallest = t;
  735. if (!tbl->ii) {
  736. printf("ishift %zu tshift %zu size %zu\n",
  737. shift2, shift, t);
  738. }
  739. /* } else { */
  740. /* break; */
  741. }
  742. }
  743. --recurse;
  744. }
  745. } else {
  746. return SIZE_MAX;
  747. }
  748. if (tbl->ii) {
  749. if (recurse) {
  750. tbl->ii_shift = shift;
  751. tbl->ii_len = numblocks;
  752. memcpy(tbl->ii, uit, numblocks);
  753. tbl->ti = tbl->ii + tbl->ii_len;
  754. tbl->ti_len = uniq * blocksize;
  755. for (i=0 ; i < uniq ; i++) {
  756. memcpy(tbl->ti + i * blocksize, ti[uniqblock[i]], blocksize);
  757. }
  758. } else {
  759. ++recurse;
  760. printf("setting ishift %zu tshift %zu\n",
  761. shift2, shift);
  762. newopt(uit, numblocks, shift2, tbl);
  763. --recurse;
  764. tbl->ti_shift = shift;
  765. tbl->ut_len = uniq * blocksize;
  766. tbl->ut = tbl->ti + tbl->ti_len;
  767. for (i=0 ; i < uniq ; i++) {
  768. memcpy(tbl->ut + i * blocksize, ti[uniqblock[i]], blocksize);
  769. }
  770. }
  771. }
  772. return smallest;
  773. }