gen_wctype.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876
  1. /*
  2. * Copyright (C) 2000-2006 Erik Andersen <andersen@uclibc.org>
  3. *
  4. * Licensed under the LGPL v2.1, see the file COPYING.LIB in this tarball.
  5. */
  6. #define _GNU_SOURCE
  7. #include <stdio.h>
  8. #include <stdlib.h>
  9. #include <string.h>
  10. #include <locale.h>
  11. #include <wctype.h>
  12. #include <limits.h>
  13. #include <stdint.h>
  14. #include <wchar.h>
  15. #include <ctype.h>
  16. #ifndef _CTYPE_H
  17. #define _CTYPE_H
  18. #endif
  19. #ifndef _WCTYPE_H
  20. #define _WCTYPE_H
  21. #endif
  22. #include UCLIBC_CTYPE_HEADER
  23. /* 0x9 : space blank */
  24. /* 0xa : space */
  25. /* 0xb : space */
  26. /* 0xc : space */
  27. /* 0xd : space */
  28. /* 0x20 : space blank */
  29. /* 0x1680 : space blank */
  30. /* 0x2000 : space blank */
  31. /* 0x2001 : space blank */
  32. /* 0x2002 : space blank */
  33. /* 0x2003 : space blank */
  34. /* 0x2004 : space blank */
  35. /* 0x2005 : space blank */
  36. /* 0x2006 : space blank */
  37. /* 0x2008 : space blank */
  38. /* 0x2009 : space blank */
  39. /* 0x200a : space blank */
  40. /* 0x200b : space blank */
  41. /* 0x2028 : space */
  42. /* 0x2029 : space */
  43. /* 0x3000 : space blank */
  44. /* typecount[ 0] = 88670 C_alpha_nonupper_nonlower */
  45. /* typecount[ 1] = 742 C_alpha_lower */
  46. /* typecount[ 2] = 4 C_alpha_upper_lower */
  47. /* typecount[ 3] = 731 C_alpha_upper */
  48. /* typecount[ 4] = 10 C_digit */
  49. /* typecount[ 5] = 10270 C_punct */
  50. /* typecount[ 6] = 0 C_graph */
  51. /* typecount[ 7] = 0 C_print_space_nonblank */
  52. /* typecount[ 8] = 14 C_print_space_blank */
  53. /* typecount[ 9] = 0 C_space_nonblank_noncntrl */
  54. /* typecount[10] = 0 C_space_blank_noncntrl */
  55. /* typecount[11] = 6 C_cntrl_space_nonblank */
  56. /* typecount[12] = 1 C_cntrl_space_blank */
  57. /* typecount[13] = 60 C_cntrl_nonspace */
  58. /* typecount[14] = 96100 C_unclassified */
  59. /* typecount[15] = 0 empty_slot */
  60. /* Set to #if 0 to restrict wchars to 16 bits. */
  61. #if 1
  62. #define RANGE 0x2ffffUL
  63. #elif 0
  64. #define RANGE 0x1ffffUL
  65. #else
  66. #define RANGE 0xffffUL /* Restrict for 16-bit wchar_t... */
  67. #endif
  68. #if 0
  69. /* Classification codes. */
  70. static const char *typename[] = {
  71. "C_unclassified",
  72. "C_alpha_nonupper_nonlower",
  73. "C_alpha_lower",
  74. "C_alpha_upper_lower",
  75. "C_alpha_upper",
  76. "C_digit",
  77. "C_punct",
  78. "C_graph",
  79. "C_print_space_nonblank",
  80. "C_print_space_blank",
  81. "C_space_nonblank_noncntrl",
  82. "C_space_blank_noncntrl",
  83. "C_cntrl_space_nonblank",
  84. "C_cntrl_space_blank",
  85. "C_cntrl_nonspace",
  86. "empty_slot"
  87. };
  88. #endif
  89. #if 0
  90. /* Taking advantage of the C99 mutual-exclusion guarantees for the various
  91. * (w)ctype classes, including the descriptions of printing and control
  92. * (w)chars, we can place each in one of the following mutually-exlusive
  93. * subsets. Since there are less than 16, we can store the data for
  94. * each (w)chars in a nibble. In contrast, glibc uses an unsigned int
  95. * per (w)char, with one bit flag for each is* type. While this allows
  96. * a simple '&' operation to determine the type vs. a range test and a
  97. * little special handling for the "blank" and "xdigit" types in my
  98. * approach, it also uses 8 times the space for the tables on the typical
  99. * 32-bit archs we supported.*/
  100. enum {
  101. __CTYPE_unclassified = 0,
  102. __CTYPE_alpha_nonupper_nonlower,
  103. __CTYPE_alpha_lower,
  104. __CTYPE_alpha_upper_lower,
  105. __CTYPE_alpha_upper,
  106. __CTYPE_digit,
  107. __CTYPE_punct,
  108. __CTYPE_graph,
  109. __CTYPE_print_space_nonblank,
  110. __CTYPE_print_space_blank,
  111. __CTYPE_space_nonblank_noncntrl,
  112. __CTYPE_space_blank_noncntrl,
  113. __CTYPE_cntrl_space_nonblank,
  114. __CTYPE_cntrl_space_blank,
  115. __CTYPE_cntrl_nonspace,
  116. };
  117. #endif
  118. #define __CTYPE_isxdigit(D,X) \
  119. (__CTYPE_isdigit(D) || (((unsigned int)(((X)|0x20) - 'a')) <= 5))
  120. #define mywalnum(x) __CTYPE_isalnum(d)
  121. #define mywalpha(x) __CTYPE_isalpha(d)
  122. #define mywblank(x) __CTYPE_isblank(d)
  123. #define mywcntrl(x) __CTYPE_iscntrl(d)
  124. #define mywdigit(x) __CTYPE_isdigit(d)
  125. #define mywgraph(x) __CTYPE_isgraph(d)
  126. #define mywlower(x) __CTYPE_islower(d)
  127. #define mywprint(x) __CTYPE_isprint(d)
  128. #define mywpunct(x) __CTYPE_ispunct(d)
  129. #define mywspace(x) __CTYPE_isspace(d)
  130. #define mywupper(x) __CTYPE_isupper(d)
  131. #define mywxdigit(x) __CTYPE_isxdigit(d,x)
  132. typedef struct {
  133. short l;
  134. short u;
  135. } uldiff_entry;
  136. typedef struct {
  137. uint16_t ii_len;
  138. uint16_t ti_len;
  139. uint16_t ut_len;
  140. unsigned char ii_shift;
  141. unsigned char ti_shift;
  142. unsigned char *ii;
  143. unsigned char *ti;
  144. unsigned char *ut;
  145. } table_data;
  146. void output_table(FILE *fp, const char *name, table_data *tbl)
  147. {
  148. size_t i;
  149. fprintf(fp, "#define __LOCALE_DATA_WC%s_II_LEN %7u\n", name, tbl->ii_len);
  150. fprintf(fp, "#define __LOCALE_DATA_WC%s_TI_LEN %7u\n", name, tbl->ti_len);
  151. fprintf(fp, "#define __LOCALE_DATA_WC%s_UT_LEN %7u\n", name, tbl->ut_len);
  152. fprintf(fp, "#define __LOCALE_DATA_WC%s_II_SHIFT %7u\n", name, tbl->ii_shift);
  153. fprintf(fp, "#define __LOCALE_DATA_WC%s_TI_SHIFT %7u\n", name, tbl->ti_shift);
  154. fprintf(fp, "\n#ifdef WANT_WC%s_data\n", name);
  155. i = tbl->ii_len + tbl->ti_len + tbl->ut_len;
  156. fprintf(fp, "\nstatic const unsigned char __LOCALE_DATA_WC%s_data[%zu] = {", name, i);
  157. for (i=0 ; i < tbl->ii_len ; i++) {
  158. if (i % 12 == 0) {
  159. fprintf(fp, "\n");
  160. }
  161. fprintf(fp, " %#04x,", tbl->ii[i]);
  162. }
  163. for (i=0 ; i < tbl->ti_len ; i++) {
  164. if (i % 12 == 0) {
  165. fprintf(fp, "\n");
  166. }
  167. fprintf(fp, " %#04x,", tbl->ti[i]);
  168. }
  169. for (i=0 ; i < tbl->ut_len ; i++) {
  170. if (i % 12 == 0) {
  171. fprintf(fp, "\n");
  172. }
  173. fprintf(fp, " %#04x,", tbl->ut[i]);
  174. }
  175. fprintf(fp, "\n};\n\n");
  176. fprintf(fp, "#endif /* WANT_WC%s_data */\n\n", name);
  177. }
  178. static void dump_table_data(table_data *tbl)
  179. {
  180. printf("ii_shift = %d ti_shift = %d\n"
  181. "ii_len = %d ti_len = %d ut_len = %d\n"
  182. "total = %d\n",
  183. tbl->ii_shift, tbl->ti_shift,
  184. tbl->ii_len, tbl->ti_len, tbl->ut_len,
  185. (int) tbl->ii_len + (int) tbl->ti_len + (int) tbl->ut_len);
  186. }
  187. /* For sorting the blocks of unsigned chars. */
  188. static size_t nu_val;
  189. int nu_memcmp(const void *a, const void *b)
  190. {
  191. return memcmp(*(unsigned char**)a, *(unsigned char**)b, nu_val);
  192. }
  193. static size_t newopt(unsigned char *ut, size_t usize, int shift, table_data *tbl);
  194. #define MAXTO 255 /* Restrict to minimal unsigned char max. */
  195. int main(int argc, char **argv)
  196. {
  197. long int u, l, tt;
  198. size_t smallest, t;
  199. unsigned int c;
  200. unsigned int d;
  201. int i, n;
  202. int ul_count = 0;
  203. uldiff_entry uldiff[MAXTO];
  204. table_data cttable;
  205. table_data ultable;
  206. table_data combtable;
  207. table_data widthtable;
  208. long int last_comb = 0;
  209. unsigned char wct[(RANGE/2)+1]; /* wctype table (nibble per wchar) */
  210. unsigned char ult[RANGE+1]; /* upper/lower table */
  211. unsigned char combt[(RANGE/4)+1]; /* combining */
  212. unsigned char widtht[(RANGE/4)+1]; /* width */
  213. wctrans_t totitle;
  214. wctype_t is_comb, is_comb3;
  215. long int typecount[16];
  216. const char *typename[16];
  217. static const char empty_slot[] = "empty_slot";
  218. int built = 0;
  219. #define INIT_TYPENAME(X) typename[__CTYPE_##X] = "C_" #X
  220. for (i=0 ; i < 16 ; i++) {
  221. typename[i] = empty_slot;
  222. }
  223. INIT_TYPENAME(unclassified);
  224. INIT_TYPENAME(alpha_nonupper_nonlower);
  225. INIT_TYPENAME(alpha_lower);
  226. INIT_TYPENAME(alpha_upper_lower);
  227. INIT_TYPENAME(alpha_upper);
  228. INIT_TYPENAME(digit);
  229. INIT_TYPENAME(punct);
  230. INIT_TYPENAME(graph);
  231. INIT_TYPENAME(print_space_nonblank);
  232. INIT_TYPENAME(print_space_blank);
  233. INIT_TYPENAME(space_nonblank_noncntrl);
  234. INIT_TYPENAME(space_blank_noncntrl);
  235. INIT_TYPENAME(cntrl_space_nonblank);
  236. INIT_TYPENAME(cntrl_space_blank);
  237. INIT_TYPENAME(cntrl_nonspace);
  238. setvbuf(stdout, NULL, _IONBF, 0);
  239. while (--argc) {
  240. if (!setlocale(LC_CTYPE, *++argv)) {
  241. printf("setlocale(LC_CTYPE,%s) failed! Skipping this locale...\n", *argv);
  242. continue;
  243. }
  244. if (!(totitle = wctrans("totitle"))) {
  245. printf("no totitle transformation.\n");
  246. }
  247. if (!(is_comb = wctype("combining"))) {
  248. printf("no combining wctype.\n");
  249. }
  250. if (!(is_comb3 = wctype("combining_level3"))) {
  251. printf("no combining_level3 wctype.\n");
  252. }
  253. if (!built) {
  254. built = 1;
  255. ul_count = 1;
  256. uldiff[0].u = uldiff[0].l = 0;
  257. memset(wct, 0, sizeof(wct));
  258. memset(combt, 0, sizeof(combt));
  259. memset(widtht, 0, sizeof(widtht));
  260. for (i = 0 ; i < 16 ; i++) {
  261. typecount[i] = 0;
  262. }
  263. for (c=0 ; c <= RANGE ; c++) {
  264. if (iswdigit(c)) {
  265. d = __CTYPE_digit;
  266. } else if (iswalpha(c)) {
  267. d = __CTYPE_alpha_nonupper_nonlower;
  268. if (iswlower(c)) {
  269. d = __CTYPE_alpha_lower;
  270. if (iswupper(c)) {
  271. d = __CTYPE_alpha_upper_lower;
  272. }
  273. } else if (iswupper(c)) {
  274. d = __CTYPE_alpha_upper;
  275. }
  276. } else if (iswpunct(c)) {
  277. d = __CTYPE_punct;
  278. } else if (iswgraph(c)) {
  279. d = __CTYPE_graph;
  280. } else if (iswprint(c)) {
  281. d = __CTYPE_print_space_nonblank;
  282. if (iswblank(c)) {
  283. d = __CTYPE_print_space_blank;
  284. }
  285. } else if (iswspace(c) && !iswcntrl(c)) {
  286. d = __CTYPE_space_nonblank_noncntrl;
  287. if (iswblank(c)) {
  288. d = __CTYPE_space_blank_noncntrl;
  289. }
  290. } else if (iswcntrl(c)) {
  291. d = __CTYPE_cntrl_nonspace;
  292. if (iswspace(c)) {
  293. d = __CTYPE_cntrl_space_nonblank;
  294. if (iswblank(c)) {
  295. d = __CTYPE_cntrl_space_blank;
  296. }
  297. }
  298. } else {
  299. d = __CTYPE_unclassified;
  300. }
  301. ++typecount[d];
  302. #if 0
  303. if (iswspace(c)) {
  304. if (iswblank(c)) {
  305. printf("%#8x : space blank\n", c);
  306. } else {
  307. printf("%#8x : space\n", c);
  308. }
  309. }
  310. #endif
  311. #if 0
  312. if (c < 256) {
  313. unsigned int glibc;
  314. glibc = 0;
  315. if (isalnum(c)) ++glibc; glibc <<= 1;
  316. if (isalpha(c)) ++glibc; glibc <<= 1;
  317. if (isblank(c)) ++glibc; glibc <<= 1;
  318. if (iscntrl(c)) ++glibc; glibc <<= 1;
  319. if (isdigit(c)) ++glibc; glibc <<= 1;
  320. if (isgraph(c)) ++glibc; glibc <<= 1;
  321. if (islower(c)) ++glibc; glibc <<= 1;
  322. if (isprint(c)) ++glibc; glibc <<= 1;
  323. if (ispunct(c)) ++glibc; glibc <<= 1;
  324. if (isspace(c)) ++glibc; glibc <<= 1;
  325. if (isupper(c)) ++glibc; glibc <<= 1;
  326. if (isxdigit(c)) ++glibc;
  327. printf("%#8x : ctype %#4x\n", c, glibc);
  328. }
  329. #endif
  330. #if 1
  331. /* Paranoid checking... */
  332. {
  333. unsigned int glibc;
  334. unsigned int mine;
  335. glibc = 0;
  336. if (iswalnum(c)) ++glibc; glibc <<= 1;
  337. if (iswalpha(c)) ++glibc; glibc <<= 1;
  338. if (iswblank(c)) ++glibc; glibc <<= 1;
  339. if (iswcntrl(c)) ++glibc; glibc <<= 1;
  340. if (iswdigit(c)) ++glibc; glibc <<= 1;
  341. if (iswgraph(c)) ++glibc; glibc <<= 1;
  342. if (iswlower(c)) ++glibc; glibc <<= 1;
  343. if (iswprint(c)) ++glibc; glibc <<= 1;
  344. if (iswpunct(c)) ++glibc; glibc <<= 1;
  345. if (iswspace(c)) ++glibc; glibc <<= 1;
  346. if (iswupper(c)) ++glibc; glibc <<= 1;
  347. if (iswxdigit(c)) ++glibc;
  348. mine = 0;
  349. if (mywalnum(c)) ++mine; mine <<= 1;
  350. if (mywalpha(c)) ++mine; mine <<= 1;
  351. if (mywblank(c)) ++mine; mine <<= 1;
  352. if (mywcntrl(c)) ++mine; mine <<= 1;
  353. if (mywdigit(c)) ++mine; mine <<= 1;
  354. if (mywgraph(c)) ++mine; mine <<= 1;
  355. if (mywlower(c)) ++mine; mine <<= 1;
  356. if (mywprint(c)) ++mine; mine <<= 1;
  357. if (mywpunct(c)) ++mine; mine <<= 1;
  358. if (mywspace(c)) ++mine; mine <<= 1;
  359. if (mywupper(c)) ++mine; mine <<= 1;
  360. if (mywxdigit(c)) ++mine;
  361. if (glibc != mine) {
  362. printf("%#8x : glibc %#4x != %#4x mine %u\n", c, glibc, mine, d);
  363. return EXIT_FAILURE;
  364. }
  365. #if 0
  366. if (iswctype(c,is_comb) || iswctype(c,is_comb3)) {
  367. /* if (!iswpunct(c)) { */
  368. printf("%#8x : %d %d %#4x\n",
  369. c, iswctype(c,is_comb),iswctype(c,is_comb3), glibc);
  370. /* } */
  371. }
  372. #endif
  373. #if 0
  374. if (iswctype(c,is_comb) || iswctype(c,is_comb3)) {
  375. if (!last_comb) {
  376. printf("%#8x - ", c);
  377. last_comb = c;
  378. } else if (last_comb + 1 < c) {
  379. printf("%#8x\n%#8x - ", last_comb, c);
  380. last_comb = c;
  381. } else {
  382. last_comb = c;
  383. }
  384. }
  385. #endif
  386. }
  387. #endif
  388. combt[c/4] |= ((((!!iswctype(c,is_comb)) << 1) | !!iswctype(c,is_comb3))
  389. << ((c & 3) << 1));
  390. /* comb3t[c/8] |= ((!!iswctype(c,is_comb3)) << (c & 7)); */
  391. /* widtht[c/4] |= (wcwidth(c) << ((c & 3) << 1)); */
  392. if (c & 1) { /* Use the high nibble for odd numbered wchars. */
  393. d <<= 4;
  394. }
  395. wct[c/2] |= d;
  396. l = (long)(int) towlower(c) - c;
  397. u = (long)(int) towupper(c) - c;
  398. ult[c] = 0;
  399. if (l || u) {
  400. if ((l != (short)l) || (u != (short)u)) {
  401. printf("range assumption error! %x %ld %ld\n", c, l, u);
  402. return EXIT_FAILURE;
  403. }
  404. for (i=0 ; i < ul_count ; i++) {
  405. if ((l == uldiff[i].l) && (u == uldiff[i].u)) {
  406. goto found;
  407. }
  408. }
  409. uldiff[ul_count].l = l;
  410. uldiff[ul_count].u = u;
  411. ++ul_count;
  412. if (ul_count > MAXTO) {
  413. printf("too many touppers/tolowers!\n");
  414. return EXIT_FAILURE;
  415. }
  416. found:
  417. ult[c] = i;
  418. }
  419. }
  420. for (i = 0 ; i < 16 ; i++) {
  421. printf("typecount[%2d] = %8ld %s\n", i, typecount[i], typename[i]);
  422. }
  423. printf("optimizing is* table..\n");
  424. n = -1;
  425. smallest = SIZE_MAX;
  426. cttable.ii = NULL;
  427. for (i=0 ; i < 14 ; i++) {
  428. t = newopt(wct, (RANGE/2)+1, i, &cttable);
  429. if (smallest >= t) {
  430. n = i;
  431. smallest = t;
  432. /* } else { */
  433. /* break; */
  434. }
  435. }
  436. printf("smallest = %zu\n", smallest);
  437. if (!(cttable.ii = malloc(smallest))) {
  438. printf("couldn't allocate space!\n");
  439. return EXIT_FAILURE;
  440. }
  441. smallest = SIZE_MAX;
  442. newopt(wct, (RANGE/2)+1, n, &cttable);
  443. ++cttable.ti_shift; /* correct for nibble mode */
  444. printf("optimizing u/l-to table..\n");
  445. smallest = SIZE_MAX;
  446. ultable.ii = NULL;
  447. for (i=0 ; i < 14 ; i++) {
  448. t = newopt(ult, RANGE+1, i, &ultable);
  449. if (smallest >= t) {
  450. n = i;
  451. smallest = t;
  452. /* } else { */
  453. /* break; */
  454. }
  455. }
  456. printf("%zu (smallest) + %zu (u/l diffs) = %zu\n",
  457. smallest, 4 * ul_count, smallest + 4 * ul_count);
  458. printf("smallest = %zu\n", smallest);
  459. if (!(ultable.ii = malloc(smallest))) {
  460. printf("couldn't allocate space!\n");
  461. return EXIT_FAILURE;
  462. }
  463. smallest = SIZE_MAX;
  464. newopt(ult, RANGE+1, n, &ultable);
  465. #if 0
  466. printf("optimizing comb table..\n");
  467. smallest = SIZE_MAX;
  468. combtable.ii = NULL;
  469. for (i=0 ; i < 14 ; i++) {
  470. t = newopt(combt, sizeof(combt), i, &combtable);
  471. if (smallest >= t) {
  472. n = i;
  473. smallest = t;
  474. /* } else { */
  475. /* break; */
  476. }
  477. }
  478. printf("smallest = %zu\n", smallest);
  479. if (!(combtable.ii = malloc(smallest))) {
  480. printf("couldn't allocate space!\n");
  481. return EXIT_FAILURE;
  482. }
  483. smallest = SIZE_MAX;
  484. newopt(combt, sizeof(combt), n, &combtable);
  485. combtable.ti_shift += 4; /* correct for 4 entries per */
  486. #endif
  487. #if 0
  488. printf("optimizing width table..\n");
  489. smallest = SIZE_MAX;
  490. widthtable.ii = NULL;
  491. for (i=0 ; i < 14 ; i++) {
  492. t = newopt(widtht, sizeof(widtht), i, &widthtable);
  493. if (smallest >= t) {
  494. n = i;
  495. smallest = t;
  496. /* } else { */
  497. /* break; */
  498. }
  499. }
  500. printf("smallest = %zu\n", smallest);
  501. if (!(widthtable.ii = malloc(smallest))) {
  502. printf("couldn't allocate space!\n");
  503. return EXIT_FAILURE;
  504. }
  505. smallest = SIZE_MAX;
  506. newopt(widtht, sizeof(widtht), n, &widthtable);
  507. widthtable.ti_shift += 4; /* correct for 4 entries per */
  508. #endif
  509. #if 0
  510. printf("optimizing comb3 table..\n");
  511. smallest = SIZE_MAX;
  512. comb3table.ii = NULL;
  513. for (i=0 ; i < 14 ; i++) {
  514. t = newopt(comb3t, sizeof(comb3t), i, &comb3table);
  515. if (smallest >= t) {
  516. n = i;
  517. smallest = t;
  518. /* } else { */
  519. /* break; */
  520. }
  521. }
  522. printf("smallest = %zu\n", smallest);
  523. if (!(comb3table.ii = malloc(smallest))) {
  524. printf("couldn't allocate space!\n");
  525. return EXIT_FAILURE;
  526. }
  527. smallest = SIZE_MAX;
  528. newopt(comb3t, sizeof(comb3t), n, &comb3table);
  529. comb3table.ti_shift += 8; /* correct for 4 entries per */
  530. #endif
  531. dump_table_data(&cttable);
  532. dump_table_data(&ultable);
  533. dump_table_data(&combtable);
  534. }
  535. printf("verifying for %s...\n", *argv);
  536. #if RANGE == 0xffffU
  537. for (c=0 ; c <= 0xffffUL ; c++)
  538. #else
  539. for (c=0 ; c <= 0x10ffffUL ; c++)
  540. #endif
  541. {
  542. unsigned int glibc;
  543. unsigned int mine;
  544. unsigned int upper, lower;
  545. #if 0
  546. #if RANGE < 0x10000UL
  547. if (c == 0x10000UL) {
  548. c = 0x30000UL; /* skip 1st and 2nd sup planes */
  549. }
  550. #elif RANGE < 0x20000UL
  551. if (c == 0x20000UL) {
  552. c = 0x30000UL; /* skip 2nd sup planes */
  553. }
  554. #endif
  555. #endif
  556. glibc = 0;
  557. if (iswalnum(c)) ++glibc; glibc <<= 1;
  558. if (iswalpha(c)) ++glibc; glibc <<= 1;
  559. if (iswblank(c)) ++glibc; glibc <<= 1;
  560. if (iswcntrl(c)) ++glibc; glibc <<= 1;
  561. if (iswdigit(c)) ++glibc; glibc <<= 1;
  562. if (iswgraph(c)) ++glibc; glibc <<= 1;
  563. if (iswlower(c)) ++glibc; glibc <<= 1;
  564. if (iswprint(c)) ++glibc; glibc <<= 1;
  565. if (iswpunct(c)) ++glibc; glibc <<= 1;
  566. if (iswspace(c)) ++glibc; glibc <<= 1;
  567. if (iswupper(c)) ++glibc; glibc <<= 1;
  568. if (iswxdigit(c)) ++glibc;
  569. {
  570. unsigned int u;
  571. int n, sc;
  572. int i0, i1;
  573. u = c;
  574. if (u <= RANGE) {
  575. sc = u & ((1 << cttable.ti_shift) - 1);
  576. u >>= cttable.ti_shift;
  577. n = u & ((1 << cttable.ii_shift) - 1);
  578. u >>= cttable.ii_shift;
  579. i0 = cttable.ii[u];
  580. i0 <<= cttable.ii_shift;
  581. i1 = cttable.ti[i0 + n];
  582. i1 <<= (cttable.ti_shift-1);
  583. d = cttable.ut[i1 + (sc >> 1)];
  584. if (sc & 1) {
  585. d >>= 4;
  586. }
  587. d &= 0x0f;
  588. } else if ((((unsigned int)(c - 0xe0020UL)) <= 0x5f) || (c == 0xe0001UL)){
  589. d = __CTYPE_punct;
  590. } else if (((unsigned int)(c - 0xf0000UL)) < 0x20000UL) {
  591. if ((c & 0xffffU) <= 0xfffdU) {
  592. d = __CTYPE_punct;
  593. } else {
  594. d = __CTYPE_unclassified;
  595. }
  596. } else {
  597. d = __CTYPE_unclassified;
  598. }
  599. mine = 0;
  600. if (mywalnum(c)) ++mine; mine <<= 1;
  601. if (mywalpha(c)) ++mine; mine <<= 1;
  602. if (mywblank(c)) ++mine; mine <<= 1;
  603. if (mywcntrl(c)) ++mine; mine <<= 1;
  604. if (mywdigit(c)) ++mine; mine <<= 1;
  605. if (mywgraph(c)) ++mine; mine <<= 1;
  606. if (mywlower(c)) ++mine; mine <<= 1;
  607. if (mywprint(c)) ++mine; mine <<= 1;
  608. if (mywpunct(c)) ++mine; mine <<= 1;
  609. if (mywspace(c)) ++mine; mine <<= 1;
  610. if (mywupper(c)) ++mine; mine <<= 1;
  611. if (mywxdigit(c)) ++mine;
  612. if (glibc != mine) {
  613. printf("%#8x : glibc %#4x != %#4x mine %d\n", c, glibc, mine, d);
  614. if (c < 0x30000UL) {
  615. printf("sc=%#x u=%#x n=%#x i0=%#x i1=%#x\n", sc, u, n, i0, i1);
  616. }
  617. }
  618. upper = lower = u = c;
  619. if (u <= RANGE) {
  620. sc = u & ((1 << ultable.ti_shift) - 1);
  621. u >>= ultable.ti_shift;
  622. n = u & ((1 << ultable.ii_shift) - 1);
  623. u >>= ultable.ii_shift;
  624. i0 = ultable.ii[u];
  625. i0 <<= ultable.ii_shift;
  626. i1 = ultable.ti[i0 + n];
  627. i1 <<= (ultable.ti_shift);
  628. i1 += sc;
  629. i0 = ultable.ut[i1];
  630. upper = c + uldiff[i0].u;
  631. lower = c + uldiff[i0].l;
  632. }
  633. if (towupper(c) != upper) {
  634. printf("%#8x : towupper glibc %#4x != %#4x mine\n",
  635. c, towupper(c), upper);
  636. }
  637. if (towlower(c) != lower) {
  638. printf("%#8x : towlower glibc %#4x != %#4x mine i0 = %d\n",
  639. c, towlower(c), lower, i0);
  640. }
  641. if (totitle && ((tt = towctrans(c, totitle)) != upper)) {
  642. printf("%#8x : totitle glibc %#4lx != %#4x mine i0 = %d\n",
  643. c, tt, upper, i0);
  644. }
  645. }
  646. if ((c & 0xfff) == 0xfff) printf(".");
  647. }
  648. printf("done\n");
  649. }
  650. if (1) {
  651. FILE *fp;
  652. if (!(fp = fopen("wctables.h", "w"))) {
  653. printf("couldn't open wctables.h!\n");
  654. return EXIT_FAILURE;
  655. }
  656. fprintf(fp, "#define __LOCALE_DATA_WC_TABLE_DOMAIN_MAX %#8lx\n\n",
  657. (unsigned long) RANGE);
  658. output_table(fp, "ctype", &cttable);
  659. output_table(fp, "uplow", &ultable);
  660. #warning fix the upper bound on the upper/lower tables... save 200 bytes or so
  661. fprintf(fp, "#define __LOCALE_DATA_WCuplow_diffs %7u\n", ul_count);
  662. fprintf(fp, "\n#ifdef WANT_WCuplow_diff_data\n\n");
  663. fprintf(fp, "\nstatic const short __LOCALE_DATA_WCuplow_diff_data[%zu] = {",
  664. 2 * (size_t) ul_count);
  665. for (i=0 ; i < ul_count ; i++) {
  666. if (i % 4 == 0) {
  667. fprintf(fp, "\n");
  668. }
  669. fprintf(fp, " %6d, %6d,", uldiff[i].u, uldiff[i].l);
  670. }
  671. fprintf(fp, "\n};\n\n");
  672. fprintf(fp, "#endif /* WANT_WCuplow_diff_data */\n\n");
  673. /* output_table(fp, "comb", &combtable); */
  674. /* output_table(fp, "width", &widthtable); */
  675. fclose(fp);
  676. }
  677. return EXIT_SUCCESS;
  678. }
  679. size_t newopt(unsigned char *ut, size_t usize, int shift, table_data *tbl)
  680. {
  681. static int recurse;
  682. unsigned char *ti[RANGE+1]; /* table index */
  683. size_t numblocks;
  684. size_t blocksize;
  685. size_t uniq;
  686. size_t i, j;
  687. size_t smallest, t;
  688. unsigned char *ii_save;
  689. int uniqblock[256];
  690. unsigned char uit[RANGE+1];
  691. int shift2;
  692. memset(uniqblock, 0x00, sizeof(uniqblock));
  693. ii_save = NULL;
  694. blocksize = 1 << shift;
  695. numblocks = usize >> shift;
  696. /* init table index */
  697. for (i=j=0 ; i < numblocks ; i++) {
  698. ti[i] = ut + j;
  699. j += blocksize;
  700. }
  701. /* sort */
  702. nu_val = blocksize;
  703. qsort(ti, numblocks, sizeof(unsigned char *), nu_memcmp);
  704. uniq = 1;
  705. uit[(ti[0]-ut)/blocksize] = 0;
  706. for (i=1 ; i < numblocks ; i++) {
  707. if (memcmp(ti[i-1], ti[i], blocksize) < 0) {
  708. if (++uniq > 255) {
  709. break;
  710. }
  711. uniqblock[uniq - 1] = i;
  712. }
  713. #if 1
  714. else if (memcmp(ti[i-1], ti[i], blocksize) > 0) {
  715. printf("bad sort %i!\n", i);
  716. abort();
  717. }
  718. #endif
  719. uit[(ti[i]-ut)/blocksize] = uniq - 1;
  720. }
  721. smallest = SIZE_MAX;
  722. shift2 = -1;
  723. if (uniq <= 255) {
  724. smallest = numblocks + uniq * blocksize;
  725. if (!recurse) {
  726. ++recurse;
  727. for (j=1 ; j < 14 ; j++) {
  728. if ((numblocks >> j) < 2) break;
  729. if (tbl) {
  730. ii_save = tbl->ii;
  731. tbl->ii = NULL;
  732. }
  733. if ((t = newopt(uit, numblocks, j, tbl)) < SIZE_MAX) {
  734. t += uniq * blocksize;
  735. }
  736. if (tbl) {
  737. tbl->ii = ii_save;
  738. }
  739. if (smallest >= t) {
  740. shift2 = j;
  741. smallest = t;
  742. if (!tbl->ii) {
  743. printf("ishift %zu tshift %zu size %zu\n",
  744. shift2, shift, t);
  745. }
  746. /* } else { */
  747. /* break; */
  748. }
  749. }
  750. --recurse;
  751. }
  752. } else {
  753. return SIZE_MAX;
  754. }
  755. if (tbl->ii) {
  756. if (recurse) {
  757. tbl->ii_shift = shift;
  758. tbl->ii_len = numblocks;
  759. memcpy(tbl->ii, uit, numblocks);
  760. tbl->ti = tbl->ii + tbl->ii_len;
  761. tbl->ti_len = uniq * blocksize;
  762. for (i=0 ; i < uniq ; i++) {
  763. memcpy(tbl->ti + i * blocksize, ti[uniqblock[i]], blocksize);
  764. }
  765. } else {
  766. ++recurse;
  767. printf("setting ishift %zu tshift %zu\n",
  768. shift2, shift);
  769. newopt(uit, numblocks, shift2, tbl);
  770. --recurse;
  771. tbl->ti_shift = shift;
  772. tbl->ut_len = uniq * blocksize;
  773. tbl->ut = tbl->ti + tbl->ti_len;
  774. for (i=0 ; i < uniq ; i++) {
  775. memcpy(tbl->ut + i * blocksize, ti[uniqblock[i]], blocksize);
  776. }
  777. }
  778. }
  779. return smallest;
  780. }