gen_wctype.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880
  1. #include <stdio.h>
  2. #include <stdlib.h>
  3. #include <string.h>
  4. #include <locale.h>
  5. #include <wctype.h>
  6. #include <limits.h>
  7. #include <stdint.h>
  8. #include <wchar.h>
  9. #include <ctype.h>
  10. #ifdef __linux__
  11. #include <sys/resource.h>
  12. #endif
  13. #ifndef _CTYPE_H
  14. #define _CTYPE_H
  15. #endif
  16. #ifndef _WCTYPE_H
  17. #define _WCTYPE_H
  18. #endif
  19. #include <bits/uClibc_ctype.h>
  20. /* 0x9 : space blank */
  21. /* 0xa : space */
  22. /* 0xb : space */
  23. /* 0xc : space */
  24. /* 0xd : space */
  25. /* 0x20 : space blank */
  26. /* 0x1680 : space blank */
  27. /* 0x2000 : space blank */
  28. /* 0x2001 : space blank */
  29. /* 0x2002 : space blank */
  30. /* 0x2003 : space blank */
  31. /* 0x2004 : space blank */
  32. /* 0x2005 : space blank */
  33. /* 0x2006 : space blank */
  34. /* 0x2008 : space blank */
  35. /* 0x2009 : space blank */
  36. /* 0x200a : space blank */
  37. /* 0x200b : space blank */
  38. /* 0x2028 : space */
  39. /* 0x2029 : space */
  40. /* 0x3000 : space blank */
  41. /* typecount[ 0] = 88670 C_alpha_nonupper_nonlower */
  42. /* typecount[ 1] = 742 C_alpha_lower */
  43. /* typecount[ 2] = 4 C_alpha_upper_lower */
  44. /* typecount[ 3] = 731 C_alpha_upper */
  45. /* typecount[ 4] = 10 C_digit */
  46. /* typecount[ 5] = 10270 C_punct */
  47. /* typecount[ 6] = 0 C_graph */
  48. /* typecount[ 7] = 0 C_print_space_nonblank */
  49. /* typecount[ 8] = 14 C_print_space_blank */
  50. /* typecount[ 9] = 0 C_space_nonblank_noncntrl */
  51. /* typecount[10] = 0 C_space_blank_noncntrl */
  52. /* typecount[11] = 6 C_cntrl_space_nonblank */
  53. /* typecount[12] = 1 C_cntrl_space_blank */
  54. /* typecount[13] = 60 C_cntrl_nonspace */
  55. /* typecount[14] = 96100 C_unclassified */
  56. /* typecount[15] = 0 empty_slot */
  57. /* Set to #if 0 to restrict wchars to 16 bits. */
  58. #if 1
  59. #define RANGE 0x2ffffUL
  60. #elif 0
  61. #define RANGE 0x1ffffUL
  62. #else
  63. #define RANGE 0xffffUL /* Restrict for 16-bit wchar_t... */
  64. #endif
  65. #if 0
  66. /* Classification codes. */
  67. static const char *typename[] = {
  68. "C_unclassified",
  69. "C_alpha_nonupper_nonlower",
  70. "C_alpha_lower",
  71. "C_alpha_upper_lower",
  72. "C_alpha_upper",
  73. "C_digit",
  74. "C_punct",
  75. "C_graph",
  76. "C_print_space_nonblank",
  77. "C_print_space_blank",
  78. "C_space_nonblank_noncntrl",
  79. "C_space_blank_noncntrl",
  80. "C_cntrl_space_nonblank",
  81. "C_cntrl_space_blank",
  82. "C_cntrl_nonspace",
  83. "empty_slot"
  84. };
  85. #endif
  86. #if 0
  87. /* Taking advantage of the C99 mutual-exclusion guarantees for the various
  88. * (w)ctype classes, including the descriptions of printing and control
  89. * (w)chars, we can place each in one of the following mutually-exlusive
  90. * subsets. Since there are less than 16, we can store the data for
  91. * each (w)chars in a nibble. In contrast, glibc uses an unsigned int
  92. * per (w)char, with one bit flag for each is* type. While this allows
  93. * a simple '&' operation to determine the type vs. a range test and a
  94. * little special handling for the "blank" and "xdigit" types in my
  95. * approach, it also uses 8 times the space for the tables on the typical
  96. * 32-bit archs we supported.*/
  97. enum {
  98. __CTYPE_unclassified = 0,
  99. __CTYPE_alpha_nonupper_nonlower,
  100. __CTYPE_alpha_lower,
  101. __CTYPE_alpha_upper_lower,
  102. __CTYPE_alpha_upper,
  103. __CTYPE_digit,
  104. __CTYPE_punct,
  105. __CTYPE_graph,
  106. __CTYPE_print_space_nonblank,
  107. __CTYPE_print_space_blank,
  108. __CTYPE_space_nonblank_noncntrl,
  109. __CTYPE_space_blank_noncntrl,
  110. __CTYPE_cntrl_space_nonblank,
  111. __CTYPE_cntrl_space_blank,
  112. __CTYPE_cntrl_nonspace,
  113. };
  114. #endif
  115. #define __CTYPE_isxdigit(D,X) \
  116. (__CTYPE_isdigit(D) || (((unsigned int)(((X)|0x20) - 'a')) <= 5))
  117. #define mywalnum(x) __CTYPE_isalnum(d)
  118. #define mywalpha(x) __CTYPE_isalpha(d)
  119. #define mywblank(x) __CTYPE_isblank(d)
  120. #define mywcntrl(x) __CTYPE_iscntrl(d)
  121. #define mywdigit(x) __CTYPE_isdigit(d)
  122. #define mywgraph(x) __CTYPE_isgraph(d)
  123. #define mywlower(x) __CTYPE_islower(d)
  124. #define mywprint(x) __CTYPE_isprint(d)
  125. #define mywpunct(x) __CTYPE_ispunct(d)
  126. #define mywspace(x) __CTYPE_isspace(d)
  127. #define mywupper(x) __CTYPE_isupper(d)
  128. #define mywxdigit(x) __CTYPE_isxdigit(d,x)
  129. typedef struct {
  130. short l;
  131. short u;
  132. } uldiff_entry;
  133. typedef struct {
  134. uint16_t ii_len;
  135. uint16_t ti_len;
  136. uint16_t ut_len;
  137. unsigned char ii_shift;
  138. unsigned char ti_shift;
  139. unsigned char *ii;
  140. unsigned char *ti;
  141. unsigned char *ut;
  142. } table_data;
  143. void output_table(FILE *fp, const char *name, table_data *tbl)
  144. {
  145. size_t i;
  146. fprintf(fp, "#define __LOCALE_DATA_WC%s_II_LEN %7u\n", name, tbl->ii_len);
  147. fprintf(fp, "#define __LOCALE_DATA_WC%s_TI_LEN %7u\n", name, tbl->ti_len);
  148. fprintf(fp, "#define __LOCALE_DATA_WC%s_UT_LEN %7u\n", name, tbl->ut_len);
  149. fprintf(fp, "#define __LOCALE_DATA_WC%s_II_SHIFT %7u\n", name, tbl->ii_shift);
  150. fprintf(fp, "#define __LOCALE_DATA_WC%s_TI_SHIFT %7u\n", name, tbl->ti_shift);
  151. fprintf(fp, "\n#ifdef WANT_WC%s_data\n", name);
  152. i = tbl->ii_len + tbl->ti_len + tbl->ut_len;
  153. fprintf(fp, "\nstatic const unsigned char __LOCALE_DATA_WC%s_data[%zu] = {", name, i);
  154. for (i=0 ; i < tbl->ii_len ; i++) {
  155. if (i % 12 == 0) {
  156. fprintf(fp, "\n");
  157. }
  158. fprintf(fp, " %#04x,", tbl->ii[i]);
  159. }
  160. for (i=0 ; i < tbl->ti_len ; i++) {
  161. if (i % 12 == 0) {
  162. fprintf(fp, "\n");
  163. }
  164. fprintf(fp, " %#04x,", tbl->ti[i]);
  165. }
  166. for (i=0 ; i < tbl->ut_len ; i++) {
  167. if (i % 12 == 0) {
  168. fprintf(fp, "\n");
  169. }
  170. fprintf(fp, " %#04x,", tbl->ut[i]);
  171. }
  172. fprintf(fp, "\n};\n\n");
  173. fprintf(fp, "#endif /* WANT_WC%s_data */\n\n", name);
  174. }
  175. static void dump_table_data(table_data *tbl)
  176. {
  177. printf("ii_shift = %d ti_shift = %d\n"
  178. "ii_len = %d ti_len = %d ut_len = %d\n"
  179. "total = %d\n",
  180. tbl->ii_shift, tbl->ti_shift,
  181. tbl->ii_len, tbl->ti_len, tbl->ut_len,
  182. (int) tbl->ii_len + (int) tbl->ti_len + (int) tbl->ut_len);
  183. }
  184. /* For sorting the blocks of unsigned chars. */
  185. static size_t nu_val;
  186. int nu_memcmp(const void *a, const void *b)
  187. {
  188. return memcmp(*(unsigned char**)a, *(unsigned char**)b, nu_val);
  189. }
  190. static size_t newopt(unsigned char *ut, size_t usize, int shift, table_data *tbl);
  191. #define MAXTO 255 /* Restrict to minimal unsigned char max. */
  192. int main(int argc, char **argv)
  193. {
  194. long int u, l, tt;
  195. size_t smallest, t;
  196. unsigned int c;
  197. unsigned int d;
  198. int i, n;
  199. int ul_count = 0;
  200. uldiff_entry uldiff[MAXTO];
  201. table_data cttable;
  202. table_data ultable;
  203. table_data combtable;
  204. table_data widthtable;
  205. long int last_comb = 0;
  206. unsigned char wct[(RANGE/2)+1]; /* wctype table (nibble per wchar) */
  207. unsigned char ult[RANGE+1]; /* upper/lower table */
  208. unsigned char combt[(RANGE/4)+1]; /* combining */
  209. unsigned char widtht[(RANGE/4)+1]; /* width */
  210. wctrans_t totitle;
  211. wctype_t is_comb, is_comb3;
  212. long int typecount[16];
  213. const char *typename[16];
  214. static const char empty_slot[] = "empty_slot";
  215. int built = 0;
  216. #ifdef __linux__
  217. struct rlimit limit;
  218. limit.rlim_max = RLIM_INFINITY;
  219. limit.rlim_cur = RLIM_INFINITY;
  220. setrlimit(RLIMIT_STACK, &limit);
  221. #endif
  222. #define INIT_TYPENAME(X) typename[__CTYPE_##X] = "C_" #X
  223. for (i=0 ; i < 16 ; i++) {
  224. typename[i] = empty_slot;
  225. }
  226. INIT_TYPENAME(unclassified);
  227. INIT_TYPENAME(alpha_nonupper_nonlower);
  228. INIT_TYPENAME(alpha_lower);
  229. INIT_TYPENAME(alpha_upper_lower);
  230. INIT_TYPENAME(alpha_upper);
  231. INIT_TYPENAME(digit);
  232. INIT_TYPENAME(punct);
  233. INIT_TYPENAME(graph);
  234. INIT_TYPENAME(print_space_nonblank);
  235. INIT_TYPENAME(print_space_blank);
  236. INIT_TYPENAME(space_nonblank_noncntrl);
  237. INIT_TYPENAME(space_blank_noncntrl);
  238. INIT_TYPENAME(cntrl_space_nonblank);
  239. INIT_TYPENAME(cntrl_space_blank);
  240. INIT_TYPENAME(cntrl_nonspace);
  241. setvbuf(stdout, NULL, _IONBF, 0);
  242. while (--argc) {
  243. if (!setlocale(LC_CTYPE, *++argv)) {
  244. printf("setlocale(LC_CTYPE,%s) failed!\n", *argv);
  245. continue;
  246. }
  247. if (!(totitle = wctrans("totitle"))) {
  248. printf("no totitle transformation.\n");
  249. }
  250. if (!(is_comb = wctype("combining"))) {
  251. printf("no combining wctype.\n");
  252. }
  253. if (!(is_comb3 = wctype("combining_level3"))) {
  254. printf("no combining_level3 wctype.\n");
  255. }
  256. if (!built) {
  257. built = 1;
  258. ul_count = 1;
  259. uldiff[0].u = uldiff[0].l = 0;
  260. memset(wct, 0, sizeof(wct));
  261. memset(combt, 0, sizeof(combt));
  262. memset(widtht, 0, sizeof(widtht));
  263. for (i = 0 ; i < 16 ; i++) {
  264. typecount[i] = 0;
  265. }
  266. for (c=0 ; c <= RANGE ; c++) {
  267. if (iswdigit(c)) {
  268. d = __CTYPE_digit;
  269. } else if (iswalpha(c)) {
  270. d = __CTYPE_alpha_nonupper_nonlower;
  271. if (iswlower(c)) {
  272. d = __CTYPE_alpha_lower;
  273. if (iswupper(c)) {
  274. d = __CTYPE_alpha_upper_lower;
  275. }
  276. } else if (iswupper(c)) {
  277. d = __CTYPE_alpha_upper;
  278. }
  279. } else if (iswpunct(c)) {
  280. d = __CTYPE_punct;
  281. } else if (iswgraph(c)) {
  282. d = __CTYPE_graph;
  283. } else if (iswprint(c)) {
  284. d = __CTYPE_print_space_nonblank;
  285. if (iswblank(c)) {
  286. d = __CTYPE_print_space_blank;
  287. }
  288. } else if (iswspace(c) && !iswcntrl(c)) {
  289. d = __CTYPE_space_nonblank_noncntrl;
  290. if (iswblank(c)) {
  291. d = __CTYPE_space_blank_noncntrl;
  292. }
  293. } else if (iswcntrl(c)) {
  294. d = __CTYPE_cntrl_nonspace;
  295. if (iswspace(c)) {
  296. d = __CTYPE_cntrl_space_nonblank;
  297. if (iswblank(c)) {
  298. d = __CTYPE_cntrl_space_blank;
  299. }
  300. }
  301. } else {
  302. d = __CTYPE_unclassified;
  303. }
  304. ++typecount[d];
  305. #if 0
  306. if (iswspace(c)) {
  307. if (iswblank(c)) {
  308. printf("%#8x : space blank\n", c);
  309. } else {
  310. printf("%#8x : space\n", c);
  311. }
  312. }
  313. #endif
  314. #if 0
  315. if (c < 256) {
  316. unsigned int glibc;
  317. glibc = 0;
  318. if (isalnum(c)) ++glibc; glibc <<= 1;
  319. if (isalpha(c)) ++glibc; glibc <<= 1;
  320. if (isblank(c)) ++glibc; glibc <<= 1;
  321. if (iscntrl(c)) ++glibc; glibc <<= 1;
  322. if (isdigit(c)) ++glibc; glibc <<= 1;
  323. if (isgraph(c)) ++glibc; glibc <<= 1;
  324. if (islower(c)) ++glibc; glibc <<= 1;
  325. if (isprint(c)) ++glibc; glibc <<= 1;
  326. if (ispunct(c)) ++glibc; glibc <<= 1;
  327. if (isspace(c)) ++glibc; glibc <<= 1;
  328. if (isupper(c)) ++glibc; glibc <<= 1;
  329. if (isxdigit(c)) ++glibc;
  330. printf("%#8x : ctype %#4x\n", c, glibc);
  331. }
  332. #endif
  333. #if 1
  334. /* Paranoid checking... */
  335. {
  336. unsigned int glibc;
  337. unsigned int mine;
  338. glibc = 0;
  339. if (iswalnum(c)) ++glibc; glibc <<= 1;
  340. if (iswalpha(c)) ++glibc; glibc <<= 1;
  341. if (iswblank(c)) ++glibc; glibc <<= 1;
  342. if (iswcntrl(c)) ++glibc; glibc <<= 1;
  343. if (iswdigit(c)) ++glibc; glibc <<= 1;
  344. if (iswgraph(c)) ++glibc; glibc <<= 1;
  345. if (iswlower(c)) ++glibc; glibc <<= 1;
  346. if (iswprint(c)) ++glibc; glibc <<= 1;
  347. if (iswpunct(c)) ++glibc; glibc <<= 1;
  348. if (iswspace(c)) ++glibc; glibc <<= 1;
  349. if (iswupper(c)) ++glibc; glibc <<= 1;
  350. if (iswxdigit(c)) ++glibc;
  351. mine = 0;
  352. if (mywalnum(c)) ++mine; mine <<= 1;
  353. if (mywalpha(c)) ++mine; mine <<= 1;
  354. if (mywblank(c)) ++mine; mine <<= 1;
  355. if (mywcntrl(c)) ++mine; mine <<= 1;
  356. if (mywdigit(c)) ++mine; mine <<= 1;
  357. if (mywgraph(c)) ++mine; mine <<= 1;
  358. if (mywlower(c)) ++mine; mine <<= 1;
  359. if (mywprint(c)) ++mine; mine <<= 1;
  360. if (mywpunct(c)) ++mine; mine <<= 1;
  361. if (mywspace(c)) ++mine; mine <<= 1;
  362. if (mywupper(c)) ++mine; mine <<= 1;
  363. if (mywxdigit(c)) ++mine;
  364. if (glibc != mine) {
  365. printf("%#8x : glibc %#4x != %#4x mine %u\n", c, glibc, mine, d);
  366. return EXIT_FAILURE;
  367. }
  368. #if 0
  369. if (iswctype(c,is_comb) || iswctype(c,is_comb3)) {
  370. /* if (!iswpunct(c)) { */
  371. printf("%#8x : %d %d %#4x\n",
  372. c, iswctype(c,is_comb),iswctype(c,is_comb3), glibc);
  373. /* } */
  374. }
  375. #endif
  376. #if 0
  377. if (iswctype(c,is_comb) || iswctype(c,is_comb3)) {
  378. if (!last_comb) {
  379. printf("%#8x - ", c);
  380. last_comb = c;
  381. } else if (last_comb + 1 < c) {
  382. printf("%#8x\n%#8x - ", last_comb, c);
  383. last_comb = c;
  384. } else {
  385. last_comb = c;
  386. }
  387. }
  388. #endif
  389. }
  390. #endif
  391. combt[c/4] |= ((((!!iswctype(c,is_comb)) << 1) | !!iswctype(c,is_comb3))
  392. << ((c & 3) << 1));
  393. /* comb3t[c/8] |= ((!!iswctype(c,is_comb3)) << (c & 7)); */
  394. /* widtht[c/4] |= (wcwidth(c) << ((c & 3) << 1)); */
  395. if (c & 1) { /* Use the high nibble for odd numbered wchars. */
  396. d <<= 4;
  397. }
  398. wct[c/2] |= d;
  399. l = (long)(int) towlower(c) - c;
  400. u = (long)(int) towupper(c) - c;
  401. ult[c] = 0;
  402. if (l || u) {
  403. if ((l != (short)l) || (u != (short)u)) {
  404. printf("range assumption error! %x %ld %ld\n", c, l, u);
  405. return EXIT_FAILURE;
  406. }
  407. for (i=0 ; i < ul_count ; i++) {
  408. if ((l == uldiff[i].l) && (u == uldiff[i].u)) {
  409. goto found;
  410. }
  411. }
  412. uldiff[ul_count].l = l;
  413. uldiff[ul_count].u = u;
  414. ++ul_count;
  415. if (ul_count > MAXTO) {
  416. printf("too many touppers/tolowers!\n");
  417. return EXIT_FAILURE;
  418. }
  419. found:
  420. ult[c] = i;
  421. }
  422. }
  423. for (i = 0 ; i < 16 ; i++) {
  424. printf("typecount[%2d] = %8ld %s\n", i, typecount[i], typename[i]);
  425. }
  426. printf("optimizing is* table..\n");
  427. n = -1;
  428. smallest = SIZE_MAX;
  429. cttable.ii = NULL;
  430. for (i=0 ; i < 14 ; i++) {
  431. t = newopt(wct, (RANGE/2)+1, i, &cttable);
  432. if (smallest >= t) {
  433. n = i;
  434. smallest = t;
  435. /* } else { */
  436. /* break; */
  437. }
  438. }
  439. printf("smallest = %zu\n", smallest);
  440. if (!(cttable.ii = malloc(smallest))) {
  441. printf("couldn't allocate space!\n");
  442. return EXIT_FAILURE;
  443. }
  444. smallest = SIZE_MAX;
  445. newopt(wct, (RANGE/2)+1, n, &cttable);
  446. ++cttable.ti_shift; /* correct for nibble mode */
  447. printf("optimizing u/l-to table..\n");
  448. smallest = SIZE_MAX;
  449. ultable.ii = NULL;
  450. for (i=0 ; i < 14 ; i++) {
  451. t = newopt(ult, RANGE+1, i, &ultable);
  452. if (smallest >= t) {
  453. n = i;
  454. smallest = t;
  455. /* } else { */
  456. /* break; */
  457. }
  458. }
  459. printf("%zu (smallest) + %zu (u/l diffs) = %zu\n",
  460. smallest, 4 * ul_count, smallest + 4 * ul_count);
  461. printf("smallest = %zu\n", smallest);
  462. if (!(ultable.ii = malloc(smallest))) {
  463. printf("couldn't allocate space!\n");
  464. return EXIT_FAILURE;
  465. }
  466. smallest = SIZE_MAX;
  467. newopt(ult, RANGE+1, n, &ultable);
  468. #if 0
  469. printf("optimizing comb table..\n");
  470. smallest = SIZE_MAX;
  471. combtable.ii = NULL;
  472. for (i=0 ; i < 14 ; i++) {
  473. t = newopt(combt, sizeof(combt), i, &combtable);
  474. if (smallest >= t) {
  475. n = i;
  476. smallest = t;
  477. /* } else { */
  478. /* break; */
  479. }
  480. }
  481. printf("smallest = %zu\n", smallest);
  482. if (!(combtable.ii = malloc(smallest))) {
  483. printf("couldn't allocate space!\n");
  484. return EXIT_FAILURE;
  485. }
  486. smallest = SIZE_MAX;
  487. newopt(combt, sizeof(combt), n, &combtable);
  488. combtable.ti_shift += 4; /* correct for 4 entries per */
  489. #endif
  490. #if 0
  491. printf("optimizing width table..\n");
  492. smallest = SIZE_MAX;
  493. widthtable.ii = NULL;
  494. for (i=0 ; i < 14 ; i++) {
  495. t = newopt(widtht, sizeof(widtht), i, &widthtable);
  496. if (smallest >= t) {
  497. n = i;
  498. smallest = t;
  499. /* } else { */
  500. /* break; */
  501. }
  502. }
  503. printf("smallest = %zu\n", smallest);
  504. if (!(widthtable.ii = malloc(smallest))) {
  505. printf("couldn't allocate space!\n");
  506. return EXIT_FAILURE;
  507. }
  508. smallest = SIZE_MAX;
  509. newopt(widtht, sizeof(widtht), n, &widthtable);
  510. widthtable.ti_shift += 4; /* correct for 4 entries per */
  511. #endif
  512. #if 0
  513. printf("optimizing comb3 table..\n");
  514. smallest = SIZE_MAX;
  515. comb3table.ii = NULL;
  516. for (i=0 ; i < 14 ; i++) {
  517. t = newopt(comb3t, sizeof(comb3t), i, &comb3table);
  518. if (smallest >= t) {
  519. n = i;
  520. smallest = t;
  521. /* } else { */
  522. /* break; */
  523. }
  524. }
  525. printf("smallest = %zu\n", smallest);
  526. if (!(comb3table.ii = malloc(smallest))) {
  527. printf("couldn't allocate space!\n");
  528. return EXIT_FAILURE;
  529. }
  530. smallest = SIZE_MAX;
  531. newopt(comb3t, sizeof(comb3t), n, &comb3table);
  532. comb3table.ti_shift += 8; /* correct for 4 entries per */
  533. #endif
  534. dump_table_data(&cttable);
  535. dump_table_data(&ultable);
  536. dump_table_data(&combtable);
  537. }
  538. printf("verifying for %s...\n", *argv);
  539. #if RANGE == 0xffffU
  540. for (c=0 ; c <= 0xffffUL ; c++)
  541. #else
  542. for (c=0 ; c <= 0x10ffffUL ; c++)
  543. #endif
  544. {
  545. unsigned int glibc;
  546. unsigned int mine;
  547. unsigned int upper, lower;
  548. #if 0
  549. #if RANGE < 0x10000UL
  550. if (c == 0x10000UL) {
  551. c = 0x30000UL; /* skip 1st and 2nd sup planes */
  552. }
  553. #elif RANGE < 0x20000UL
  554. if (c == 0x20000UL) {
  555. c = 0x30000UL; /* skip 2nd sup planes */
  556. }
  557. #endif
  558. #endif
  559. glibc = 0;
  560. if (iswalnum(c)) ++glibc; glibc <<= 1;
  561. if (iswalpha(c)) ++glibc; glibc <<= 1;
  562. if (iswblank(c)) ++glibc; glibc <<= 1;
  563. if (iswcntrl(c)) ++glibc; glibc <<= 1;
  564. if (iswdigit(c)) ++glibc; glibc <<= 1;
  565. if (iswgraph(c)) ++glibc; glibc <<= 1;
  566. if (iswlower(c)) ++glibc; glibc <<= 1;
  567. if (iswprint(c)) ++glibc; glibc <<= 1;
  568. if (iswpunct(c)) ++glibc; glibc <<= 1;
  569. if (iswspace(c)) ++glibc; glibc <<= 1;
  570. if (iswupper(c)) ++glibc; glibc <<= 1;
  571. if (iswxdigit(c)) ++glibc;
  572. {
  573. unsigned int u;
  574. int n, sc;
  575. int i0, i1;
  576. u = c;
  577. if (u <= RANGE) {
  578. sc = u & ((1 << cttable.ti_shift) - 1);
  579. u >>= cttable.ti_shift;
  580. n = u & ((1 << cttable.ii_shift) - 1);
  581. u >>= cttable.ii_shift;
  582. i0 = cttable.ii[u];
  583. i0 <<= cttable.ii_shift;
  584. i1 = cttable.ti[i0 + n];
  585. i1 <<= (cttable.ti_shift-1);
  586. d = cttable.ut[i1 + (sc >> 1)];
  587. if (sc & 1) {
  588. d >>= 4;
  589. }
  590. d &= 0x0f;
  591. } else if ((((unsigned int)(c - 0xe0020UL)) <= 0x5f) || (c == 0xe0001UL)){
  592. d = __CTYPE_punct;
  593. } else if (((unsigned int)(c - 0xf0000UL)) < 0x20000UL) {
  594. if ((c & 0xffffU) <= 0xfffdU) {
  595. d = __CTYPE_punct;
  596. } else {
  597. d = __CTYPE_unclassified;
  598. }
  599. } else {
  600. d = __CTYPE_unclassified;
  601. }
  602. mine = 0;
  603. if (mywalnum(c)) ++mine; mine <<= 1;
  604. if (mywalpha(c)) ++mine; mine <<= 1;
  605. if (mywblank(c)) ++mine; mine <<= 1;
  606. if (mywcntrl(c)) ++mine; mine <<= 1;
  607. if (mywdigit(c)) ++mine; mine <<= 1;
  608. if (mywgraph(c)) ++mine; mine <<= 1;
  609. if (mywlower(c)) ++mine; mine <<= 1;
  610. if (mywprint(c)) ++mine; mine <<= 1;
  611. if (mywpunct(c)) ++mine; mine <<= 1;
  612. if (mywspace(c)) ++mine; mine <<= 1;
  613. if (mywupper(c)) ++mine; mine <<= 1;
  614. if (mywxdigit(c)) ++mine;
  615. if (glibc != mine) {
  616. printf("%#8x : glibc %#4x != %#4x mine %d\n", c, glibc, mine, d);
  617. if (c < 0x30000UL) {
  618. printf("sc=%#x u=%#x n=%#x i0=%#x i1=%#x\n", sc, u, n, i0, i1);
  619. }
  620. }
  621. upper = lower = u = c;
  622. if (u <= RANGE) {
  623. sc = u & ((1 << ultable.ti_shift) - 1);
  624. u >>= ultable.ti_shift;
  625. n = u & ((1 << ultable.ii_shift) - 1);
  626. u >>= ultable.ii_shift;
  627. i0 = ultable.ii[u];
  628. i0 <<= ultable.ii_shift;
  629. i1 = ultable.ti[i0 + n];
  630. i1 <<= (ultable.ti_shift);
  631. i1 += sc;
  632. i0 = ultable.ut[i1];
  633. upper = c + uldiff[i0].u;
  634. lower = c + uldiff[i0].l;
  635. }
  636. if (towupper(c) != upper) {
  637. printf("%#8x : towupper glibc %#4x != %#4x mine\n",
  638. c, towupper(c), upper);
  639. }
  640. if (towlower(c) != lower) {
  641. printf("%#8x : towlower glibc %#4x != %#4x mine i0 = %d\n",
  642. c, towlower(c), lower, i0);
  643. }
  644. if (totitle && ((tt = towctrans(c, totitle)) != upper)) {
  645. printf("%#8x : totitle glibc %#4lx != %#4x mine i0 = %d\n",
  646. c, tt, upper, i0);
  647. }
  648. }
  649. if ((c & 0xfff) == 0xfff) printf(".");
  650. }
  651. printf("done\n");
  652. }
  653. if (1) {
  654. FILE *fp;
  655. if (!(fp = fopen("wctables.h", "w"))) {
  656. printf("couldn't open wctables.h!\n");
  657. return EXIT_FAILURE;
  658. }
  659. fprintf(fp, "#define __LOCALE_DATA_WC_TABLE_DOMAIN_MAX %#8lx\n\n",
  660. (unsigned long) RANGE);
  661. output_table(fp, "ctype", &cttable);
  662. output_table(fp, "uplow", &ultable);
  663. #warning fix the upper bound on the upper/lower tables... save 200 bytes or so
  664. fprintf(fp, "#define __LOCALE_DATA_WCuplow_diffs %7u\n", ul_count);
  665. fprintf(fp, "\n#ifdef WANT_WCuplow_diff_data\n\n");
  666. fprintf(fp, "\nstatic const short __LOCALE_DATA_WCuplow_diff_data[%zu] = {",
  667. 2 * (size_t) ul_count);
  668. for (i=0 ; i < ul_count ; i++) {
  669. if (i % 4 == 0) {
  670. fprintf(fp, "\n");
  671. }
  672. fprintf(fp, " %6d, %6d,", uldiff[i].u, uldiff[i].l);
  673. }
  674. fprintf(fp, "\n};\n\n");
  675. fprintf(fp, "#endif /* WANT_WCuplow_diff_data */\n\n");
  676. /* output_table(fp, "comb", &combtable); */
  677. /* output_table(fp, "width", &widthtable); */
  678. fclose(fp);
  679. }
  680. return EXIT_SUCCESS;
  681. }
  682. size_t newopt(unsigned char *ut, size_t usize, int shift, table_data *tbl)
  683. {
  684. static int recurse = 0;
  685. unsigned char *ti[RANGE+1]; /* table index */
  686. size_t numblocks;
  687. size_t blocksize;
  688. size_t uniq;
  689. size_t i, j;
  690. size_t smallest, t;
  691. unsigned char *ii_save;
  692. int uniqblock[256];
  693. unsigned char uit[RANGE+1];
  694. int shift2;
  695. ii_save = NULL;
  696. blocksize = 1 << shift;
  697. numblocks = usize >> shift;
  698. /* init table index */
  699. for (i=j=0 ; i < numblocks ; i++) {
  700. ti[i] = ut + j;
  701. j += blocksize;
  702. }
  703. /* sort */
  704. nu_val = blocksize;
  705. qsort(ti, numblocks, sizeof(unsigned char *), nu_memcmp);
  706. uniq = 1;
  707. uit[(ti[0]-ut)/blocksize] = 0;
  708. for (i=1 ; i < numblocks ; i++) {
  709. if (memcmp(ti[i-1], ti[i], blocksize) < 0) {
  710. if (++uniq > 255) {
  711. break;
  712. }
  713. uniqblock[uniq - 1] = i;
  714. }
  715. #if 1
  716. else if (memcmp(ti[i-1], ti[i], blocksize) > 0) {
  717. printf("bad sort %i!\n", i);
  718. abort();
  719. }
  720. #endif
  721. uit[(ti[i]-ut)/blocksize] = uniq - 1;
  722. }
  723. smallest = SIZE_MAX;
  724. shift2 = -1;
  725. if (uniq <= 255) {
  726. smallest = numblocks + uniq * blocksize;
  727. if (!recurse) {
  728. ++recurse;
  729. for (j=1 ; j < 14 ; j++) {
  730. if ((numblocks >> j) < 2) break;
  731. if (tbl) {
  732. ii_save = tbl->ii;
  733. tbl->ii = NULL;
  734. }
  735. if ((t = newopt(uit, numblocks, j, tbl)) < SIZE_MAX) {
  736. t += uniq * blocksize;
  737. }
  738. if (tbl) {
  739. tbl->ii = ii_save;
  740. }
  741. if (smallest >= t) {
  742. shift2 = j;
  743. smallest = t;
  744. if (!tbl->ii) {
  745. printf("ishift %zu tshift %zu size %zu\n",
  746. shift2, shift, t);
  747. }
  748. /* } else { */
  749. /* break; */
  750. }
  751. }
  752. --recurse;
  753. }
  754. } else {
  755. return SIZE_MAX;
  756. }
  757. if (tbl->ii) {
  758. if (recurse) {
  759. tbl->ii_shift = shift;
  760. tbl->ii_len = numblocks;
  761. memcpy(tbl->ii, uit, numblocks);
  762. tbl->ti = tbl->ii + tbl->ii_len;
  763. tbl->ti_len = uniq * blocksize;
  764. for (i=0 ; i < uniq ; i++) {
  765. memcpy(tbl->ti + i * blocksize, ti[uniqblock[i]], blocksize);
  766. }
  767. } else {
  768. ++recurse;
  769. printf("setting ishift %zu tshift %zu\n",
  770. shift2, shift);
  771. newopt(uit, numblocks, shift2, tbl);
  772. --recurse;
  773. tbl->ti_shift = shift;
  774. tbl->ut_len = uniq * blocksize;
  775. tbl->ut = tbl->ti + tbl->ti_len;
  776. for (i=0 ; i < uniq ; i++) {
  777. memcpy(tbl->ut + i * blocksize, ti[uniqblock[i]], blocksize);
  778. }
  779. }
  780. }
  781. return smallest;
  782. }