gen_wctype.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882
  1. #define _GNU_SOURCE
  2. #include <stdio.h>
  3. #include <stdlib.h>
  4. #include <string.h>
  5. #include <locale.h>
  6. #include <wctype.h>
  7. #include <limits.h>
  8. #include <stdint.h>
  9. #include <wchar.h>
  10. #include <ctype.h>
  11. #ifdef __linux__
  12. #include <sys/resource.h>
  13. #endif
  14. #ifndef _CTYPE_H
  15. #define _CTYPE_H
  16. #endif
  17. #ifndef _WCTYPE_H
  18. #define _WCTYPE_H
  19. #endif
  20. #include "../../libc/sysdeps/linux/common/bits/uClibc_ctype.h"
  21. /* 0x9 : space blank */
  22. /* 0xa : space */
  23. /* 0xb : space */
  24. /* 0xc : space */
  25. /* 0xd : space */
  26. /* 0x20 : space blank */
  27. /* 0x1680 : space blank */
  28. /* 0x2000 : space blank */
  29. /* 0x2001 : space blank */
  30. /* 0x2002 : space blank */
  31. /* 0x2003 : space blank */
  32. /* 0x2004 : space blank */
  33. /* 0x2005 : space blank */
  34. /* 0x2006 : space blank */
  35. /* 0x2008 : space blank */
  36. /* 0x2009 : space blank */
  37. /* 0x200a : space blank */
  38. /* 0x200b : space blank */
  39. /* 0x2028 : space */
  40. /* 0x2029 : space */
  41. /* 0x3000 : space blank */
  42. /* typecount[ 0] = 88670 C_alpha_nonupper_nonlower */
  43. /* typecount[ 1] = 742 C_alpha_lower */
  44. /* typecount[ 2] = 4 C_alpha_upper_lower */
  45. /* typecount[ 3] = 731 C_alpha_upper */
  46. /* typecount[ 4] = 10 C_digit */
  47. /* typecount[ 5] = 10270 C_punct */
  48. /* typecount[ 6] = 0 C_graph */
  49. /* typecount[ 7] = 0 C_print_space_nonblank */
  50. /* typecount[ 8] = 14 C_print_space_blank */
  51. /* typecount[ 9] = 0 C_space_nonblank_noncntrl */
  52. /* typecount[10] = 0 C_space_blank_noncntrl */
  53. /* typecount[11] = 6 C_cntrl_space_nonblank */
  54. /* typecount[12] = 1 C_cntrl_space_blank */
  55. /* typecount[13] = 60 C_cntrl_nonspace */
  56. /* typecount[14] = 96100 C_unclassified */
  57. /* typecount[15] = 0 empty_slot */
  58. /* Set to #if 0 to restrict wchars to 16 bits. */
  59. #if 1
  60. #define RANGE 0x2ffffUL
  61. #elif 0
  62. #define RANGE 0x1ffffUL
  63. #else
  64. #define RANGE 0xffffUL /* Restrict for 16-bit wchar_t... */
  65. #endif
  66. #if 0
  67. /* Classification codes. */
  68. static const char *typename[] = {
  69. "C_unclassified",
  70. "C_alpha_nonupper_nonlower",
  71. "C_alpha_lower",
  72. "C_alpha_upper_lower",
  73. "C_alpha_upper",
  74. "C_digit",
  75. "C_punct",
  76. "C_graph",
  77. "C_print_space_nonblank",
  78. "C_print_space_blank",
  79. "C_space_nonblank_noncntrl",
  80. "C_space_blank_noncntrl",
  81. "C_cntrl_space_nonblank",
  82. "C_cntrl_space_blank",
  83. "C_cntrl_nonspace",
  84. "empty_slot"
  85. };
  86. #endif
  87. #if 0
  88. /* Taking advantage of the C99 mutual-exclusion guarantees for the various
  89. * (w)ctype classes, including the descriptions of printing and control
  90. * (w)chars, we can place each in one of the following mutually-exlusive
  91. * subsets. Since there are less than 16, we can store the data for
  92. * each (w)chars in a nibble. In contrast, glibc uses an unsigned int
  93. * per (w)char, with one bit flag for each is* type. While this allows
  94. * a simple '&' operation to determine the type vs. a range test and a
  95. * little special handling for the "blank" and "xdigit" types in my
  96. * approach, it also uses 8 times the space for the tables on the typical
  97. * 32-bit archs we supported.*/
  98. enum {
  99. __CTYPE_unclassified = 0,
  100. __CTYPE_alpha_nonupper_nonlower,
  101. __CTYPE_alpha_lower,
  102. __CTYPE_alpha_upper_lower,
  103. __CTYPE_alpha_upper,
  104. __CTYPE_digit,
  105. __CTYPE_punct,
  106. __CTYPE_graph,
  107. __CTYPE_print_space_nonblank,
  108. __CTYPE_print_space_blank,
  109. __CTYPE_space_nonblank_noncntrl,
  110. __CTYPE_space_blank_noncntrl,
  111. __CTYPE_cntrl_space_nonblank,
  112. __CTYPE_cntrl_space_blank,
  113. __CTYPE_cntrl_nonspace,
  114. };
  115. #endif
  116. #define __CTYPE_isxdigit(D,X) \
  117. (__CTYPE_isdigit(D) || (((unsigned int)(((X)|0x20) - 'a')) <= 5))
  118. #define mywalnum(x) __CTYPE_isalnum(d)
  119. #define mywalpha(x) __CTYPE_isalpha(d)
  120. #define mywblank(x) __CTYPE_isblank(d)
  121. #define mywcntrl(x) __CTYPE_iscntrl(d)
  122. #define mywdigit(x) __CTYPE_isdigit(d)
  123. #define mywgraph(x) __CTYPE_isgraph(d)
  124. #define mywlower(x) __CTYPE_islower(d)
  125. #define mywprint(x) __CTYPE_isprint(d)
  126. #define mywpunct(x) __CTYPE_ispunct(d)
  127. #define mywspace(x) __CTYPE_isspace(d)
  128. #define mywupper(x) __CTYPE_isupper(d)
  129. #define mywxdigit(x) __CTYPE_isxdigit(d,x)
  130. typedef struct {
  131. short l;
  132. short u;
  133. } uldiff_entry;
  134. typedef struct {
  135. uint16_t ii_len;
  136. uint16_t ti_len;
  137. uint16_t ut_len;
  138. unsigned char ii_shift;
  139. unsigned char ti_shift;
  140. unsigned char *ii;
  141. unsigned char *ti;
  142. unsigned char *ut;
  143. } table_data;
  144. void output_table(FILE *fp, const char *name, table_data *tbl)
  145. {
  146. size_t i;
  147. fprintf(fp, "#define __LOCALE_DATA_WC%s_II_LEN %7u\n", name, tbl->ii_len);
  148. fprintf(fp, "#define __LOCALE_DATA_WC%s_TI_LEN %7u\n", name, tbl->ti_len);
  149. fprintf(fp, "#define __LOCALE_DATA_WC%s_UT_LEN %7u\n", name, tbl->ut_len);
  150. fprintf(fp, "#define __LOCALE_DATA_WC%s_II_SHIFT %7u\n", name, tbl->ii_shift);
  151. fprintf(fp, "#define __LOCALE_DATA_WC%s_TI_SHIFT %7u\n", name, tbl->ti_shift);
  152. fprintf(fp, "\n#ifdef WANT_WC%s_data\n", name);
  153. i = tbl->ii_len + tbl->ti_len + tbl->ut_len;
  154. fprintf(fp, "\nstatic const unsigned char __LOCALE_DATA_WC%s_data[%zu] = {", name, i);
  155. for (i=0 ; i < tbl->ii_len ; i++) {
  156. if (i % 12 == 0) {
  157. fprintf(fp, "\n");
  158. }
  159. fprintf(fp, " %#04x,", tbl->ii[i]);
  160. }
  161. for (i=0 ; i < tbl->ti_len ; i++) {
  162. if (i % 12 == 0) {
  163. fprintf(fp, "\n");
  164. }
  165. fprintf(fp, " %#04x,", tbl->ti[i]);
  166. }
  167. for (i=0 ; i < tbl->ut_len ; i++) {
  168. if (i % 12 == 0) {
  169. fprintf(fp, "\n");
  170. }
  171. fprintf(fp, " %#04x,", tbl->ut[i]);
  172. }
  173. fprintf(fp, "\n};\n\n");
  174. fprintf(fp, "#endif /* WANT_WC%s_data */\n\n", name);
  175. }
  176. static void dump_table_data(table_data *tbl)
  177. {
  178. printf("ii_shift = %d ti_shift = %d\n"
  179. "ii_len = %d ti_len = %d ut_len = %d\n"
  180. "total = %d\n",
  181. tbl->ii_shift, tbl->ti_shift,
  182. tbl->ii_len, tbl->ti_len, tbl->ut_len,
  183. (int) tbl->ii_len + (int) tbl->ti_len + (int) tbl->ut_len);
  184. }
  185. /* For sorting the blocks of unsigned chars. */
  186. static size_t nu_val;
  187. int nu_memcmp(const void *a, const void *b)
  188. {
  189. return memcmp(*(unsigned char**)a, *(unsigned char**)b, nu_val);
  190. }
  191. static size_t newopt(unsigned char *ut, size_t usize, int shift, table_data *tbl);
  192. #define MAXTO 255 /* Restrict to minimal unsigned char max. */
  193. int main(int argc, char **argv)
  194. {
  195. long int u, l, tt;
  196. size_t smallest, t;
  197. unsigned int c;
  198. unsigned int d;
  199. int i, n;
  200. int ul_count = 0;
  201. uldiff_entry uldiff[MAXTO];
  202. table_data cttable;
  203. table_data ultable;
  204. table_data combtable;
  205. table_data widthtable;
  206. long int last_comb = 0;
  207. unsigned char wct[(RANGE/2)+1]; /* wctype table (nibble per wchar) */
  208. unsigned char ult[RANGE+1]; /* upper/lower table */
  209. unsigned char combt[(RANGE/4)+1]; /* combining */
  210. unsigned char widtht[(RANGE/4)+1]; /* width */
  211. wctrans_t totitle;
  212. wctype_t is_comb, is_comb3;
  213. long int typecount[16];
  214. const char *typename[16];
  215. static const char empty_slot[] = "empty_slot";
  216. int built = 0;
  217. #ifdef __linux__
  218. struct rlimit limit;
  219. limit.rlim_max = RLIM_INFINITY;
  220. limit.rlim_cur = RLIM_INFINITY;
  221. setrlimit(RLIMIT_STACK, &limit);
  222. #endif
  223. #define INIT_TYPENAME(X) typename[__CTYPE_##X] = "C_" #X
  224. for (i=0 ; i < 16 ; i++) {
  225. typename[i] = empty_slot;
  226. }
  227. INIT_TYPENAME(unclassified);
  228. INIT_TYPENAME(alpha_nonupper_nonlower);
  229. INIT_TYPENAME(alpha_lower);
  230. INIT_TYPENAME(alpha_upper_lower);
  231. INIT_TYPENAME(alpha_upper);
  232. INIT_TYPENAME(digit);
  233. INIT_TYPENAME(punct);
  234. INIT_TYPENAME(graph);
  235. INIT_TYPENAME(print_space_nonblank);
  236. INIT_TYPENAME(print_space_blank);
  237. INIT_TYPENAME(space_nonblank_noncntrl);
  238. INIT_TYPENAME(space_blank_noncntrl);
  239. INIT_TYPENAME(cntrl_space_nonblank);
  240. INIT_TYPENAME(cntrl_space_blank);
  241. INIT_TYPENAME(cntrl_nonspace);
  242. setvbuf(stdout, NULL, _IONBF, 0);
  243. while (--argc) {
  244. if (!setlocale(LC_CTYPE, *++argv)) {
  245. printf("setlocale(LC_CTYPE,%s) failed!\n", *argv);
  246. continue;
  247. }
  248. if (!(totitle = wctrans("totitle"))) {
  249. printf("no totitle transformation.\n");
  250. }
  251. if (!(is_comb = wctype("combining"))) {
  252. printf("no combining wctype.\n");
  253. }
  254. if (!(is_comb3 = wctype("combining_level3"))) {
  255. printf("no combining_level3 wctype.\n");
  256. }
  257. if (!built) {
  258. built = 1;
  259. ul_count = 1;
  260. uldiff[0].u = uldiff[0].l = 0;
  261. memset(wct, 0, sizeof(wct));
  262. memset(combt, 0, sizeof(combt));
  263. memset(widtht, 0, sizeof(widtht));
  264. for (i = 0 ; i < 16 ; i++) {
  265. typecount[i] = 0;
  266. }
  267. for (c=0 ; c <= RANGE ; c++) {
  268. if (iswdigit(c)) {
  269. d = __CTYPE_digit;
  270. } else if (iswalpha(c)) {
  271. d = __CTYPE_alpha_nonupper_nonlower;
  272. if (iswlower(c)) {
  273. d = __CTYPE_alpha_lower;
  274. if (iswupper(c)) {
  275. d = __CTYPE_alpha_upper_lower;
  276. }
  277. } else if (iswupper(c)) {
  278. d = __CTYPE_alpha_upper;
  279. }
  280. } else if (iswpunct(c)) {
  281. d = __CTYPE_punct;
  282. } else if (iswgraph(c)) {
  283. d = __CTYPE_graph;
  284. } else if (iswprint(c)) {
  285. d = __CTYPE_print_space_nonblank;
  286. if (iswblank(c)) {
  287. d = __CTYPE_print_space_blank;
  288. }
  289. } else if (iswspace(c) && !iswcntrl(c)) {
  290. d = __CTYPE_space_nonblank_noncntrl;
  291. if (iswblank(c)) {
  292. d = __CTYPE_space_blank_noncntrl;
  293. }
  294. } else if (iswcntrl(c)) {
  295. d = __CTYPE_cntrl_nonspace;
  296. if (iswspace(c)) {
  297. d = __CTYPE_cntrl_space_nonblank;
  298. if (iswblank(c)) {
  299. d = __CTYPE_cntrl_space_blank;
  300. }
  301. }
  302. } else {
  303. d = __CTYPE_unclassified;
  304. }
  305. ++typecount[d];
  306. #if 0
  307. if (iswspace(c)) {
  308. if (iswblank(c)) {
  309. printf("%#8x : space blank\n", c);
  310. } else {
  311. printf("%#8x : space\n", c);
  312. }
  313. }
  314. #endif
  315. #if 0
  316. if (c < 256) {
  317. unsigned int glibc;
  318. glibc = 0;
  319. if (isalnum(c)) ++glibc; glibc <<= 1;
  320. if (isalpha(c)) ++glibc; glibc <<= 1;
  321. if (isblank(c)) ++glibc; glibc <<= 1;
  322. if (iscntrl(c)) ++glibc; glibc <<= 1;
  323. if (isdigit(c)) ++glibc; glibc <<= 1;
  324. if (isgraph(c)) ++glibc; glibc <<= 1;
  325. if (islower(c)) ++glibc; glibc <<= 1;
  326. if (isprint(c)) ++glibc; glibc <<= 1;
  327. if (ispunct(c)) ++glibc; glibc <<= 1;
  328. if (isspace(c)) ++glibc; glibc <<= 1;
  329. if (isupper(c)) ++glibc; glibc <<= 1;
  330. if (isxdigit(c)) ++glibc;
  331. printf("%#8x : ctype %#4x\n", c, glibc);
  332. }
  333. #endif
  334. #if 1
  335. /* Paranoid checking... */
  336. {
  337. unsigned int glibc;
  338. unsigned int mine;
  339. glibc = 0;
  340. if (iswalnum(c)) ++glibc; glibc <<= 1;
  341. if (iswalpha(c)) ++glibc; glibc <<= 1;
  342. if (iswblank(c)) ++glibc; glibc <<= 1;
  343. if (iswcntrl(c)) ++glibc; glibc <<= 1;
  344. if (iswdigit(c)) ++glibc; glibc <<= 1;
  345. if (iswgraph(c)) ++glibc; glibc <<= 1;
  346. if (iswlower(c)) ++glibc; glibc <<= 1;
  347. if (iswprint(c)) ++glibc; glibc <<= 1;
  348. if (iswpunct(c)) ++glibc; glibc <<= 1;
  349. if (iswspace(c)) ++glibc; glibc <<= 1;
  350. if (iswupper(c)) ++glibc; glibc <<= 1;
  351. if (iswxdigit(c)) ++glibc;
  352. mine = 0;
  353. if (mywalnum(c)) ++mine; mine <<= 1;
  354. if (mywalpha(c)) ++mine; mine <<= 1;
  355. if (mywblank(c)) ++mine; mine <<= 1;
  356. if (mywcntrl(c)) ++mine; mine <<= 1;
  357. if (mywdigit(c)) ++mine; mine <<= 1;
  358. if (mywgraph(c)) ++mine; mine <<= 1;
  359. if (mywlower(c)) ++mine; mine <<= 1;
  360. if (mywprint(c)) ++mine; mine <<= 1;
  361. if (mywpunct(c)) ++mine; mine <<= 1;
  362. if (mywspace(c)) ++mine; mine <<= 1;
  363. if (mywupper(c)) ++mine; mine <<= 1;
  364. if (mywxdigit(c)) ++mine;
  365. if (glibc != mine) {
  366. printf("%#8x : glibc %#4x != %#4x mine %u\n", c, glibc, mine, d);
  367. return EXIT_FAILURE;
  368. }
  369. #if 0
  370. if (iswctype(c,is_comb) || iswctype(c,is_comb3)) {
  371. /* if (!iswpunct(c)) { */
  372. printf("%#8x : %d %d %#4x\n",
  373. c, iswctype(c,is_comb),iswctype(c,is_comb3), glibc);
  374. /* } */
  375. }
  376. #endif
  377. #if 0
  378. if (iswctype(c,is_comb) || iswctype(c,is_comb3)) {
  379. if (!last_comb) {
  380. printf("%#8x - ", c);
  381. last_comb = c;
  382. } else if (last_comb + 1 < c) {
  383. printf("%#8x\n%#8x - ", last_comb, c);
  384. last_comb = c;
  385. } else {
  386. last_comb = c;
  387. }
  388. }
  389. #endif
  390. }
  391. #endif
  392. combt[c/4] |= ((((!!iswctype(c,is_comb)) << 1) | !!iswctype(c,is_comb3))
  393. << ((c & 3) << 1));
  394. /* comb3t[c/8] |= ((!!iswctype(c,is_comb3)) << (c & 7)); */
  395. /* widtht[c/4] |= (wcwidth(c) << ((c & 3) << 1)); */
  396. if (c & 1) { /* Use the high nibble for odd numbered wchars. */
  397. d <<= 4;
  398. }
  399. wct[c/2] |= d;
  400. l = (long)(int) towlower(c) - c;
  401. u = (long)(int) towupper(c) - c;
  402. ult[c] = 0;
  403. if (l || u) {
  404. if ((l != (short)l) || (u != (short)u)) {
  405. printf("range assumption error! %x %ld %ld\n", c, l, u);
  406. return EXIT_FAILURE;
  407. }
  408. for (i=0 ; i < ul_count ; i++) {
  409. if ((l == uldiff[i].l) && (u == uldiff[i].u)) {
  410. goto found;
  411. }
  412. }
  413. uldiff[ul_count].l = l;
  414. uldiff[ul_count].u = u;
  415. ++ul_count;
  416. if (ul_count > MAXTO) {
  417. printf("too many touppers/tolowers!\n");
  418. return EXIT_FAILURE;
  419. }
  420. found:
  421. ult[c] = i;
  422. }
  423. }
  424. for (i = 0 ; i < 16 ; i++) {
  425. printf("typecount[%2d] = %8ld %s\n", i, typecount[i], typename[i]);
  426. }
  427. printf("optimizing is* table..\n");
  428. n = -1;
  429. smallest = SIZE_MAX;
  430. cttable.ii = NULL;
  431. for (i=0 ; i < 14 ; i++) {
  432. t = newopt(wct, (RANGE/2)+1, i, &cttable);
  433. if (smallest >= t) {
  434. n = i;
  435. smallest = t;
  436. /* } else { */
  437. /* break; */
  438. }
  439. }
  440. printf("smallest = %zu\n", smallest);
  441. if (!(cttable.ii = malloc(smallest))) {
  442. printf("couldn't allocate space!\n");
  443. return EXIT_FAILURE;
  444. }
  445. smallest = SIZE_MAX;
  446. newopt(wct, (RANGE/2)+1, n, &cttable);
  447. ++cttable.ti_shift; /* correct for nibble mode */
  448. printf("optimizing u/l-to table..\n");
  449. smallest = SIZE_MAX;
  450. ultable.ii = NULL;
  451. for (i=0 ; i < 14 ; i++) {
  452. t = newopt(ult, RANGE+1, i, &ultable);
  453. if (smallest >= t) {
  454. n = i;
  455. smallest = t;
  456. /* } else { */
  457. /* break; */
  458. }
  459. }
  460. printf("%zu (smallest) + %zu (u/l diffs) = %zu\n",
  461. smallest, 4 * ul_count, smallest + 4 * ul_count);
  462. printf("smallest = %zu\n", smallest);
  463. if (!(ultable.ii = malloc(smallest))) {
  464. printf("couldn't allocate space!\n");
  465. return EXIT_FAILURE;
  466. }
  467. smallest = SIZE_MAX;
  468. newopt(ult, RANGE+1, n, &ultable);
  469. #if 0
  470. printf("optimizing comb table..\n");
  471. smallest = SIZE_MAX;
  472. combtable.ii = NULL;
  473. for (i=0 ; i < 14 ; i++) {
  474. t = newopt(combt, sizeof(combt), i, &combtable);
  475. if (smallest >= t) {
  476. n = i;
  477. smallest = t;
  478. /* } else { */
  479. /* break; */
  480. }
  481. }
  482. printf("smallest = %zu\n", smallest);
  483. if (!(combtable.ii = malloc(smallest))) {
  484. printf("couldn't allocate space!\n");
  485. return EXIT_FAILURE;
  486. }
  487. smallest = SIZE_MAX;
  488. newopt(combt, sizeof(combt), n, &combtable);
  489. combtable.ti_shift += 4; /* correct for 4 entries per */
  490. #endif
  491. #if 0
  492. printf("optimizing width table..\n");
  493. smallest = SIZE_MAX;
  494. widthtable.ii = NULL;
  495. for (i=0 ; i < 14 ; i++) {
  496. t = newopt(widtht, sizeof(widtht), i, &widthtable);
  497. if (smallest >= t) {
  498. n = i;
  499. smallest = t;
  500. /* } else { */
  501. /* break; */
  502. }
  503. }
  504. printf("smallest = %zu\n", smallest);
  505. if (!(widthtable.ii = malloc(smallest))) {
  506. printf("couldn't allocate space!\n");
  507. return EXIT_FAILURE;
  508. }
  509. smallest = SIZE_MAX;
  510. newopt(widtht, sizeof(widtht), n, &widthtable);
  511. widthtable.ti_shift += 4; /* correct for 4 entries per */
  512. #endif
  513. #if 0
  514. printf("optimizing comb3 table..\n");
  515. smallest = SIZE_MAX;
  516. comb3table.ii = NULL;
  517. for (i=0 ; i < 14 ; i++) {
  518. t = newopt(comb3t, sizeof(comb3t), i, &comb3table);
  519. if (smallest >= t) {
  520. n = i;
  521. smallest = t;
  522. /* } else { */
  523. /* break; */
  524. }
  525. }
  526. printf("smallest = %zu\n", smallest);
  527. if (!(comb3table.ii = malloc(smallest))) {
  528. printf("couldn't allocate space!\n");
  529. return EXIT_FAILURE;
  530. }
  531. smallest = SIZE_MAX;
  532. newopt(comb3t, sizeof(comb3t), n, &comb3table);
  533. comb3table.ti_shift += 8; /* correct for 4 entries per */
  534. #endif
  535. dump_table_data(&cttable);
  536. dump_table_data(&ultable);
  537. dump_table_data(&combtable);
  538. }
  539. printf("verifying for %s...\n", *argv);
  540. #if RANGE == 0xffffU
  541. for (c=0 ; c <= 0xffffUL ; c++)
  542. #else
  543. for (c=0 ; c <= 0x10ffffUL ; c++)
  544. #endif
  545. {
  546. unsigned int glibc;
  547. unsigned int mine;
  548. unsigned int upper, lower;
  549. #if 0
  550. #if RANGE < 0x10000UL
  551. if (c == 0x10000UL) {
  552. c = 0x30000UL; /* skip 1st and 2nd sup planes */
  553. }
  554. #elif RANGE < 0x20000UL
  555. if (c == 0x20000UL) {
  556. c = 0x30000UL; /* skip 2nd sup planes */
  557. }
  558. #endif
  559. #endif
  560. glibc = 0;
  561. if (iswalnum(c)) ++glibc; glibc <<= 1;
  562. if (iswalpha(c)) ++glibc; glibc <<= 1;
  563. if (iswblank(c)) ++glibc; glibc <<= 1;
  564. if (iswcntrl(c)) ++glibc; glibc <<= 1;
  565. if (iswdigit(c)) ++glibc; glibc <<= 1;
  566. if (iswgraph(c)) ++glibc; glibc <<= 1;
  567. if (iswlower(c)) ++glibc; glibc <<= 1;
  568. if (iswprint(c)) ++glibc; glibc <<= 1;
  569. if (iswpunct(c)) ++glibc; glibc <<= 1;
  570. if (iswspace(c)) ++glibc; glibc <<= 1;
  571. if (iswupper(c)) ++glibc; glibc <<= 1;
  572. if (iswxdigit(c)) ++glibc;
  573. {
  574. unsigned int u;
  575. int n, sc;
  576. int i0, i1;
  577. u = c;
  578. if (u <= RANGE) {
  579. sc = u & ((1 << cttable.ti_shift) - 1);
  580. u >>= cttable.ti_shift;
  581. n = u & ((1 << cttable.ii_shift) - 1);
  582. u >>= cttable.ii_shift;
  583. i0 = cttable.ii[u];
  584. i0 <<= cttable.ii_shift;
  585. i1 = cttable.ti[i0 + n];
  586. i1 <<= (cttable.ti_shift-1);
  587. d = cttable.ut[i1 + (sc >> 1)];
  588. if (sc & 1) {
  589. d >>= 4;
  590. }
  591. d &= 0x0f;
  592. } else if ((((unsigned int)(c - 0xe0020UL)) <= 0x5f) || (c == 0xe0001UL)){
  593. d = __CTYPE_punct;
  594. } else if (((unsigned int)(c - 0xf0000UL)) < 0x20000UL) {
  595. if ((c & 0xffffU) <= 0xfffdU) {
  596. d = __CTYPE_punct;
  597. } else {
  598. d = __CTYPE_unclassified;
  599. }
  600. } else {
  601. d = __CTYPE_unclassified;
  602. }
  603. mine = 0;
  604. if (mywalnum(c)) ++mine; mine <<= 1;
  605. if (mywalpha(c)) ++mine; mine <<= 1;
  606. if (mywblank(c)) ++mine; mine <<= 1;
  607. if (mywcntrl(c)) ++mine; mine <<= 1;
  608. if (mywdigit(c)) ++mine; mine <<= 1;
  609. if (mywgraph(c)) ++mine; mine <<= 1;
  610. if (mywlower(c)) ++mine; mine <<= 1;
  611. if (mywprint(c)) ++mine; mine <<= 1;
  612. if (mywpunct(c)) ++mine; mine <<= 1;
  613. if (mywspace(c)) ++mine; mine <<= 1;
  614. if (mywupper(c)) ++mine; mine <<= 1;
  615. if (mywxdigit(c)) ++mine;
  616. if (glibc != mine) {
  617. printf("%#8x : glibc %#4x != %#4x mine %d\n", c, glibc, mine, d);
  618. if (c < 0x30000UL) {
  619. printf("sc=%#x u=%#x n=%#x i0=%#x i1=%#x\n", sc, u, n, i0, i1);
  620. }
  621. }
  622. upper = lower = u = c;
  623. if (u <= RANGE) {
  624. sc = u & ((1 << ultable.ti_shift) - 1);
  625. u >>= ultable.ti_shift;
  626. n = u & ((1 << ultable.ii_shift) - 1);
  627. u >>= ultable.ii_shift;
  628. i0 = ultable.ii[u];
  629. i0 <<= ultable.ii_shift;
  630. i1 = ultable.ti[i0 + n];
  631. i1 <<= (ultable.ti_shift);
  632. i1 += sc;
  633. i0 = ultable.ut[i1];
  634. upper = c + uldiff[i0].u;
  635. lower = c + uldiff[i0].l;
  636. }
  637. if (towupper(c) != upper) {
  638. printf("%#8x : towupper glibc %#4x != %#4x mine\n",
  639. c, towupper(c), upper);
  640. }
  641. if (towlower(c) != lower) {
  642. printf("%#8x : towlower glibc %#4x != %#4x mine i0 = %d\n",
  643. c, towlower(c), lower, i0);
  644. }
  645. if (totitle && ((tt = towctrans(c, totitle)) != upper)) {
  646. printf("%#8x : totitle glibc %#4lx != %#4x mine i0 = %d\n",
  647. c, tt, upper, i0);
  648. }
  649. }
  650. if ((c & 0xfff) == 0xfff) printf(".");
  651. }
  652. printf("done\n");
  653. }
  654. if (1) {
  655. FILE *fp;
  656. if (!(fp = fopen("wctables.h", "w"))) {
  657. printf("couldn't open wctables.h!\n");
  658. return EXIT_FAILURE;
  659. }
  660. fprintf(fp, "#define __LOCALE_DATA_WC_TABLE_DOMAIN_MAX %#8lx\n\n",
  661. (unsigned long) RANGE);
  662. output_table(fp, "ctype", &cttable);
  663. output_table(fp, "uplow", &ultable);
  664. #warning fix the upper bound on the upper/lower tables... save 200 bytes or so
  665. fprintf(fp, "#define __LOCALE_DATA_WCuplow_diffs %7u\n", ul_count);
  666. fprintf(fp, "\n#ifdef WANT_WCuplow_diff_data\n\n");
  667. fprintf(fp, "\nstatic const short __LOCALE_DATA_WCuplow_diff_data[%zu] = {",
  668. 2 * (size_t) ul_count);
  669. for (i=0 ; i < ul_count ; i++) {
  670. if (i % 4 == 0) {
  671. fprintf(fp, "\n");
  672. }
  673. fprintf(fp, " %6d, %6d,", uldiff[i].u, uldiff[i].l);
  674. }
  675. fprintf(fp, "\n};\n\n");
  676. fprintf(fp, "#endif /* WANT_WCuplow_diff_data */\n\n");
  677. /* output_table(fp, "comb", &combtable); */
  678. /* output_table(fp, "width", &widthtable); */
  679. fclose(fp);
  680. }
  681. return EXIT_SUCCESS;
  682. }
  683. size_t newopt(unsigned char *ut, size_t usize, int shift, table_data *tbl)
  684. {
  685. static int recurse = 0;
  686. unsigned char *ti[RANGE+1]; /* table index */
  687. size_t numblocks;
  688. size_t blocksize;
  689. size_t uniq;
  690. size_t i, j;
  691. size_t smallest, t;
  692. unsigned char *ii_save;
  693. int uniqblock[256];
  694. unsigned char uit[RANGE+1];
  695. int shift2;
  696. ii_save = NULL;
  697. blocksize = 1 << shift;
  698. numblocks = usize >> shift;
  699. /* init table index */
  700. for (i=j=0 ; i < numblocks ; i++) {
  701. ti[i] = ut + j;
  702. j += blocksize;
  703. }
  704. /* sort */
  705. nu_val = blocksize;
  706. qsort(ti, numblocks, sizeof(unsigned char *), nu_memcmp);
  707. uniq = 1;
  708. uit[(ti[0]-ut)/blocksize] = 0;
  709. for (i=1 ; i < numblocks ; i++) {
  710. if (memcmp(ti[i-1], ti[i], blocksize) < 0) {
  711. if (++uniq > 255) {
  712. break;
  713. }
  714. uniqblock[uniq - 1] = i;
  715. }
  716. #if 1
  717. else if (memcmp(ti[i-1], ti[i], blocksize) > 0) {
  718. printf("bad sort %i!\n", i);
  719. abort();
  720. }
  721. #endif
  722. uit[(ti[i]-ut)/blocksize] = uniq - 1;
  723. }
  724. smallest = SIZE_MAX;
  725. shift2 = -1;
  726. if (uniq <= 255) {
  727. smallest = numblocks + uniq * blocksize;
  728. if (!recurse) {
  729. ++recurse;
  730. for (j=1 ; j < 14 ; j++) {
  731. if ((numblocks >> j) < 2) break;
  732. if (tbl) {
  733. ii_save = tbl->ii;
  734. tbl->ii = NULL;
  735. }
  736. if ((t = newopt(uit, numblocks, j, tbl)) < SIZE_MAX) {
  737. t += uniq * blocksize;
  738. }
  739. if (tbl) {
  740. tbl->ii = ii_save;
  741. }
  742. if (smallest >= t) {
  743. shift2 = j;
  744. smallest = t;
  745. if (!tbl->ii) {
  746. printf("ishift %zu tshift %zu size %zu\n",
  747. shift2, shift, t);
  748. }
  749. /* } else { */
  750. /* break; */
  751. }
  752. }
  753. --recurse;
  754. }
  755. } else {
  756. return SIZE_MAX;
  757. }
  758. if (tbl->ii) {
  759. if (recurse) {
  760. tbl->ii_shift = shift;
  761. tbl->ii_len = numblocks;
  762. memcpy(tbl->ii, uit, numblocks);
  763. tbl->ti = tbl->ii + tbl->ii_len;
  764. tbl->ti_len = uniq * blocksize;
  765. for (i=0 ; i < uniq ; i++) {
  766. memcpy(tbl->ti + i * blocksize, ti[uniqblock[i]], blocksize);
  767. }
  768. } else {
  769. ++recurse;
  770. printf("setting ishift %zu tshift %zu\n",
  771. shift2, shift);
  772. newopt(uit, numblocks, shift2, tbl);
  773. --recurse;
  774. tbl->ti_shift = shift;
  775. tbl->ut_len = uniq * blocksize;
  776. tbl->ut = tbl->ti + tbl->ti_len;
  777. for (i=0 ; i < uniq ; i++) {
  778. memcpy(tbl->ut + i * blocksize, ti[uniqblock[i]], blocksize);
  779. }
  780. }
  781. }
  782. return smallest;
  783. }