gen_wctype.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870
  1. #define _GNU_SOURCE
  2. #include <stdio.h>
  3. #include <stdlib.h>
  4. #include <string.h>
  5. #include <locale.h>
  6. #include <wctype.h>
  7. #include <limits.h>
  8. #include <stdint.h>
  9. #include <wchar.h>
  10. #include <ctype.h>
  11. #ifndef _CTYPE_H
  12. #define _CTYPE_H
  13. #endif
  14. #ifndef _WCTYPE_H
  15. #define _WCTYPE_H
  16. #endif
  17. #include "../../libc/sysdeps/linux/common/bits/uClibc_ctype.h"
  18. /* 0x9 : space blank */
  19. /* 0xa : space */
  20. /* 0xb : space */
  21. /* 0xc : space */
  22. /* 0xd : space */
  23. /* 0x20 : space blank */
  24. /* 0x1680 : space blank */
  25. /* 0x2000 : space blank */
  26. /* 0x2001 : space blank */
  27. /* 0x2002 : space blank */
  28. /* 0x2003 : space blank */
  29. /* 0x2004 : space blank */
  30. /* 0x2005 : space blank */
  31. /* 0x2006 : space blank */
  32. /* 0x2008 : space blank */
  33. /* 0x2009 : space blank */
  34. /* 0x200a : space blank */
  35. /* 0x200b : space blank */
  36. /* 0x2028 : space */
  37. /* 0x2029 : space */
  38. /* 0x3000 : space blank */
  39. /* typecount[ 0] = 88670 C_alpha_nonupper_nonlower */
  40. /* typecount[ 1] = 742 C_alpha_lower */
  41. /* typecount[ 2] = 4 C_alpha_upper_lower */
  42. /* typecount[ 3] = 731 C_alpha_upper */
  43. /* typecount[ 4] = 10 C_digit */
  44. /* typecount[ 5] = 10270 C_punct */
  45. /* typecount[ 6] = 0 C_graph */
  46. /* typecount[ 7] = 0 C_print_space_nonblank */
  47. /* typecount[ 8] = 14 C_print_space_blank */
  48. /* typecount[ 9] = 0 C_space_nonblank_noncntrl */
  49. /* typecount[10] = 0 C_space_blank_noncntrl */
  50. /* typecount[11] = 6 C_cntrl_space_nonblank */
  51. /* typecount[12] = 1 C_cntrl_space_blank */
  52. /* typecount[13] = 60 C_cntrl_nonspace */
  53. /* typecount[14] = 96100 C_unclassified */
  54. /* typecount[15] = 0 empty_slot */
  55. /* Set to #if 0 to restrict wchars to 16 bits. */
  56. #if 1
  57. #define RANGE 0x2ffffUL
  58. #elif 0
  59. #define RANGE 0x1ffffUL
  60. #else
  61. #define RANGE 0xffffUL /* Restrict for 16-bit wchar_t... */
  62. #endif
  63. #if 0
  64. /* Classification codes. */
  65. static const char *typename[] = {
  66. "C_unclassified",
  67. "C_alpha_nonupper_nonlower",
  68. "C_alpha_lower",
  69. "C_alpha_upper_lower",
  70. "C_alpha_upper",
  71. "C_digit",
  72. "C_punct",
  73. "C_graph",
  74. "C_print_space_nonblank",
  75. "C_print_space_blank",
  76. "C_space_nonblank_noncntrl",
  77. "C_space_blank_noncntrl",
  78. "C_cntrl_space_nonblank",
  79. "C_cntrl_space_blank",
  80. "C_cntrl_nonspace",
  81. "empty_slot"
  82. };
  83. #endif
  84. #if 0
  85. /* Taking advantage of the C99 mutual-exclusion guarantees for the various
  86. * (w)ctype classes, including the descriptions of printing and control
  87. * (w)chars, we can place each in one of the following mutually-exlusive
  88. * subsets. Since there are less than 16, we can store the data for
  89. * each (w)chars in a nibble. In contrast, glibc uses an unsigned int
  90. * per (w)char, with one bit flag for each is* type. While this allows
  91. * a simple '&' operation to determine the type vs. a range test and a
  92. * little special handling for the "blank" and "xdigit" types in my
  93. * approach, it also uses 8 times the space for the tables on the typical
  94. * 32-bit archs we supported.*/
  95. enum {
  96. __CTYPE_unclassified = 0,
  97. __CTYPE_alpha_nonupper_nonlower,
  98. __CTYPE_alpha_lower,
  99. __CTYPE_alpha_upper_lower,
  100. __CTYPE_alpha_upper,
  101. __CTYPE_digit,
  102. __CTYPE_punct,
  103. __CTYPE_graph,
  104. __CTYPE_print_space_nonblank,
  105. __CTYPE_print_space_blank,
  106. __CTYPE_space_nonblank_noncntrl,
  107. __CTYPE_space_blank_noncntrl,
  108. __CTYPE_cntrl_space_nonblank,
  109. __CTYPE_cntrl_space_blank,
  110. __CTYPE_cntrl_nonspace,
  111. };
  112. #endif
  113. #define __CTYPE_isxdigit(D,X) \
  114. (__CTYPE_isdigit(D) || (((unsigned int)(((X)|0x20) - 'a')) <= 5))
  115. #define mywalnum(x) __CTYPE_isalnum(d)
  116. #define mywalpha(x) __CTYPE_isalpha(d)
  117. #define mywblank(x) __CTYPE_isblank(d)
  118. #define mywcntrl(x) __CTYPE_iscntrl(d)
  119. #define mywdigit(x) __CTYPE_isdigit(d)
  120. #define mywgraph(x) __CTYPE_isgraph(d)
  121. #define mywlower(x) __CTYPE_islower(d)
  122. #define mywprint(x) __CTYPE_isprint(d)
  123. #define mywpunct(x) __CTYPE_ispunct(d)
  124. #define mywspace(x) __CTYPE_isspace(d)
  125. #define mywupper(x) __CTYPE_isupper(d)
  126. #define mywxdigit(x) __CTYPE_isxdigit(d,x)
  127. typedef struct {
  128. short l;
  129. short u;
  130. } uldiff_entry;
  131. typedef struct {
  132. uint16_t ii_len;
  133. uint16_t ti_len;
  134. uint16_t ut_len;
  135. unsigned char ii_shift;
  136. unsigned char ti_shift;
  137. unsigned char *ii;
  138. unsigned char *ti;
  139. unsigned char *ut;
  140. } table_data;
  141. void output_table(FILE *fp, const char *name, table_data *tbl)
  142. {
  143. size_t i;
  144. fprintf(fp, "#define __LOCALE_DATA_WC%s_II_LEN %7u\n", name, tbl->ii_len);
  145. fprintf(fp, "#define __LOCALE_DATA_WC%s_TI_LEN %7u\n", name, tbl->ti_len);
  146. fprintf(fp, "#define __LOCALE_DATA_WC%s_UT_LEN %7u\n", name, tbl->ut_len);
  147. fprintf(fp, "#define __LOCALE_DATA_WC%s_II_SHIFT %7u\n", name, tbl->ii_shift);
  148. fprintf(fp, "#define __LOCALE_DATA_WC%s_TI_SHIFT %7u\n", name, tbl->ti_shift);
  149. fprintf(fp, "\n#ifdef WANT_WC%s_data\n", name);
  150. i = tbl->ii_len + tbl->ti_len + tbl->ut_len;
  151. fprintf(fp, "\nstatic const unsigned char __LOCALE_DATA_WC%s_data[%zu] = {", name, i);
  152. for (i=0 ; i < tbl->ii_len ; i++) {
  153. if (i % 12 == 0) {
  154. fprintf(fp, "\n");
  155. }
  156. fprintf(fp, " %#04x,", tbl->ii[i]);
  157. }
  158. for (i=0 ; i < tbl->ti_len ; i++) {
  159. if (i % 12 == 0) {
  160. fprintf(fp, "\n");
  161. }
  162. fprintf(fp, " %#04x,", tbl->ti[i]);
  163. }
  164. for (i=0 ; i < tbl->ut_len ; i++) {
  165. if (i % 12 == 0) {
  166. fprintf(fp, "\n");
  167. }
  168. fprintf(fp, " %#04x,", tbl->ut[i]);
  169. }
  170. fprintf(fp, "\n};\n\n");
  171. fprintf(fp, "#endif /* WANT_WC%s_data */\n\n", name);
  172. }
  173. static void dump_table_data(table_data *tbl)
  174. {
  175. printf("ii_shift = %d ti_shift = %d\n"
  176. "ii_len = %d ti_len = %d ut_len = %d\n"
  177. "total = %d\n",
  178. tbl->ii_shift, tbl->ti_shift,
  179. tbl->ii_len, tbl->ti_len, tbl->ut_len,
  180. (int) tbl->ii_len + (int) tbl->ti_len + (int) tbl->ut_len);
  181. }
  182. /* For sorting the blocks of unsigned chars. */
  183. static size_t nu_val;
  184. int nu_memcmp(const void *a, const void *b)
  185. {
  186. return memcmp(*(unsigned char**)a, *(unsigned char**)b, nu_val);
  187. }
  188. static size_t newopt(unsigned char *ut, size_t usize, int shift, table_data *tbl);
  189. #define MAXTO 255 /* Restrict to minimal unsigned char max. */
  190. int main(int argc, char **argv)
  191. {
  192. long int u, l, tt;
  193. size_t smallest, t;
  194. unsigned int c;
  195. unsigned int d;
  196. int i, n;
  197. int ul_count = 0;
  198. uldiff_entry uldiff[MAXTO];
  199. table_data cttable;
  200. table_data ultable;
  201. table_data combtable;
  202. table_data widthtable;
  203. long int last_comb = 0;
  204. unsigned char wct[(RANGE/2)+1]; /* wctype table (nibble per wchar) */
  205. unsigned char ult[RANGE+1]; /* upper/lower table */
  206. unsigned char combt[(RANGE/4)+1]; /* combining */
  207. unsigned char widtht[(RANGE/4)+1]; /* width */
  208. wctrans_t totitle;
  209. wctype_t is_comb, is_comb3;
  210. long int typecount[16];
  211. const char *typename[16];
  212. static const char empty_slot[] = "empty_slot";
  213. int built = 0;
  214. #define INIT_TYPENAME(X) typename[__CTYPE_##X] = "C_" #X
  215. for (i=0 ; i < 16 ; i++) {
  216. typename[i] = empty_slot;
  217. }
  218. INIT_TYPENAME(unclassified);
  219. INIT_TYPENAME(alpha_nonupper_nonlower);
  220. INIT_TYPENAME(alpha_lower);
  221. INIT_TYPENAME(alpha_upper_lower);
  222. INIT_TYPENAME(alpha_upper);
  223. INIT_TYPENAME(digit);
  224. INIT_TYPENAME(punct);
  225. INIT_TYPENAME(graph);
  226. INIT_TYPENAME(print_space_nonblank);
  227. INIT_TYPENAME(print_space_blank);
  228. INIT_TYPENAME(space_nonblank_noncntrl);
  229. INIT_TYPENAME(space_blank_noncntrl);
  230. INIT_TYPENAME(cntrl_space_nonblank);
  231. INIT_TYPENAME(cntrl_space_blank);
  232. INIT_TYPENAME(cntrl_nonspace);
  233. setvbuf(stdout, NULL, _IONBF, 0);
  234. while (--argc) {
  235. if (!setlocale(LC_CTYPE, *++argv)) {
  236. printf("setlocale(LC_CTYPE,%s) failed!\n", *argv);
  237. continue;
  238. }
  239. if (!(totitle = wctrans("totitle"))) {
  240. printf("no totitle transformation.\n");
  241. }
  242. if (!(is_comb = wctype("combining"))) {
  243. printf("no combining wctype.\n");
  244. }
  245. if (!(is_comb3 = wctype("combining_level3"))) {
  246. printf("no combining_level3 wctype.\n");
  247. }
  248. if (!built) {
  249. built = 1;
  250. ul_count = 1;
  251. uldiff[0].u = uldiff[0].l = 0;
  252. memset(wct, 0, sizeof(wct));
  253. memset(combt, 0, sizeof(combt));
  254. memset(widtht, 0, sizeof(widtht));
  255. for (i = 0 ; i < 16 ; i++) {
  256. typecount[i] = 0;
  257. }
  258. for (c=0 ; c <= RANGE ; c++) {
  259. if (iswdigit(c)) {
  260. d = __CTYPE_digit;
  261. } else if (iswalpha(c)) {
  262. d = __CTYPE_alpha_nonupper_nonlower;
  263. if (iswlower(c)) {
  264. d = __CTYPE_alpha_lower;
  265. if (iswupper(c)) {
  266. d = __CTYPE_alpha_upper_lower;
  267. }
  268. } else if (iswupper(c)) {
  269. d = __CTYPE_alpha_upper;
  270. }
  271. } else if (iswpunct(c)) {
  272. d = __CTYPE_punct;
  273. } else if (iswgraph(c)) {
  274. d = __CTYPE_graph;
  275. } else if (iswprint(c)) {
  276. d = __CTYPE_print_space_nonblank;
  277. if (iswblank(c)) {
  278. d = __CTYPE_print_space_blank;
  279. }
  280. } else if (iswspace(c) && !iswcntrl(c)) {
  281. d = __CTYPE_space_nonblank_noncntrl;
  282. if (iswblank(c)) {
  283. d = __CTYPE_space_blank_noncntrl;
  284. }
  285. } else if (iswcntrl(c)) {
  286. d = __CTYPE_cntrl_nonspace;
  287. if (iswspace(c)) {
  288. d = __CTYPE_cntrl_space_nonblank;
  289. if (iswblank(c)) {
  290. d = __CTYPE_cntrl_space_blank;
  291. }
  292. }
  293. } else {
  294. d = __CTYPE_unclassified;
  295. }
  296. ++typecount[d];
  297. #if 0
  298. if (iswspace(c)) {
  299. if (iswblank(c)) {
  300. printf("%#8x : space blank\n", c);
  301. } else {
  302. printf("%#8x : space\n", c);
  303. }
  304. }
  305. #endif
  306. #if 0
  307. if (c < 256) {
  308. unsigned int glibc;
  309. glibc = 0;
  310. if (isalnum(c)) ++glibc; glibc <<= 1;
  311. if (isalpha(c)) ++glibc; glibc <<= 1;
  312. if (isblank(c)) ++glibc; glibc <<= 1;
  313. if (iscntrl(c)) ++glibc; glibc <<= 1;
  314. if (isdigit(c)) ++glibc; glibc <<= 1;
  315. if (isgraph(c)) ++glibc; glibc <<= 1;
  316. if (islower(c)) ++glibc; glibc <<= 1;
  317. if (isprint(c)) ++glibc; glibc <<= 1;
  318. if (ispunct(c)) ++glibc; glibc <<= 1;
  319. if (isspace(c)) ++glibc; glibc <<= 1;
  320. if (isupper(c)) ++glibc; glibc <<= 1;
  321. if (isxdigit(c)) ++glibc;
  322. printf("%#8x : ctype %#4x\n", c, glibc);
  323. }
  324. #endif
  325. #if 1
  326. /* Paranoid checking... */
  327. {
  328. unsigned int glibc;
  329. unsigned int mine;
  330. glibc = 0;
  331. if (iswalnum(c)) ++glibc; glibc <<= 1;
  332. if (iswalpha(c)) ++glibc; glibc <<= 1;
  333. if (iswblank(c)) ++glibc; glibc <<= 1;
  334. if (iswcntrl(c)) ++glibc; glibc <<= 1;
  335. if (iswdigit(c)) ++glibc; glibc <<= 1;
  336. if (iswgraph(c)) ++glibc; glibc <<= 1;
  337. if (iswlower(c)) ++glibc; glibc <<= 1;
  338. if (iswprint(c)) ++glibc; glibc <<= 1;
  339. if (iswpunct(c)) ++glibc; glibc <<= 1;
  340. if (iswspace(c)) ++glibc; glibc <<= 1;
  341. if (iswupper(c)) ++glibc; glibc <<= 1;
  342. if (iswxdigit(c)) ++glibc;
  343. mine = 0;
  344. if (mywalnum(c)) ++mine; mine <<= 1;
  345. if (mywalpha(c)) ++mine; mine <<= 1;
  346. if (mywblank(c)) ++mine; mine <<= 1;
  347. if (mywcntrl(c)) ++mine; mine <<= 1;
  348. if (mywdigit(c)) ++mine; mine <<= 1;
  349. if (mywgraph(c)) ++mine; mine <<= 1;
  350. if (mywlower(c)) ++mine; mine <<= 1;
  351. if (mywprint(c)) ++mine; mine <<= 1;
  352. if (mywpunct(c)) ++mine; mine <<= 1;
  353. if (mywspace(c)) ++mine; mine <<= 1;
  354. if (mywupper(c)) ++mine; mine <<= 1;
  355. if (mywxdigit(c)) ++mine;
  356. if (glibc != mine) {
  357. printf("%#8x : glibc %#4x != %#4x mine %u\n", c, glibc, mine, d);
  358. return EXIT_FAILURE;
  359. }
  360. #if 0
  361. if (iswctype(c,is_comb) || iswctype(c,is_comb3)) {
  362. /* if (!iswpunct(c)) { */
  363. printf("%#8x : %d %d %#4x\n",
  364. c, iswctype(c,is_comb),iswctype(c,is_comb3), glibc);
  365. /* } */
  366. }
  367. #endif
  368. #if 0
  369. if (iswctype(c,is_comb) || iswctype(c,is_comb3)) {
  370. if (!last_comb) {
  371. printf("%#8x - ", c);
  372. last_comb = c;
  373. } else if (last_comb + 1 < c) {
  374. printf("%#8x\n%#8x - ", last_comb, c);
  375. last_comb = c;
  376. } else {
  377. last_comb = c;
  378. }
  379. }
  380. #endif
  381. }
  382. #endif
  383. combt[c/4] |= ((((!!iswctype(c,is_comb)) << 1) | !!iswctype(c,is_comb3))
  384. << ((c & 3) << 1));
  385. /* comb3t[c/8] |= ((!!iswctype(c,is_comb3)) << (c & 7)); */
  386. /* widtht[c/4] |= (wcwidth(c) << ((c & 3) << 1)); */
  387. if (c & 1) { /* Use the high nibble for odd numbered wchars. */
  388. d <<= 4;
  389. }
  390. wct[c/2] |= d;
  391. l = (long)(int) towlower(c) - c;
  392. u = (long)(int) towupper(c) - c;
  393. ult[c] = 0;
  394. if (l || u) {
  395. if ((l != (short)l) || (u != (short)u)) {
  396. printf("range assumption error! %x %ld %ld\n", c, l, u);
  397. return EXIT_FAILURE;
  398. }
  399. for (i=0 ; i < ul_count ; i++) {
  400. if ((l == uldiff[i].l) && (u == uldiff[i].u)) {
  401. goto found;
  402. }
  403. }
  404. uldiff[ul_count].l = l;
  405. uldiff[ul_count].u = u;
  406. ++ul_count;
  407. if (ul_count > MAXTO) {
  408. printf("too many touppers/tolowers!\n");
  409. return EXIT_FAILURE;
  410. }
  411. found:
  412. ult[c] = i;
  413. }
  414. }
  415. for (i = 0 ; i < 16 ; i++) {
  416. printf("typecount[%2d] = %8ld %s\n", i, typecount[i], typename[i]);
  417. }
  418. printf("optimizing is* table..\n");
  419. n = -1;
  420. smallest = SIZE_MAX;
  421. cttable.ii = NULL;
  422. for (i=0 ; i < 14 ; i++) {
  423. t = newopt(wct, (RANGE/2)+1, i, &cttable);
  424. if (smallest >= t) {
  425. n = i;
  426. smallest = t;
  427. /* } else { */
  428. /* break; */
  429. }
  430. }
  431. printf("smallest = %zu\n", smallest);
  432. if (!(cttable.ii = malloc(smallest))) {
  433. printf("couldn't allocate space!\n");
  434. return EXIT_FAILURE;
  435. }
  436. smallest = SIZE_MAX;
  437. newopt(wct, (RANGE/2)+1, n, &cttable);
  438. ++cttable.ti_shift; /* correct for nibble mode */
  439. printf("optimizing u/l-to table..\n");
  440. smallest = SIZE_MAX;
  441. ultable.ii = NULL;
  442. for (i=0 ; i < 14 ; i++) {
  443. t = newopt(ult, RANGE+1, i, &ultable);
  444. if (smallest >= t) {
  445. n = i;
  446. smallest = t;
  447. /* } else { */
  448. /* break; */
  449. }
  450. }
  451. printf("%zu (smallest) + %zu (u/l diffs) = %zu\n",
  452. smallest, 4 * ul_count, smallest + 4 * ul_count);
  453. printf("smallest = %zu\n", smallest);
  454. if (!(ultable.ii = malloc(smallest))) {
  455. printf("couldn't allocate space!\n");
  456. return EXIT_FAILURE;
  457. }
  458. smallest = SIZE_MAX;
  459. newopt(ult, RANGE+1, n, &ultable);
  460. #if 0
  461. printf("optimizing comb table..\n");
  462. smallest = SIZE_MAX;
  463. combtable.ii = NULL;
  464. for (i=0 ; i < 14 ; i++) {
  465. t = newopt(combt, sizeof(combt), i, &combtable);
  466. if (smallest >= t) {
  467. n = i;
  468. smallest = t;
  469. /* } else { */
  470. /* break; */
  471. }
  472. }
  473. printf("smallest = %zu\n", smallest);
  474. if (!(combtable.ii = malloc(smallest))) {
  475. printf("couldn't allocate space!\n");
  476. return EXIT_FAILURE;
  477. }
  478. smallest = SIZE_MAX;
  479. newopt(combt, sizeof(combt), n, &combtable);
  480. combtable.ti_shift += 4; /* correct for 4 entries per */
  481. #endif
  482. #if 0
  483. printf("optimizing width table..\n");
  484. smallest = SIZE_MAX;
  485. widthtable.ii = NULL;
  486. for (i=0 ; i < 14 ; i++) {
  487. t = newopt(widtht, sizeof(widtht), i, &widthtable);
  488. if (smallest >= t) {
  489. n = i;
  490. smallest = t;
  491. /* } else { */
  492. /* break; */
  493. }
  494. }
  495. printf("smallest = %zu\n", smallest);
  496. if (!(widthtable.ii = malloc(smallest))) {
  497. printf("couldn't allocate space!\n");
  498. return EXIT_FAILURE;
  499. }
  500. smallest = SIZE_MAX;
  501. newopt(widtht, sizeof(widtht), n, &widthtable);
  502. widthtable.ti_shift += 4; /* correct for 4 entries per */
  503. #endif
  504. #if 0
  505. printf("optimizing comb3 table..\n");
  506. smallest = SIZE_MAX;
  507. comb3table.ii = NULL;
  508. for (i=0 ; i < 14 ; i++) {
  509. t = newopt(comb3t, sizeof(comb3t), i, &comb3table);
  510. if (smallest >= t) {
  511. n = i;
  512. smallest = t;
  513. /* } else { */
  514. /* break; */
  515. }
  516. }
  517. printf("smallest = %zu\n", smallest);
  518. if (!(comb3table.ii = malloc(smallest))) {
  519. printf("couldn't allocate space!\n");
  520. return EXIT_FAILURE;
  521. }
  522. smallest = SIZE_MAX;
  523. newopt(comb3t, sizeof(comb3t), n, &comb3table);
  524. comb3table.ti_shift += 8; /* correct for 4 entries per */
  525. #endif
  526. dump_table_data(&cttable);
  527. dump_table_data(&ultable);
  528. dump_table_data(&combtable);
  529. }
  530. printf("verifying for %s...\n", *argv);
  531. #if RANGE == 0xffffU
  532. for (c=0 ; c <= 0xffffUL ; c++)
  533. #else
  534. for (c=0 ; c <= 0x10ffffUL ; c++)
  535. #endif
  536. {
  537. unsigned int glibc;
  538. unsigned int mine;
  539. unsigned int upper, lower;
  540. #if 0
  541. #if RANGE < 0x10000UL
  542. if (c == 0x10000UL) {
  543. c = 0x30000UL; /* skip 1st and 2nd sup planes */
  544. }
  545. #elif RANGE < 0x20000UL
  546. if (c == 0x20000UL) {
  547. c = 0x30000UL; /* skip 2nd sup planes */
  548. }
  549. #endif
  550. #endif
  551. glibc = 0;
  552. if (iswalnum(c)) ++glibc; glibc <<= 1;
  553. if (iswalpha(c)) ++glibc; glibc <<= 1;
  554. if (iswblank(c)) ++glibc; glibc <<= 1;
  555. if (iswcntrl(c)) ++glibc; glibc <<= 1;
  556. if (iswdigit(c)) ++glibc; glibc <<= 1;
  557. if (iswgraph(c)) ++glibc; glibc <<= 1;
  558. if (iswlower(c)) ++glibc; glibc <<= 1;
  559. if (iswprint(c)) ++glibc; glibc <<= 1;
  560. if (iswpunct(c)) ++glibc; glibc <<= 1;
  561. if (iswspace(c)) ++glibc; glibc <<= 1;
  562. if (iswupper(c)) ++glibc; glibc <<= 1;
  563. if (iswxdigit(c)) ++glibc;
  564. {
  565. unsigned int u;
  566. int n, sc;
  567. int i0, i1;
  568. u = c;
  569. if (u <= RANGE) {
  570. sc = u & ((1 << cttable.ti_shift) - 1);
  571. u >>= cttable.ti_shift;
  572. n = u & ((1 << cttable.ii_shift) - 1);
  573. u >>= cttable.ii_shift;
  574. i0 = cttable.ii[u];
  575. i0 <<= cttable.ii_shift;
  576. i1 = cttable.ti[i0 + n];
  577. i1 <<= (cttable.ti_shift-1);
  578. d = cttable.ut[i1 + (sc >> 1)];
  579. if (sc & 1) {
  580. d >>= 4;
  581. }
  582. d &= 0x0f;
  583. } else if ((((unsigned int)(c - 0xe0020UL)) <= 0x5f) || (c == 0xe0001UL)){
  584. d = __CTYPE_punct;
  585. } else if (((unsigned int)(c - 0xf0000UL)) < 0x20000UL) {
  586. if ((c & 0xffffU) <= 0xfffdU) {
  587. d = __CTYPE_punct;
  588. } else {
  589. d = __CTYPE_unclassified;
  590. }
  591. } else {
  592. d = __CTYPE_unclassified;
  593. }
  594. mine = 0;
  595. if (mywalnum(c)) ++mine; mine <<= 1;
  596. if (mywalpha(c)) ++mine; mine <<= 1;
  597. if (mywblank(c)) ++mine; mine <<= 1;
  598. if (mywcntrl(c)) ++mine; mine <<= 1;
  599. if (mywdigit(c)) ++mine; mine <<= 1;
  600. if (mywgraph(c)) ++mine; mine <<= 1;
  601. if (mywlower(c)) ++mine; mine <<= 1;
  602. if (mywprint(c)) ++mine; mine <<= 1;
  603. if (mywpunct(c)) ++mine; mine <<= 1;
  604. if (mywspace(c)) ++mine; mine <<= 1;
  605. if (mywupper(c)) ++mine; mine <<= 1;
  606. if (mywxdigit(c)) ++mine;
  607. if (glibc != mine) {
  608. printf("%#8x : glibc %#4x != %#4x mine %d\n", c, glibc, mine, d);
  609. if (c < 0x30000UL) {
  610. printf("sc=%#x u=%#x n=%#x i0=%#x i1=%#x\n", sc, u, n, i0, i1);
  611. }
  612. }
  613. upper = lower = u = c;
  614. if (u <= RANGE) {
  615. sc = u & ((1 << ultable.ti_shift) - 1);
  616. u >>= ultable.ti_shift;
  617. n = u & ((1 << ultable.ii_shift) - 1);
  618. u >>= ultable.ii_shift;
  619. i0 = ultable.ii[u];
  620. i0 <<= ultable.ii_shift;
  621. i1 = ultable.ti[i0 + n];
  622. i1 <<= (ultable.ti_shift);
  623. i1 += sc;
  624. i0 = ultable.ut[i1];
  625. upper = c + uldiff[i0].u;
  626. lower = c + uldiff[i0].l;
  627. }
  628. if (towupper(c) != upper) {
  629. printf("%#8x : towupper glibc %#4x != %#4x mine\n",
  630. c, towupper(c), upper);
  631. }
  632. if (towlower(c) != lower) {
  633. printf("%#8x : towlower glibc %#4x != %#4x mine i0 = %d\n",
  634. c, towlower(c), lower, i0);
  635. }
  636. if (totitle && ((tt = towctrans(c, totitle)) != upper)) {
  637. printf("%#8x : totitle glibc %#4lx != %#4x mine i0 = %d\n",
  638. c, tt, upper, i0);
  639. }
  640. }
  641. if ((c & 0xfff) == 0xfff) printf(".");
  642. }
  643. printf("done\n");
  644. }
  645. if (1) {
  646. FILE *fp;
  647. if (!(fp = fopen("wctables.h", "w"))) {
  648. printf("couldn't open wctables.h!\n");
  649. return EXIT_FAILURE;
  650. }
  651. fprintf(fp, "#define __LOCALE_DATA_WC_TABLE_DOMAIN_MAX %#8lx\n\n",
  652. (unsigned long) RANGE);
  653. output_table(fp, "ctype", &cttable);
  654. output_table(fp, "uplow", &ultable);
  655. #warning fix the upper bound on the upper/lower tables... save 200 bytes or so
  656. fprintf(fp, "#define __LOCALE_DATA_WCuplow_diffs %7u\n", ul_count);
  657. fprintf(fp, "\n#ifdef WANT_WCuplow_diff_data\n\n");
  658. fprintf(fp, "\nstatic const short __LOCALE_DATA_WCuplow_diff_data[%zu] = {",
  659. 2 * (size_t) ul_count);
  660. for (i=0 ; i < ul_count ; i++) {
  661. if (i % 4 == 0) {
  662. fprintf(fp, "\n");
  663. }
  664. fprintf(fp, " %6d, %6d,", uldiff[i].u, uldiff[i].l);
  665. }
  666. fprintf(fp, "\n};\n\n");
  667. fprintf(fp, "#endif /* WANT_WCuplow_diff_data */\n\n");
  668. /* output_table(fp, "comb", &combtable); */
  669. /* output_table(fp, "width", &widthtable); */
  670. fclose(fp);
  671. }
  672. return EXIT_SUCCESS;
  673. }
  674. size_t newopt(unsigned char *ut, size_t usize, int shift, table_data *tbl)
  675. {
  676. static int recurse = 0;
  677. unsigned char *ti[RANGE+1]; /* table index */
  678. size_t numblocks;
  679. size_t blocksize;
  680. size_t uniq;
  681. size_t i, j;
  682. size_t smallest, t;
  683. unsigned char *ii_save;
  684. int uniqblock[256];
  685. unsigned char uit[RANGE+1];
  686. int shift2;
  687. ii_save = NULL;
  688. blocksize = 1 << shift;
  689. numblocks = usize >> shift;
  690. /* init table index */
  691. for (i=j=0 ; i < numblocks ; i++) {
  692. ti[i] = ut + j;
  693. j += blocksize;
  694. }
  695. /* sort */
  696. nu_val = blocksize;
  697. qsort(ti, numblocks, sizeof(unsigned char *), nu_memcmp);
  698. uniq = 1;
  699. uit[(ti[0]-ut)/blocksize] = 0;
  700. for (i=1 ; i < numblocks ; i++) {
  701. if (memcmp(ti[i-1], ti[i], blocksize) < 0) {
  702. if (++uniq > 255) {
  703. break;
  704. }
  705. uniqblock[uniq - 1] = i;
  706. }
  707. #if 1
  708. else if (memcmp(ti[i-1], ti[i], blocksize) > 0) {
  709. printf("bad sort %i!\n", i);
  710. abort();
  711. }
  712. #endif
  713. uit[(ti[i]-ut)/blocksize] = uniq - 1;
  714. }
  715. smallest = SIZE_MAX;
  716. shift2 = -1;
  717. if (uniq <= 255) {
  718. smallest = numblocks + uniq * blocksize;
  719. if (!recurse) {
  720. ++recurse;
  721. for (j=1 ; j < 14 ; j++) {
  722. if ((numblocks >> j) < 2) break;
  723. if (tbl) {
  724. ii_save = tbl->ii;
  725. tbl->ii = NULL;
  726. }
  727. if ((t = newopt(uit, numblocks, j, tbl)) < SIZE_MAX) {
  728. t += uniq * blocksize;
  729. }
  730. if (tbl) {
  731. tbl->ii = ii_save;
  732. }
  733. if (smallest >= t) {
  734. shift2 = j;
  735. smallest = t;
  736. if (!tbl->ii) {
  737. printf("ishift %zu tshift %zu size %zu\n",
  738. shift2, shift, t);
  739. }
  740. /* } else { */
  741. /* break; */
  742. }
  743. }
  744. --recurse;
  745. }
  746. } else {
  747. return SIZE_MAX;
  748. }
  749. if (tbl->ii) {
  750. if (recurse) {
  751. tbl->ii_shift = shift;
  752. tbl->ii_len = numblocks;
  753. memcpy(tbl->ii, uit, numblocks);
  754. tbl->ti = tbl->ii + tbl->ii_len;
  755. tbl->ti_len = uniq * blocksize;
  756. for (i=0 ; i < uniq ; i++) {
  757. memcpy(tbl->ti + i * blocksize, ti[uniqblock[i]], blocksize);
  758. }
  759. } else {
  760. ++recurse;
  761. printf("setting ishift %zu tshift %zu\n",
  762. shift2, shift);
  763. newopt(uit, numblocks, shift2, tbl);
  764. --recurse;
  765. tbl->ti_shift = shift;
  766. tbl->ut_len = uniq * blocksize;
  767. tbl->ut = tbl->ti + tbl->ti_len;
  768. for (i=0 ; i < uniq ; i++) {
  769. memcpy(tbl->ut + i * blocksize, ti[uniqblock[i]], blocksize);
  770. }
  771. }
  772. }
  773. return smallest;
  774. }