gen_wctype.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813
  1. /*
  2. * Copyright (C) 2000-2006 Erik Andersen <andersen@uclibc.org>
  3. *
  4. * Licensed under the LGPL v2.1, see the file COPYING.LIB in this tarball.
  5. */
  6. #define _GNU_SOURCE
  7. #include <stdio.h>
  8. #include <stdlib.h>
  9. #include <string.h>
  10. #include <locale.h>
  11. #include <wctype.h>
  12. #include <limits.h>
  13. #include <stdint.h>
  14. #include <wchar.h>
  15. #include <ctype.h>
  16. #include "include/bits/uClibc_charclass.h"
  17. /* 0x9 : space blank */
  18. /* 0xa : space */
  19. /* 0xb : space */
  20. /* 0xc : space */
  21. /* 0xd : space */
  22. /* 0x20 : space blank */
  23. /* 0x1680 : space blank */
  24. /* 0x2000 : space blank */
  25. /* 0x2001 : space blank */
  26. /* 0x2002 : space blank */
  27. /* 0x2003 : space blank */
  28. /* 0x2004 : space blank */
  29. /* 0x2005 : space blank */
  30. /* 0x2006 : space blank */
  31. /* 0x2008 : space blank */
  32. /* 0x2009 : space blank */
  33. /* 0x200a : space blank */
  34. /* 0x200b : space blank */
  35. /* 0x2028 : space */
  36. /* 0x2029 : space */
  37. /* 0x3000 : space blank */
  38. /* typecount[ 0] = 88670 C_alpha_nonupper_nonlower */
  39. /* typecount[ 1] = 742 C_alpha_lower */
  40. /* typecount[ 2] = 4 C_alpha_upper_lower */
  41. /* typecount[ 3] = 731 C_alpha_upper */
  42. /* typecount[ 4] = 10 C_digit */
  43. /* typecount[ 5] = 10270 C_punct */
  44. /* typecount[ 6] = 0 C_graph */
  45. /* typecount[ 7] = 0 C_print_space_nonblank */
  46. /* typecount[ 8] = 14 C_print_space_blank */
  47. /* typecount[ 9] = 0 C_space_nonblank_noncntrl */
  48. /* typecount[10] = 0 C_space_blank_noncntrl */
  49. /* typecount[11] = 6 C_cntrl_space_nonblank */
  50. /* typecount[12] = 1 C_cntrl_space_blank */
  51. /* typecount[13] = 60 C_cntrl_nonspace */
  52. /* typecount[14] = 96100 C_unclassified */
  53. /* typecount[15] = 0 empty_slot */
  54. /* Set to #if 0 to restrict wchars to 16 bits. */
  55. #if 1
  56. #define RANGE 0x2ffffUL
  57. #elif 0
  58. #define RANGE 0x1ffffUL
  59. #else
  60. #define RANGE 0xffffUL /* Restrict for 16-bit wchar_t... */
  61. #endif
  62. /* Some macros that test for various (w)ctype classes when passed one of the
  63. * designator values enumerated above. */
  64. #define mywalnum(D,C) ((unsigned)(D - 1) <= (__CTYPE_digit - 1))
  65. #define mywalpha(D,C) ((unsigned)(D - 1) <= (__CTYPE_alpha_upper - 1))
  66. #define mywblank(D,C) ((unsigned)(D - __CTYPE_print_space_nonblank) <= 5 && (D & 1))
  67. #define mywcntrl(D,C) ((unsigned)(D - __CTYPE_cntrl_space_nonblank) <= 2)
  68. #define mywdigit(D,C) (D == __CTYPE_digit)
  69. #define mywgraph(D,C) ((unsigned)(D - 1) <= (__CTYPE_graph - 1))
  70. #define mywlower(D,C) ((unsigned)(D - __CTYPE_alpha_lower) <= 1)
  71. #define mywprint(D,C) ((unsigned)(D - 1) <= (__CTYPE_print_space_blank - 1))
  72. #define mywpunct(D,C) (D == __CTYPE_punct)
  73. #define mywspace(D,C) ((unsigned)(D - __CTYPE_print_space_nonblank) <= 5)
  74. #define mywupper(D,C) ((unsigned)(D - __CTYPE_alpha_upper_lower) <= 1)
  75. /* #define mywxdigit(D,C) -- isxdigit is untestable this way.
  76. * But that's ok as isxdigit() (and isdigit() too) are locale-invariant. */
  77. #define mywxdigit(D,C) (mywdigit(D,C) || (unsigned)(((C) | 0x20) - 'a') <= 5)
  78. typedef struct {
  79. int32_t l;
  80. int32_t u;
  81. } uldiff_entry;
  82. typedef struct {
  83. uint16_t ii_len;
  84. uint16_t ti_len;
  85. uint16_t ut_len;
  86. unsigned char ii_shift;
  87. unsigned char ti_shift;
  88. unsigned char *ii;
  89. unsigned char *ti;
  90. unsigned char *ut;
  91. } table_data;
  92. static unsigned verbose;
  93. #define verbose_msg(msg...) if (verbose) fprintf(stderr, msg)
  94. void output_table(const char *name, table_data *tbl)
  95. {
  96. size_t i;
  97. printf("#define __LOCALE_DATA_WC%s_II_LEN %7u\n", name, tbl->ii_len);
  98. printf("#define __LOCALE_DATA_WC%s_TI_LEN %7u\n", name, tbl->ti_len);
  99. printf("#define __LOCALE_DATA_WC%s_UT_LEN %7u\n", name, tbl->ut_len);
  100. printf("#define __LOCALE_DATA_WC%s_II_SHIFT %7u\n", name, tbl->ii_shift);
  101. printf("#define __LOCALE_DATA_WC%s_TI_SHIFT %7u\n", name, tbl->ti_shift);
  102. printf("\n#ifdef WANT_WC%s_data\n", name);
  103. i = tbl->ii_len + tbl->ti_len + tbl->ut_len;
  104. printf("\nstatic const unsigned char __LOCALE_DATA_WC%s_data[%zu] = {", name, i);
  105. for (i = 0; i < tbl->ii_len; i++) {
  106. if (i % 12 == 0) {
  107. printf("\n");
  108. }
  109. printf(" %#04x,", tbl->ii[i]);
  110. }
  111. for (i = 0; i < tbl->ti_len; i++) {
  112. if (i % 12 == 0) {
  113. printf("\n");
  114. }
  115. printf(" %#04x,", tbl->ti[i]);
  116. }
  117. for (i = 0; i < tbl->ut_len; i++) {
  118. if (i % 12 == 0) {
  119. printf("\n");
  120. }
  121. printf(" %#04x,", tbl->ut[i]);
  122. }
  123. printf("\n};\n\n");
  124. printf("#endif /* WANT_WC%s_data */\n\n", name);
  125. }
  126. static void dump_table_data(table_data *tbl)
  127. {
  128. verbose_msg("ii_shift = %d ti_shift = %d\n"
  129. "ii_len = %d ti_len = %d ut_len = %d\n"
  130. "total = %d\n",
  131. tbl->ii_shift, tbl->ti_shift,
  132. tbl->ii_len, tbl->ti_len, tbl->ut_len,
  133. (int) tbl->ii_len + (int) tbl->ti_len + (int) tbl->ut_len);
  134. }
  135. /* For sorting the blocks of unsigned chars. */
  136. static size_t nu_val;
  137. int nu_memcmp(const void *a, const void *b)
  138. {
  139. return memcmp(*(unsigned char**)a, *(unsigned char**)b, nu_val);
  140. }
  141. static size_t newopt(unsigned char *ut, size_t usize, int shift, table_data *tbl);
  142. #define MAXTO 255 /* Restrict to minimal unsigned char max. */
  143. int main(int argc, char **argv)
  144. {
  145. long int u, l, tt;
  146. size_t smallest, t;
  147. unsigned int c;
  148. unsigned int d;
  149. int i, n;
  150. int ul_count = 0;
  151. uldiff_entry uldiff[MAXTO];
  152. table_data cttable;
  153. table_data ultable;
  154. #if 0
  155. table_data combtable;
  156. table_data widthtable;
  157. long int last_comb = 0;
  158. #endif
  159. unsigned char wct[(RANGE/2)+1]; /* wctype table (nibble per wchar) */
  160. unsigned char ult[RANGE+1]; /* upper/lower table */
  161. unsigned char combt[(RANGE/4)+1]; /* combining */
  162. unsigned char widtht[(RANGE/4)+1]; /* width */
  163. wctrans_t totitle;
  164. wctype_t is_comb, is_comb3;
  165. long int typecount[16];
  166. const char *typename[16];
  167. static const char empty_slot[] = "empty_slot";
  168. int built = 0;
  169. #define INIT_TYPENAME(X) typename[__CTYPE_##X] = "C_" #X
  170. for (i = 0; i < 16; i++) {
  171. typename[i] = empty_slot;
  172. }
  173. INIT_TYPENAME(unclassified);
  174. INIT_TYPENAME(alpha_nonupper_nonlower);
  175. INIT_TYPENAME(alpha_lower);
  176. INIT_TYPENAME(alpha_upper_lower);
  177. INIT_TYPENAME(alpha_upper);
  178. INIT_TYPENAME(digit);
  179. INIT_TYPENAME(punct);
  180. INIT_TYPENAME(graph);
  181. INIT_TYPENAME(print_space_nonblank);
  182. INIT_TYPENAME(print_space_blank);
  183. INIT_TYPENAME(space_nonblank_noncntrl);
  184. INIT_TYPENAME(space_blank_noncntrl);
  185. INIT_TYPENAME(cntrl_space_nonblank);
  186. INIT_TYPENAME(cntrl_space_blank);
  187. INIT_TYPENAME(cntrl_nonspace);
  188. memset(&cttable, 0, sizeof(table_data));
  189. memset(&ultable, 0, sizeof(table_data));
  190. #if 0
  191. memset(combtable, 0, sizeof(table_data));
  192. memset(widthtable, 0, sizeof(table_data));
  193. #endif
  194. setvbuf(stdout, NULL, _IONBF, 0);
  195. while (--argc) {
  196. ++argv;
  197. if (!strcmp(*argv, "-v")) {
  198. ++verbose;
  199. continue;
  200. }
  201. /* setlocale might be just a stub */
  202. if (!setlocale(LC_CTYPE, *argv)) {
  203. verbose_msg("setlocale(LC_CTYPE,%s) failed! Skipping this locale...\n", *argv);
  204. continue;
  205. }
  206. if (!(totitle = wctrans("totitle"))) {
  207. verbose_msg("no totitle transformation.\n");
  208. }
  209. if (!(is_comb = wctype("combining"))) {
  210. verbose_msg("no combining wctype.\n");
  211. }
  212. if (!(is_comb3 = wctype("combining_level3"))) {
  213. verbose_msg("no combining_level3 wctype.\n");
  214. }
  215. if (!built) {
  216. built = 1;
  217. ul_count = 1;
  218. uldiff[0].u = uldiff[0].l = 0;
  219. memset(wct, 0, sizeof(wct));
  220. memset(combt, 0, sizeof(combt));
  221. memset(widtht, 0, sizeof(widtht));
  222. for (i = 0; i < 16; i++) {
  223. typecount[i] = 0;
  224. }
  225. for (c = 0; c <= RANGE; c++) {
  226. if (iswdigit(c)) {
  227. d = __CTYPE_digit;
  228. } else if (iswalpha(c)) {
  229. d = __CTYPE_alpha_nonupper_nonlower;
  230. if (iswlower(c)) {
  231. d = __CTYPE_alpha_lower;
  232. if (iswupper(c)) {
  233. d = __CTYPE_alpha_upper_lower;
  234. }
  235. } else if (iswupper(c)) {
  236. d = __CTYPE_alpha_upper;
  237. }
  238. } else if (iswpunct(c)) {
  239. d = __CTYPE_punct;
  240. } else if (iswgraph(c)) {
  241. d = __CTYPE_graph;
  242. } else if (iswprint(c)) {
  243. d = __CTYPE_print_space_nonblank;
  244. if (iswblank(c)) {
  245. d = __CTYPE_print_space_blank;
  246. }
  247. } else if (iswspace(c) && !iswcntrl(c)) {
  248. d = __CTYPE_space_nonblank_noncntrl;
  249. if (iswblank(c)) {
  250. d = __CTYPE_space_blank_noncntrl;
  251. }
  252. } else if (iswcntrl(c)) {
  253. d = __CTYPE_cntrl_nonspace;
  254. if (iswspace(c)) {
  255. d = __CTYPE_cntrl_space_nonblank;
  256. if (iswblank(c)) {
  257. d = __CTYPE_cntrl_space_blank;
  258. }
  259. }
  260. } else {
  261. d = __CTYPE_unclassified;
  262. }
  263. ++typecount[d];
  264. #if 0
  265. if (iswspace(c)) {
  266. if (iswblank(c)) {
  267. verbose_msg("%#8x : space blank\n", c);
  268. } else {
  269. verbose_msg("%#8x : space\n", c);
  270. }
  271. }
  272. #endif
  273. #if 0
  274. if (c < 256) {
  275. unsigned int curr_stdclib;
  276. curr_stdclib = 0;
  277. if (isalnum(c)) ++curr_stdclib; curr_stdclib <<= 1;
  278. if (isalpha(c)) ++curr_stdclib; curr_stdclib <<= 1;
  279. if (isblank(c)) ++curr_stdclib; curr_stdclib <<= 1;
  280. if (iscntrl(c)) ++curr_stdclib; curr_stdclib <<= 1;
  281. if (isdigit(c)) ++curr_stdclib; curr_stdclib <<= 1;
  282. if (isgraph(c)) ++curr_stdclib; curr_stdclib <<= 1;
  283. if (islower(c)) ++curr_stdclib; curr_stdclib <<= 1;
  284. if (isprint(c)) ++curr_stdclib; curr_stdclib <<= 1;
  285. if (ispunct(c)) ++curr_stdclib; curr_stdclib <<= 1;
  286. if (isspace(c)) ++curr_stdclib; curr_stdclib <<= 1;
  287. if (isupper(c)) ++curr_stdclib; curr_stdclib <<= 1;
  288. if (isxdigit(c)) ++curr_stdclib;
  289. verbose_msg("%#8x : ctype %#4x\n", c, curr_stdclib);
  290. }
  291. #endif
  292. #if 1
  293. /* Paranoid checking... */
  294. {
  295. unsigned int curr_stdclib;
  296. unsigned int mine;
  297. curr_stdclib = 0;
  298. if (iswalnum(c)) ++curr_stdclib; curr_stdclib <<= 1;
  299. if (iswalpha(c)) ++curr_stdclib; curr_stdclib <<= 1;
  300. if (iswblank(c)) ++curr_stdclib; curr_stdclib <<= 1;
  301. if (iswcntrl(c)) ++curr_stdclib; curr_stdclib <<= 1;
  302. if (iswdigit(c)) ++curr_stdclib; curr_stdclib <<= 1;
  303. if (iswgraph(c)) ++curr_stdclib; curr_stdclib <<= 1;
  304. if (iswlower(c)) ++curr_stdclib; curr_stdclib <<= 1;
  305. if (iswprint(c)) ++curr_stdclib; curr_stdclib <<= 1;
  306. if (iswpunct(c)) ++curr_stdclib; curr_stdclib <<= 1;
  307. if (iswspace(c)) ++curr_stdclib; curr_stdclib <<= 1;
  308. if (iswupper(c)) ++curr_stdclib; curr_stdclib <<= 1;
  309. if (iswxdigit(c)) ++curr_stdclib;
  310. mine = 0;
  311. if (mywalnum(d,c)) ++mine; mine <<= 1;
  312. if (mywalpha(d,c)) ++mine; mine <<= 1;
  313. if (mywblank(d,c)) ++mine; mine <<= 1;
  314. if (mywcntrl(d,c)) ++mine; mine <<= 1;
  315. if (mywdigit(d,c)) ++mine; mine <<= 1;
  316. if (mywgraph(d,c)) ++mine; mine <<= 1;
  317. if (mywlower(d,c)) ++mine; mine <<= 1;
  318. if (mywprint(d,c)) ++mine; mine <<= 1;
  319. if (mywpunct(d,c)) ++mine; mine <<= 1;
  320. if (mywspace(d,c)) ++mine; mine <<= 1;
  321. if (mywupper(d,c)) ++mine; mine <<= 1;
  322. if (mywxdigit(d,c)) ++mine;
  323. if (curr_stdclib != mine) {
  324. verbose_msg("%#8x : curr_stdclib %#4x != %#4x mine %u\n", c, curr_stdclib, mine, d);
  325. return EXIT_FAILURE;
  326. }
  327. #if 0
  328. if (iswctype(c,is_comb) || iswctype(c,is_comb3)) {
  329. /* if (!iswpunct(c)) { */
  330. verbose_msg("%#8x : %d %d %#4x\n",
  331. c, iswctype(c,is_comb),iswctype(c,is_comb3), curr_stdclib);
  332. /* } */
  333. }
  334. #endif
  335. #if 0
  336. if (iswctype(c,is_comb) || iswctype(c,is_comb3)) {
  337. if (!last_comb) {
  338. verbose_msg("%#8x - ", c);
  339. last_comb = c;
  340. } else if (last_comb + 1 < c) {
  341. verbose_msg("%#8x\n%#8x - ", last_comb, c);
  342. last_comb = c;
  343. } else {
  344. last_comb = c;
  345. }
  346. }
  347. #endif
  348. }
  349. #endif
  350. combt[c/4] |= ((((!!iswctype(c,is_comb)) << 1) | !!iswctype(c,is_comb3))
  351. << ((c & 3) << 1));
  352. /* comb3t[c/8] |= ((!!iswctype(c,is_comb3)) << (c & 7)); */
  353. /* widtht[c/4] |= (wcwidth(c) << ((c & 3) << 1)); */
  354. if (c & 1) { /* Use the high nibble for odd numbered wchars. */
  355. d <<= 4;
  356. }
  357. wct[c/2] |= d;
  358. l = (long)(int) towlower(c) - c;
  359. u = (long)(int) towupper(c) - c;
  360. ult[c] = 0;
  361. if (l || u) {
  362. if ((l != (int32_t)l) || (u != (int32_t)u)) {
  363. verbose_msg("range assumption error! %x %ld %ld\n", c, l, u);
  364. return EXIT_FAILURE;
  365. }
  366. for (i = 0; i < ul_count; i++) {
  367. if ((l == uldiff[i].l) && (u == uldiff[i].u)) {
  368. goto found;
  369. }
  370. }
  371. uldiff[ul_count].l = l;
  372. uldiff[ul_count].u = u;
  373. ++ul_count;
  374. if (ul_count > MAXTO) {
  375. verbose_msg("too many touppers/tolowers!\n");
  376. return EXIT_FAILURE;
  377. }
  378. found:
  379. ult[c] = i;
  380. }
  381. }
  382. for (i = 0; i < 16; i++) {
  383. verbose_msg("typecount[%2d] = %8ld %s\n", i, typecount[i], typename[i]);
  384. }
  385. verbose_msg("optimizing is* table..\n");
  386. n = -1;
  387. smallest = SIZE_MAX;
  388. cttable.ii = NULL;
  389. for (i = 0; i < 14; i++) {
  390. t = newopt(wct, (RANGE/2)+1, i, &cttable);
  391. if (smallest >= t) {
  392. n = i;
  393. smallest = t;
  394. /* } else { */
  395. /* break; */
  396. }
  397. }
  398. verbose_msg("smallest = %zu\n", smallest);
  399. if (!(cttable.ii = malloc(smallest))) {
  400. verbose_msg("couldn't allocate space!\n");
  401. return EXIT_FAILURE;
  402. }
  403. smallest = SIZE_MAX;
  404. newopt(wct, (RANGE/2)+1, n, &cttable);
  405. ++cttable.ti_shift; /* correct for nibble mode */
  406. verbose_msg("optimizing u/l-to table..\n");
  407. smallest = SIZE_MAX;
  408. ultable.ii = NULL;
  409. for (i = 0; i < 14; i++) {
  410. t = newopt(ult, RANGE+1, i, &ultable);
  411. if (smallest >= t) {
  412. n = i;
  413. smallest = t;
  414. /* } else { */
  415. /* break; */
  416. }
  417. }
  418. verbose_msg("%lu (smallest) + %lu (u/l diffs) = %lu\n",
  419. (unsigned long) smallest,
  420. (unsigned long) (4 * ul_count),
  421. (unsigned long) (smallest + 4 * ul_count)
  422. );
  423. verbose_msg("smallest = %zu\n", smallest);
  424. if (!(ultable.ii = malloc(smallest))) {
  425. verbose_msg("couldn't allocate space!\n");
  426. return EXIT_FAILURE;
  427. }
  428. smallest = SIZE_MAX;
  429. newopt(ult, RANGE+1, n, &ultable);
  430. #if 0
  431. verbose_msg("optimizing comb table..\n");
  432. smallest = SIZE_MAX;
  433. combtable.ii = NULL;
  434. for (i = 0; i < 14; i++) {
  435. t = newopt(combt, sizeof(combt), i, &combtable);
  436. if (smallest >= t) {
  437. n = i;
  438. smallest = t;
  439. /* } else { */
  440. /* break; */
  441. }
  442. }
  443. verbose_msg("smallest = %zu\n", smallest);
  444. if (!(combtable.ii = malloc(smallest))) {
  445. verbose_msg("couldn't allocate space!\n");
  446. return EXIT_FAILURE;
  447. }
  448. smallest = SIZE_MAX;
  449. newopt(combt, sizeof(combt), n, &combtable);
  450. combtable.ti_shift += 4; /* correct for 4 entries per */
  451. #endif
  452. #if 0
  453. verbose_msg("optimizing width table..\n");
  454. smallest = SIZE_MAX;
  455. widthtable.ii = NULL;
  456. for (i = 0; i < 14; i++) {
  457. t = newopt(widtht, sizeof(widtht), i, &widthtable);
  458. if (smallest >= t) {
  459. n = i;
  460. smallest = t;
  461. /* } else { */
  462. /* break; */
  463. }
  464. }
  465. verbose_msg("smallest = %zu\n", smallest);
  466. if (!(widthtable.ii = malloc(smallest))) {
  467. verbose_msg("couldn't allocate space!\n");
  468. return EXIT_FAILURE;
  469. }
  470. smallest = SIZE_MAX;
  471. newopt(widtht, sizeof(widtht), n, &widthtable);
  472. widthtable.ti_shift += 4; /* correct for 4 entries per */
  473. #endif
  474. #if 0
  475. verbose_msg("optimizing comb3 table..\n");
  476. smallest = SIZE_MAX;
  477. comb3table.ii = NULL;
  478. for (i = 0; i < 14; i++) {
  479. t = newopt(comb3t, sizeof(comb3t), i, &comb3table);
  480. if (smallest >= t) {
  481. n = i;
  482. smallest = t;
  483. /* } else { */
  484. /* break; */
  485. }
  486. }
  487. verbose_msg("smallest = %zu\n", smallest);
  488. if (!(comb3table.ii = malloc(smallest))) {
  489. verbose_msg("couldn't allocate space!\n");
  490. return EXIT_FAILURE;
  491. }
  492. smallest = SIZE_MAX;
  493. newopt(comb3t, sizeof(comb3t), n, &comb3table);
  494. comb3table.ti_shift += 8; /* correct for 4 entries per */
  495. #endif
  496. dump_table_data(&cttable);
  497. dump_table_data(&ultable);
  498. #if 0
  499. dump_table_data(&combtable);
  500. #endif
  501. }
  502. verbose_msg("verifying for %s...\n", *argv);
  503. #if RANGE == 0xffffU
  504. for (c = 0; c <= 0xffffUL; c++)
  505. #else
  506. for (c = 0; c <= 0x10ffffUL; c++)
  507. #endif
  508. {
  509. unsigned int curr_stdclib;
  510. unsigned int mine;
  511. unsigned int upper, lower;
  512. #if 0
  513. #if RANGE < 0x10000UL
  514. if (c == 0x10000UL) {
  515. c = 0x30000UL; /* skip 1st and 2nd sup planes */
  516. }
  517. #elif RANGE < 0x20000UL
  518. if (c == 0x20000UL) {
  519. c = 0x30000UL; /* skip 2nd sup planes */
  520. }
  521. #endif
  522. #endif
  523. curr_stdclib = 0;
  524. if (iswalnum(c)) ++curr_stdclib; curr_stdclib <<= 1;
  525. if (iswalpha(c)) ++curr_stdclib; curr_stdclib <<= 1;
  526. if (iswblank(c)) ++curr_stdclib; curr_stdclib <<= 1;
  527. if (iswcntrl(c)) ++curr_stdclib; curr_stdclib <<= 1;
  528. if (iswdigit(c)) ++curr_stdclib; curr_stdclib <<= 1;
  529. if (iswgraph(c)) ++curr_stdclib; curr_stdclib <<= 1;
  530. if (iswlower(c)) ++curr_stdclib; curr_stdclib <<= 1;
  531. if (iswprint(c)) ++curr_stdclib; curr_stdclib <<= 1;
  532. if (iswpunct(c)) ++curr_stdclib; curr_stdclib <<= 1;
  533. if (iswspace(c)) ++curr_stdclib; curr_stdclib <<= 1;
  534. if (iswupper(c)) ++curr_stdclib; curr_stdclib <<= 1;
  535. if (iswxdigit(c)) ++curr_stdclib;
  536. {
  537. unsigned int u;
  538. int n = 0, sc = 0; /* = 0 for verbose_msg only */
  539. int i0 = 0, i1 = 0;
  540. u = c;
  541. if (u <= RANGE) {
  542. sc = u & ((1 << cttable.ti_shift) - 1);
  543. u >>= cttable.ti_shift;
  544. n = u & ((1 << cttable.ii_shift) - 1);
  545. u >>= cttable.ii_shift;
  546. i0 = cttable.ii[u];
  547. i0 <<= cttable.ii_shift;
  548. i1 = cttable.ti[i0 + n];
  549. i1 <<= (cttable.ti_shift - 1);
  550. d = cttable.ut[i1 + (sc >> 1)];
  551. if (sc & 1) {
  552. d >>= 4;
  553. }
  554. d &= 0x0f;
  555. } else if (((unsigned)(c - 0xe0020UL) <= 0x5f) || (c == 0xe0001UL)) {
  556. d = __CTYPE_punct;
  557. } else if ((unsigned)(c - 0xf0000UL) < 0x20000UL) {
  558. if ((c & 0xffffU) <= 0xfffdU) {
  559. d = __CTYPE_punct;
  560. } else {
  561. d = __CTYPE_unclassified;
  562. }
  563. } else {
  564. d = __CTYPE_unclassified;
  565. }
  566. mine = 0;
  567. if (mywalnum(d,c)) ++mine; mine <<= 1;
  568. if (mywalpha(d,c)) ++mine; mine <<= 1;
  569. if (mywblank(d,c)) ++mine; mine <<= 1;
  570. if (mywcntrl(d,c)) ++mine; mine <<= 1;
  571. if (mywdigit(d,c)) ++mine; mine <<= 1;
  572. if (mywgraph(d,c)) ++mine; mine <<= 1;
  573. if (mywlower(d,c)) ++mine; mine <<= 1;
  574. if (mywprint(d,c)) ++mine; mine <<= 1;
  575. if (mywpunct(d,c)) ++mine; mine <<= 1;
  576. if (mywspace(d,c)) ++mine; mine <<= 1;
  577. if (mywupper(d,c)) ++mine; mine <<= 1;
  578. if (mywxdigit(d,c)) ++mine;
  579. if (curr_stdclib != mine) {
  580. verbose_msg("%#8x : curr_stdclib %#4x != %#4x mine %d\n", c, curr_stdclib, mine, d);
  581. if (c < 0x30000UL) {
  582. verbose_msg("sc=%#x u=%#x n=%#x i0=%#x i1=%#x\n", sc, u, n, i0, i1);
  583. }
  584. }
  585. upper = lower = u = c;
  586. if (u <= RANGE) {
  587. sc = u & ((1 << ultable.ti_shift) - 1);
  588. u >>= ultable.ti_shift;
  589. n = u & ((1 << ultable.ii_shift) - 1);
  590. u >>= ultable.ii_shift;
  591. i0 = ultable.ii[u];
  592. i0 <<= ultable.ii_shift;
  593. i1 = ultable.ti[i0 + n];
  594. i1 <<= (ultable.ti_shift);
  595. i1 += sc;
  596. i0 = ultable.ut[i1];
  597. upper = c + uldiff[i0].u;
  598. lower = c + uldiff[i0].l;
  599. }
  600. if (towupper(c) != upper) {
  601. verbose_msg("%#8x : towupper curr_stdclib %#4x != %#4x mine\n",
  602. c, towupper(c), upper);
  603. }
  604. if (towlower(c) != lower) {
  605. verbose_msg("%#8x : towlower curr_stdclib %#4x != %#4x mine i0 = %d\n",
  606. c, towlower(c), lower, i0);
  607. }
  608. if (totitle && ((tt = towctrans(c, totitle)) != upper)) {
  609. verbose_msg("%#8x : totitle curr_stdclib %#4lx != %#4x mine i0 = %d\n",
  610. c, tt, upper, i0);
  611. }
  612. }
  613. if ((c & 0xfff) == 0xfff) verbose_msg(".");
  614. }
  615. verbose_msg("done\n");
  616. }
  617. if (built) {
  618. printf("#define __LOCALE_DATA_WC_TABLE_DOMAIN_MAX %#8lx\n\n",
  619. (unsigned long) RANGE);
  620. output_table("ctype", &cttable);
  621. output_table("uplow", &ultable);
  622. printf("#define __LOCALE_DATA_WCuplow_diffs %7u\n", ul_count);
  623. printf("\n#ifdef WANT_WCuplow_diff_data\n\n");
  624. printf("\nstatic const int32_t __LOCALE_DATA_WCuplow_diff_data[%zu] = {",
  625. 2 * (size_t) ul_count);
  626. for (i = 0; i < ul_count; i++) {
  627. if (i % 4 == 0) {
  628. printf("\n");
  629. }
  630. printf(" %6d, %6d,", uldiff[i].u, uldiff[i].l);
  631. }
  632. printf("\n};\n\n");
  633. printf("#endif /* WANT_WCuplow_diff_data */\n\n");
  634. /* output_table("comb", &combtable); */
  635. /* output_table("width", &widthtable); */
  636. }
  637. return !built;
  638. }
  639. size_t newopt(unsigned char *ut, size_t usize, int shift, table_data *tbl)
  640. {
  641. static int recurse;
  642. unsigned char *ti[RANGE+1]; /* table index */
  643. size_t numblocks;
  644. size_t blocksize;
  645. size_t uniq;
  646. size_t i, j;
  647. size_t smallest, t;
  648. unsigned char *ii_save;
  649. int uniqblock[256];
  650. unsigned char uit[RANGE+1];
  651. int shift2;
  652. memset(uniqblock, 0x00, sizeof(uniqblock));
  653. ii_save = NULL;
  654. blocksize = 1 << shift;
  655. numblocks = usize >> shift;
  656. /* init table index */
  657. for (i=j = 0; i < numblocks; i++) {
  658. ti[i] = ut + j;
  659. j += blocksize;
  660. }
  661. /* sort */
  662. nu_val = blocksize;
  663. qsort(ti, numblocks, sizeof(unsigned char *), nu_memcmp);
  664. uniq = 1;
  665. uit[(ti[0]-ut)/blocksize] = 0;
  666. for (i=1; i < numblocks; i++) {
  667. if (memcmp(ti[i-1], ti[i], blocksize) < 0) {
  668. if (++uniq > 255) {
  669. break;
  670. }
  671. uniqblock[uniq - 1] = i;
  672. }
  673. #if 1
  674. else if (memcmp(ti[i-1], ti[i], blocksize) > 0) {
  675. verbose_msg("bad sort %li!\n", (long) i);
  676. abort();
  677. }
  678. #endif
  679. uit[(ti[i]-ut)/blocksize] = uniq - 1;
  680. }
  681. smallest = SIZE_MAX;
  682. shift2 = -1;
  683. if (uniq > 255)
  684. return SIZE_MAX;
  685. smallest = numblocks + uniq * blocksize;
  686. if (!recurse) {
  687. ++recurse;
  688. for (j=1; j < 14; j++) {
  689. if ((numblocks >> j) < 2) break;
  690. if (tbl) {
  691. ii_save = tbl->ii;
  692. tbl->ii = NULL;
  693. }
  694. if ((t = newopt(uit, numblocks, j, tbl)) < SIZE_MAX) {
  695. t += uniq * blocksize;
  696. }
  697. if (tbl) {
  698. tbl->ii = ii_save;
  699. }
  700. if (smallest >= t) {
  701. shift2 = j;
  702. smallest = t;
  703. if (!tbl->ii) {
  704. verbose_msg("ishift %u tshift %u size %lu\n",
  705. shift2, shift, (unsigned long) t);
  706. }
  707. /* } else { */
  708. /* break; */
  709. }
  710. }
  711. --recurse;
  712. }
  713. if (tbl->ii) {
  714. if (recurse) {
  715. tbl->ii_shift = shift;
  716. tbl->ii_len = numblocks;
  717. memcpy(tbl->ii, uit, numblocks);
  718. tbl->ti = tbl->ii + tbl->ii_len;
  719. tbl->ti_len = uniq * blocksize;
  720. for (i = 0; i < uniq; i++) {
  721. memcpy(tbl->ti + i * blocksize, ti[uniqblock[i]], blocksize);
  722. }
  723. } else {
  724. ++recurse;
  725. verbose_msg("setting ishift %u tshift %u\n",
  726. shift2, shift);
  727. newopt(uit, numblocks, shift2, tbl);
  728. --recurse;
  729. tbl->ti_shift = shift;
  730. tbl->ut_len = uniq * blocksize;
  731. tbl->ut = tbl->ti + tbl->ti_len;
  732. for (i = 0; i < uniq; i++) {
  733. memcpy(tbl->ut + i * blocksize, ti[uniqblock[i]], blocksize);
  734. }
  735. }
  736. }
  737. return smallest;
  738. }
  739. /* vi: set sw=4 ts=4: */