gen_collate.c 97 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940
  1. /* TODO:
  2. *
  3. * add UNDEFINED at end if not specified
  4. * convert POSITION -> FORWARD,POSITION
  5. *
  6. *
  7. * deal with lowercase in <Uhhhh>
  8. *
  9. * what about reorders that keep the same rule?
  10. *
  11. * remove "unused" collation elements? (probably doesn't save much)
  12. *
  13. * add_rule function ... returns index into rule table after possibly adding custom-indexed rule
  14. * but don't forget about multichar weights... replace with strings of indexes
  15. *
  16. */
  17. #ifndef _GNU_SOURCE
  18. #define _GNU_SOURCE
  19. #endif
  20. #include <stddef.h>
  21. #include <stdio.h>
  22. #include <stdlib.h>
  23. #include <string.h>
  24. #include <stdint.h>
  25. #include <stdarg.h>
  26. #include <limits.h>
  27. #include <ctype.h>
  28. #include <assert.h>
  29. #include <search.h>
  30. typedef struct {
  31. char *name; /* */
  32. int num_weights; /* */
  33. int ii_shift; /* */
  34. int ti_shift; /* */
  35. int ii_len; /* */
  36. int ti_len; /* */
  37. int max_weight; /* */
  38. int num_col_base; /* */
  39. int max_col_index; /* */
  40. int undefined_idx; /* */
  41. int range_low; /* */
  42. int range_count; /* high - low */
  43. int range_base_weight; /* */
  44. int num_starters; /* */
  45. int range_rule_offset; /* */
  46. int wcs2colidt_offset; /* */
  47. int index2weight_offset; /* */
  48. int index2ruleidx_offset; /* */
  49. int multistart_offset; /* */
  50. } base_locale_t;
  51. #define BASE_LOCALE_LEN 20
  52. static base_locale_t base_locale_array[BASE_LOCALE_LEN];
  53. static size_t base_locale_len;
  54. typedef struct {
  55. char *name; /* */
  56. int base_idx; /* */
  57. int undefined_idx; /* */
  58. int overrides_offset; /* */
  59. int multistart_offset; /* */
  60. } der_locale_t;
  61. #define DER_LOCALE_LEN 300
  62. static der_locale_t der_locale_array[DER_LOCALE_LEN];
  63. static size_t der_locale_len;
  64. #define OVERRIDE_LEN 50000
  65. static uint16_t override_buffer[OVERRIDE_LEN];
  66. static size_t override_len;
  67. #define MULTISTART_LEN 10000
  68. static uint16_t multistart_buffer[MULTISTART_LEN];
  69. static size_t multistart_len;
  70. #define WCS2COLIDT_LEN 200000
  71. static uint16_t wcs2colidt_buffer[WCS2COLIDT_LEN];
  72. static size_t wcs2colidt_len;
  73. #define INDEX2WEIGHT_LEN 200000
  74. static uint16_t index2weight_buffer[INDEX2WEIGHT_LEN];
  75. static size_t index2weight_len;
  76. static uint16_t index2ruleidx_buffer[INDEX2WEIGHT_LEN];
  77. static size_t index2ruleidx_len;
  78. #define WEIGHTSTR_LEN 10000
  79. static uint16_t weightstr_buffer[WEIGHTSTR_LEN];
  80. static size_t weightstr_len;
  81. #define RULETABLE_LEN (1L<<16)
  82. static uint16_t ruletable_buffer[RULETABLE_LEN];
  83. static size_t ruletable_len;
  84. #define RANGE (0x10000UL)
  85. typedef uint16_t tbl_item;
  86. static uint16_t u16_buf[10000];
  87. static int u16_buf_len;
  88. static int u16_starter;
  89. typedef struct {
  90. uint16_t ii_len;
  91. uint16_t ti_len;
  92. uint16_t ut_len;
  93. unsigned char ii_shift;
  94. unsigned char ti_shift;
  95. tbl_item *ii;
  96. tbl_item *ti;
  97. tbl_item *ut;
  98. } table_data;
  99. static size_t newopt(tbl_item *ut, size_t usize, int shift, table_data *tbl);
  100. #define MAX_COLLATION_WEIGHTS 4
  101. #define MAX_FNO 1
  102. #define MAX_FILES (MAX_FNO + 1)
  103. static FILE *fstack[MAX_FILES];
  104. static char *fname[MAX_FILES];
  105. static int lineno[MAX_FILES];
  106. static int fno = -1;
  107. static tbl_item wcs2index[RANGE];
  108. static char linebuf[1024];
  109. static char *pos;
  110. static char *pos_e = NULL;
  111. static char end_of_token = 0; /* slot to save */
  112. #define IN_ORDER 0x01
  113. #define IN_REORDER 0x02
  114. #define IN_REORDER_SECTIONS 0x04
  115. static int order_state;
  116. static int cur_num_weights; /* number of weights in current use */
  117. static char cur_rule[MAX_COLLATION_WEIGHTS];
  118. static int anonsection = 0;
  119. typedef struct ll_item_struct ll_item_t;
  120. struct ll_item_struct {
  121. ll_item_t *next;
  122. ll_item_t *prev;
  123. void *data;
  124. int data_type;
  125. int idx;
  126. };
  127. static ll_item_t *reorder_section_ptr = NULL;
  128. static int superset;
  129. static int superset_order_start_cnt; /* only support one order for now */
  130. static int superset_in_sync;
  131. static ll_item_t *comm_cur_ptr;
  132. static ll_item_t *comm_prev_ptr;
  133. enum {
  134. R_FORWARD = 0x01,
  135. R_POSITION = 0x02,
  136. R_BACKWARD = 0x04 /* must be largest in value */
  137. };
  138. typedef struct {
  139. size_t num_weights;
  140. char rule[MAX_COLLATION_WEIGHTS];
  141. const char *colitem[MAX_COLLATION_WEIGHTS];
  142. } weight_t;
  143. static void *root_weight = NULL;
  144. size_t unique_weights = 0;
  145. typedef struct {
  146. const char *symbol;
  147. weight_t *weight;
  148. } weighted_item_t;
  149. typedef struct {
  150. const char *symbol1;
  151. const char *symbol2;
  152. int length;
  153. weight_t *weight;
  154. } range_item_t;
  155. typedef struct {
  156. const char *name;
  157. ll_item_t *itm_list; /* weighted_item_t list .. circular!!! */
  158. size_t num_items;
  159. size_t num_rules;
  160. char rules[MAX_COLLATION_WEIGHTS];
  161. } section_t;
  162. static section_t *cur_section = NULL;
  163. typedef struct {
  164. const char *symbol;
  165. ll_item_t *node;
  166. } wi_index_t;
  167. typedef struct col_locale_struct col_locale_t;
  168. struct col_locale_struct {
  169. char *name;
  170. void *root_colitem; /* all base and derived, or just derived */
  171. void *root_element;
  172. void *root_scripts;
  173. void *root_wi_index;
  174. void *root_wi_index_reordered;
  175. ll_item_t *section_list;
  176. col_locale_t *base_locale; /* null if this is a base */
  177. void *root_derived_wi;
  178. ll_item_t *derived_list;
  179. void *root_starter_char;
  180. void *root_starter_all;
  181. ll_item_t *undefined_idx;
  182. };
  183. typedef struct {
  184. const char *symbol;
  185. int idx;
  186. } col_index_t;
  187. static void *root_col_locale = NULL;
  188. typedef struct {
  189. const char *keyword;
  190. void (*handler)(void);
  191. } keyword_table_t;
  192. typedef struct {
  193. const char *string;
  194. const char *element; /* NULL if collating symbol */
  195. } colitem_t;
  196. static col_locale_t *cur_base = NULL;
  197. static col_locale_t *cur_derived = NULL;
  198. static col_locale_t *cur_col = NULL;
  199. static void *root_sym = NULL;
  200. static size_t num_sym = 0;
  201. static size_t mem_sym = 0;
  202. static void error_msg(const char *fmt, ...) __attribute__ ((noreturn, format (printf, 1, 2)));
  203. static void *xmalloc(size_t n);
  204. static char *xsymdup(const char *s); /* only allocate once... store in a tree */
  205. static void pushfile(char *filename);
  206. static void popfile(void);
  207. static void processfile(void);
  208. static int iscommentchar(int);
  209. static void eatwhitespace(void);
  210. static int next_line(void);
  211. static char *next_token(void);
  212. static void do_unrecognized(void);
  213. static col_locale_t *new_col_locale(char *name);
  214. static ll_item_t *new_ll_item(int data_type, void *data);
  215. static weight_t *register_weight(weight_t *w);
  216. static size_t ll_len(ll_item_t *l);
  217. static size_t ll_count(ll_item_t *l, int mask);
  218. static void add_wi_index(ll_item_t *l);
  219. static size_t tnumnodes(const void *root);
  220. static ll_item_t *find_wi_index(const char *sym, col_locale_t *cl);
  221. static void mark_reordered(const char *sym);
  222. static ll_item_t *find_wi_index_reordered(const char *sym);
  223. static ll_item_t *next_comm_ptr(void);
  224. static ll_item_t *init_comm_ptr(void);
  225. static ll_item_t *find_ll_last(ll_item_t *p);
  226. static void dump_weights(const char *name);
  227. static void finalize_base(void);
  228. static int is_ucode(const char *s);
  229. static int sym_cmp(const void *n1, const void *n2);
  230. static void do_starter_lists(col_locale_t *cl);
  231. static void dump_base_locale(int n);
  232. static void dump_der_locale(int n);
  233. static void dump_collate(FILE *fp);
  234. enum {
  235. DT_SECTION = 0x01,
  236. DT_WEIGHTED = 0x02,
  237. DT_REORDER = 0x04, /* a section to support reorder_after */
  238. DT_COL_LOCALE = 0x08,
  239. DT_RANGE = 0x10,
  240. };
  241. static section_t *new_section(const char *name)
  242. {
  243. section_t *p;
  244. char buf[128];
  245. p = xmalloc(sizeof(section_t));
  246. if (!name) { /* anonymous section */
  247. name = buf;
  248. snprintf(buf, sizeof(buf), "anon%05d", anonsection);
  249. ++anonsection;
  250. } else if (*name != '<') { /* reorder */
  251. name = buf;
  252. snprintf(buf, sizeof(buf), "%s %05d", cur_col->name, anonsection);
  253. ++anonsection;
  254. }
  255. #warning devel code
  256. /* fprintf(stderr, "section %s\n", name); */
  257. p->name = xsymdup(name);
  258. p->itm_list = NULL;
  259. p->num_items = 0;
  260. p->num_rules = 0;
  261. memset(p->rules, 0, MAX_COLLATION_WEIGHTS);
  262. /* cur_num_weights = p->num_rules = 0; */
  263. /* memset(p->rules, 0, MAX_COLLATION_WEIGHTS); */
  264. /* memset(cur_rule, R_FORWARD, 4); */
  265. #warning devel code
  266. if (*p->name == 'a') {
  267. cur_num_weights = p->num_rules = 4;
  268. memset(p->rules, R_FORWARD, 4);
  269. memset(cur_rule, R_FORWARD, 4);
  270. p->rules[3] |= R_POSITION;
  271. cur_rule[3] |= R_POSITION;
  272. }
  273. /* fprintf(stderr, "new section %s -- cur_num_weights = %d\n", p->name, cur_num_weights); */
  274. return p;
  275. }
  276. static void do_order_start(void);
  277. static void do_order_end(void);
  278. static void do_reorder_after(void);
  279. static void do_reorder_end(void);
  280. static void do_reorder_sections_after(void);
  281. static void do_reorder_sections_end(void);
  282. static void do_copy(void);
  283. static void do_colsym(void);
  284. static void do_colele(void);
  285. static void do_script(void);
  286. static void do_range(void);
  287. static col_locale_t *new_col_locale(char *name);
  288. static int colitem_cmp(const void *n1, const void *n2);
  289. static int colelement_cmp(const void *n1, const void *n2);
  290. static void del_colitem(colitem_t *p);
  291. static colitem_t *new_colitem(char *item, char *def);
  292. static void add_colitem(char *item, char *def);
  293. static void add_script(const char *s);
  294. static unsigned int add_rule(weighted_item_t *wi);
  295. static unsigned int add_range_rule(range_item_t *ri);
  296. static const keyword_table_t keyword_table[] = {
  297. { "collating-symbol", do_colsym },
  298. { "collating-element", do_colele },
  299. { "script", do_script },
  300. { "copy", do_copy },
  301. { "order_start", do_order_start },
  302. { "order_end", do_order_end },
  303. { "order-end", do_order_end },
  304. { "reorder-after", do_reorder_after },
  305. { "reorder-end", do_reorder_end },
  306. { "reorder-sections-after", do_reorder_sections_after },
  307. { "reorder-sections-end", do_reorder_sections_end },
  308. { "UCLIBC_RANGE", do_range },
  309. { NULL, do_unrecognized }
  310. };
  311. static void do_unrecognized(void)
  312. {
  313. #if 1
  314. error_msg("warning: unrecognized: %s", pos);
  315. #else
  316. /* fprintf(stderr, "warning: unrecognized initial keyword \"%s\"\n", pos); */
  317. fprintf(stderr, "warning: unrecognized: %s", pos);
  318. if (end_of_token) {
  319. fprintf(stderr, "%c%s", end_of_token, pos_e+1);
  320. }
  321. fprintf(stderr, "\n");
  322. #endif
  323. }
  324. /* typedef struct { */
  325. /* const char *symbol1; */
  326. /* const char *symbol2; */
  327. /* int length; */
  328. /* weight_t *weight; */
  329. /* } range_item_t; */
  330. static void do_range(void)
  331. {
  332. range_item_t *ri;
  333. weight_t w;
  334. int i;
  335. char *s;
  336. char *s1;
  337. char *s2;
  338. const char **ci;
  339. ll_item_t *lli;
  340. assert(!superset);
  341. assert(order_state == IN_ORDER);
  342. s1 = next_token();
  343. if (!s1) {
  344. error_msg("missing start of range");
  345. }
  346. if (!is_ucode(s1)) {
  347. error_msg("start of range is not a ucode: %s", s1);
  348. }
  349. s1 = xsymdup(s1);
  350. s2 = next_token();
  351. if (!s2) {
  352. error_msg("missing end of range");
  353. }
  354. if (!is_ucode(s2)) {
  355. error_msg("end of range is not a ucode: %s", s2);
  356. }
  357. s2 = xsymdup(s2);
  358. ri = (range_item_t *) xmalloc(sizeof(range_item_t));
  359. ri->symbol1 = s1;
  360. ri->symbol2 = s2;
  361. ri->length = strtoul(s2+2, NULL, 16) - strtoul(s1+2, NULL, 16);
  362. if (ri->length <= 0) {
  363. error_msg("illegal range length %d", ri->length);
  364. }
  365. s = next_token();
  366. w.num_weights = cur_num_weights;
  367. for (i=0 ; i < cur_num_weights ; i++) {
  368. w.rule[i] = cur_rule[i];
  369. }
  370. ci = w.colitem + (i-1);
  371. /* now i == cur_num_weights */
  372. #define STR_DITTO "."
  373. while (s && *s && i) {
  374. --i;
  375. if (*s == ';') {
  376. ci[-i] = xsymdup(STR_DITTO);
  377. if (*++s) {
  378. continue;
  379. }
  380. }
  381. if (*s) {
  382. ci[-i] = xsymdup(s);
  383. }
  384. s = next_token();
  385. if (s) {
  386. if (*s == ';') {
  387. ++s;
  388. } else if (i) {
  389. error_msg("missing seperator");
  390. }
  391. }
  392. }
  393. if (s) {
  394. error_msg("too many weights: %d %d |%s| %d", cur_num_weights, i, s, (int)*s);
  395. }
  396. while (i) { /* missing weights are not an error */
  397. --i;
  398. ci[-i] = xsymdup(STR_DITTO);
  399. }
  400. ri->weight = register_weight(&w);
  401. /* if ((i = is_ucode(t)) != 0) { */
  402. /* assert(!t[i]); */
  403. /* add_colitem(t, NULL); */
  404. /* } */
  405. lli = new_ll_item(DT_RANGE, ri);
  406. if (!cur_section->itm_list) {
  407. /* printf("creating new item list: %s\n", wi->symbol); */
  408. cur_section->itm_list = lli;
  409. lli->prev = lli->next = lli;
  410. ++cur_section->num_items;
  411. } else {
  412. insque(lli, cur_section->itm_list->prev);
  413. /* printf("adding item to list: %d - %s\n", ll_len(cur_section->itm_list), wi->symbol); */
  414. ++cur_section->num_items;
  415. }
  416. /* add_wi_index(lli); */
  417. }
  418. static weighted_item_t *add_weight(char *t)
  419. {
  420. weighted_item_t *wi;
  421. weight_t w;
  422. int i;
  423. char *s;
  424. const char **ci;
  425. t = xsymdup(t);
  426. s = next_token();
  427. w.num_weights = cur_num_weights;
  428. for (i=0 ; i < cur_num_weights ; i++) {
  429. w.rule[i] = cur_rule[i];
  430. }
  431. ci = w.colitem + (i-1);
  432. /* now i == cur_num_weights */
  433. while (s && *s && i) {
  434. --i;
  435. if (*s == ';') {
  436. ci[-i] = xsymdup(STR_DITTO);
  437. if (*++s) {
  438. continue;
  439. }
  440. }
  441. if (*s) {
  442. if (!strcmp(s,t)) {
  443. s = STR_DITTO;
  444. }
  445. ci[-i] = xsymdup(s);
  446. }
  447. s = next_token();
  448. if (s) {
  449. if (*s == ';') {
  450. ++s;
  451. } else if (i) {
  452. error_msg("missing seperator");
  453. }
  454. }
  455. }
  456. if (s) {
  457. error_msg("too many weights: %d %d |%s| %d", cur_num_weights, i, s, (int)*s);
  458. }
  459. while (i) { /* missing weights are not an error */
  460. --i;
  461. ci[-i] = xsymdup(STR_DITTO);
  462. }
  463. wi = xmalloc(sizeof(weighted_item_t));
  464. wi->symbol = t;
  465. wi->weight = register_weight(&w);
  466. if ((i = is_ucode(t)) != 0) {
  467. assert(!t[i]);
  468. add_colitem(t, NULL);
  469. }
  470. return wi;
  471. }
  472. static void add_superset_weight(char *t)
  473. {
  474. ll_item_t *lli;
  475. weighted_item_t *wi;
  476. if (!comm_cur_ptr
  477. || (strcmp(t, ((weighted_item_t *)(comm_cur_ptr->data))->symbol) != 0)
  478. ) { /* now out of sync */
  479. if (superset_in_sync) { /* need a new section */
  480. superset_in_sync = 0;
  481. cur_section = new_section("R");
  482. cur_num_weights = cur_section->num_rules
  483. = ((section_t *)(cur_base->section_list->data))->num_rules;
  484. memcpy(cur_rule,
  485. ((section_t *)(cur_base->section_list->data))->rules,
  486. MAX_COLLATION_WEIGHTS);
  487. memcpy(cur_section->rules,
  488. ((section_t *)(cur_base->section_list->data))->rules,
  489. MAX_COLLATION_WEIGHTS);
  490. insque(new_ll_item(DT_REORDER, cur_section), find_ll_last(cur_col->section_list));
  491. assert(comm_prev_ptr);
  492. lli = new_ll_item(DT_REORDER, cur_section);
  493. lli->prev = lli->next = lli;
  494. insque(lli, comm_prev_ptr);
  495. /* fprintf(stderr, " subsection -----------------------\n"); */
  496. }
  497. /* fprintf(stderr, " %s %s\n", t, ((weighted_item_t *)(comm_cur_ptr->data))->symbol); */
  498. wi = add_weight(t);
  499. lli = new_ll_item(DT_WEIGHTED, wi);
  500. mark_reordered(wi->symbol);
  501. /* printf("reorder: %s\n", t); */
  502. if (!cur_section->itm_list) {
  503. cur_section->itm_list = lli;
  504. lli->prev = lli->next = lli;
  505. ++cur_section->num_items;
  506. } else {
  507. insque(lli, cur_section->itm_list->prev);
  508. ++cur_section->num_items;
  509. }
  510. add_wi_index(lli);
  511. } else { /* in sync */
  512. superset_in_sync = 1;
  513. next_comm_ptr();
  514. }
  515. }
  516. static void do_weight(char *t)
  517. {
  518. weighted_item_t *wi;
  519. ll_item_t *lli;
  520. if (superset) {
  521. add_superset_weight(t);
  522. return;
  523. }
  524. switch(order_state) {
  525. case 0:
  526. /* fprintf(stdout, "no-order weight: %s\n", t); */
  527. /* break; */
  528. case IN_ORDER:
  529. /* in a section */
  530. /* fprintf(stdout, "weight: %s\n", t); */
  531. wi = add_weight(t);
  532. lli = new_ll_item(DT_WEIGHTED, wi);
  533. if (!cur_section->itm_list) {
  534. /* fprintf(stdout, "creating new item list: %s %s %p\n", wi->symbol, cur_section->name, lli); */
  535. cur_section->itm_list = lli;
  536. lli->prev = lli->next = lli;
  537. ++cur_section->num_items;
  538. } else {
  539. insque(lli, cur_section->itm_list->prev);
  540. /* fprintf(stdout, "adding item to list: %d - %s %p\n", ll_len(cur_section->itm_list), wi->symbol, lli); */
  541. ++cur_section->num_items;
  542. }
  543. add_wi_index(lli);
  544. break;
  545. case IN_REORDER:
  546. /* std rule - but in a block with an insert-after pt */
  547. wi = add_weight(t);
  548. lli = new_ll_item(DT_WEIGHTED, wi);
  549. mark_reordered(wi->symbol);
  550. /* fprintf(stdout, "reorder: %s %s %p\n", t, cur_section->name, lli); */
  551. if (!cur_section->itm_list) {
  552. cur_section->itm_list = lli;
  553. lli->prev = lli->next = lli;
  554. ++cur_section->num_items;
  555. } else {
  556. insque(lli, cur_section->itm_list->prev);
  557. ++cur_section->num_items;
  558. }
  559. add_wi_index(lli);
  560. break;
  561. case IN_REORDER_SECTIONS:
  562. t = xsymdup(t);
  563. if (next_token() != NULL) {
  564. error_msg("trailing text in reorder section item: %s", pos);
  565. }
  566. lli = cur_col->section_list;
  567. do {
  568. if (lli->data_type & DT_SECTION) {
  569. if (!strcmp(((section_t *)(lli->data))->name, t)) {
  570. lli->data_type = DT_REORDER;
  571. lli = new_ll_item(DT_REORDER, (section_t *)(lli->data));
  572. insque(lli, reorder_section_ptr);
  573. reorder_section_ptr = lli;
  574. return;
  575. }
  576. }
  577. lli = lli->next;
  578. } while (lli);
  579. error_msg("reorder_sections_after for non-base item currently not supported: %s", t);
  580. /* fprintf(stdout, "reorder_secitons: %s\n", t); */
  581. break;
  582. default:
  583. error_msg("invalid order_state %d", order_state);
  584. }
  585. }
  586. static int col_locale_cmp(const void *n1, const void *n2)
  587. {
  588. return strcmp(((const col_locale_t *) n1)->name, ((const col_locale_t *) n2)->name);
  589. }
  590. static void processfile(void)
  591. {
  592. char *t;
  593. const keyword_table_t *k;
  594. order_state = 0;
  595. #warning devel code
  596. /* cur_num_weights = 0; */
  597. /* cur_num_weights = 4; */
  598. /* memset(cur_rule, R_FORWARD, 4); */
  599. if (cur_col != cur_base) {
  600. cur_col->base_locale = cur_base;
  601. cur_col->undefined_idx = cur_base->undefined_idx;
  602. if (!cur_base->derived_list) {
  603. cur_base->derived_list = new_ll_item(DT_COL_LOCALE, cur_col);
  604. } else {
  605. insque(new_ll_item(DT_COL_LOCALE, cur_col), find_ll_last(cur_base->derived_list));
  606. }
  607. }
  608. if (tfind(cur_col, &root_col_locale, col_locale_cmp)) {
  609. error_msg("attempt to read locale: %s", cur_col->name);
  610. }
  611. if (!tsearch(cur_col, &root_col_locale, col_locale_cmp)) {
  612. error_msg("OUT OF MEMORY!");
  613. }
  614. if (superset) {
  615. superset_order_start_cnt = 0;
  616. superset_in_sync = 0;
  617. init_comm_ptr();
  618. }
  619. while (next_line()) {
  620. /* printf("%5d:", lineno[fno]); */
  621. /* while ((t = next_token()) != NULL) { */
  622. /* printf(" |%s|", t); */
  623. /* printf("\n"); */
  624. /* } */
  625. t = next_token();
  626. assert(t);
  627. assert(t == pos);
  628. if ((*t == '<') || (!strcmp(t, "UNDEFINED"))) {
  629. do_weight(t);
  630. } else {
  631. for (k = keyword_table ; k->keyword ; k++) {
  632. if (!strcmp(k->keyword, t)) {
  633. break;
  634. }
  635. }
  636. k->handler();
  637. }
  638. }
  639. if (cur_base == cur_col) {
  640. fprintf(stderr, "Base: %15s", cur_col->name);
  641. } else {
  642. #if 1
  643. if (!cur_col->undefined_idx) {
  644. #if 0
  645. if (superset) {
  646. if (superset_order_start_cnt == 1) {
  647. --superset_order_start_cnt; /* ugh.. hack this */
  648. }
  649. }
  650. #endif
  651. /* This is an awful hack to get around the problem of unspecified UNDEFINED
  652. * definitions in the supported locales derived from iso14651_t1. */
  653. if (!strcmp(cur_base->name, "iso14651_t1")) {
  654. fprintf(stderr, "Warning: adding UNDEFINED entry for %s\n", cur_col->name);
  655. strcpy(linebuf, "script <UNDEFINED_SECTION>\n");
  656. pos_e = NULL;
  657. pos = linebuf;
  658. t = next_token();
  659. assert(t);
  660. assert(t == pos);
  661. do_script();
  662. strcpy(linebuf, "order_start <UNDEFINED_SECTION>;forward;backward;forward;forward,position\n");
  663. pos_e = NULL;
  664. pos = linebuf;
  665. t = next_token();
  666. assert(t);
  667. assert(t == pos);
  668. do_order_start();
  669. strcpy(linebuf, "UNDEFINED IGNORE;IGNORE;IGNORE\n");
  670. pos_e = NULL;
  671. pos = linebuf;
  672. t = next_token();
  673. assert(t);
  674. assert(t == pos);
  675. do_weight(t);
  676. strcpy(linebuf, "order_end\n");
  677. pos_e = NULL;
  678. pos = linebuf;
  679. t = next_token();
  680. assert(t);
  681. assert(t == pos);
  682. do_order_end();
  683. } else {
  684. error_msg("no definition of UNDEFINED for %s", cur_col->name);
  685. }
  686. }
  687. #endif
  688. fprintf(stderr, " Der: %15s", cur_col->name);
  689. }
  690. {
  691. ll_item_t *p = cur_col->section_list;
  692. fprintf(stderr, "%6u weights", tnumnodes(cur_col->root_wi_index));
  693. if (cur_base) {
  694. fprintf(stderr, " %6u der %6u reor %6u starter - %u new stubs",
  695. tnumnodes(cur_base->root_derived_wi),
  696. tnumnodes(cur_base->root_wi_index_reordered),
  697. tnumnodes(cur_base->root_starter_char),
  698. ll_count(cur_col->section_list, DT_REORDER));
  699. }
  700. fprintf(stderr, "\n");
  701. #if 0
  702. while (p) {
  703. assert(((section_t *)(p->data))->num_items ==
  704. ll_len(((section_t *)(p->data))->itm_list));
  705. if (!p->next &&
  706. ((*((section_t *)(p->data))->name == 'a')
  707. && (((section_t *)(p->data))->num_items == 0))
  708. ) {
  709. break;
  710. }
  711. if (!(p->data_type & DT_REORDER)) {
  712. if ((*((section_t *)(p->data))->name != 'a')
  713. || (((section_t *)(p->data))->num_items > 0)
  714. ) {
  715. fprintf(stderr,
  716. /* "\t%-15s %zu\n", */
  717. "\t%-15s %6u\n",
  718. ((section_t *)(p->data))->name,
  719. ((section_t *)(p->data))->num_items);
  720. }
  721. }
  722. p = p->next;
  723. }
  724. #endif
  725. }
  726. }
  727. static void print_colnode(const void *ptr, VISIT order, int level)
  728. {
  729. const colitem_t *p = *(const colitem_t **) ptr;
  730. if (order == postorder || order == leaf) {
  731. printf("collating item = \"%s\"", p->string);
  732. if (p->element) {
  733. printf(" is %s", p->element);
  734. }
  735. printf("\n");
  736. }
  737. }
  738. static void print_weight_node(const void *ptr, VISIT order, int level)
  739. {
  740. const weight_t *p = *(const weight_t **) ptr;
  741. int i;
  742. if (order == postorder || order == leaf) {
  743. printf("weight: (%d) ", p->num_weights);
  744. for (i = 0 ; i < p->num_weights ; i++) {
  745. if (p->rule[i] & R_FORWARD) {
  746. printf("F");
  747. }
  748. if (p->rule[i] & R_BACKWARD) {
  749. printf("B");
  750. }
  751. if (p->rule[i] & R_POSITION) {
  752. printf("P");
  753. }
  754. printf(",");
  755. }
  756. for (i = 0 ; i < p->num_weights ; i++) {
  757. printf(" %s", p->colitem[i]);
  758. }
  759. printf("\n");
  760. }
  761. }
  762. typedef struct {
  763. const char *der_name;
  764. int base_locale;
  765. } deps_t;
  766. enum {
  767. BASE_iso14651_t1,
  768. BASE_comm,
  769. BASE_cs_CZ,
  770. BASE_ar_SA,
  771. BASE_th_TH,
  772. BASE_ja_JP,
  773. BASE_ko_KR,
  774. BASE_MAX
  775. };
  776. static const char *base_name[] = {
  777. "iso14651_t1",
  778. "comm",
  779. "cs_CZ",
  780. "ar_SA",
  781. "th_TH",
  782. "ja_JP",
  783. "ko_KR"
  784. };
  785. static ll_item_t *locale_list[BASE_MAX];
  786. static void init_locale_list(void)
  787. {
  788. int i;
  789. for (i=0 ; i < BASE_MAX ; i++) {
  790. locale_list[i] = (ll_item_t *) xmalloc(sizeof(ll_item_t));
  791. locale_list[i]->prev = locale_list[i]->next = locale_list[i];
  792. locale_list[i]->data = (void *) base_name[i];
  793. }
  794. }
  795. deps_t deps[] = {
  796. { "af_ZA", BASE_iso14651_t1 },
  797. { "am_ET", BASE_iso14651_t1 },
  798. { "ar_AE", BASE_iso14651_t1 },
  799. { "ar_BH", BASE_iso14651_t1 },
  800. { "ar_DZ", BASE_iso14651_t1 },
  801. { "ar_EG", BASE_iso14651_t1 },
  802. { "ar_IN", BASE_iso14651_t1 },
  803. { "ar_IQ", BASE_iso14651_t1 },
  804. { "ar_JO", BASE_iso14651_t1 },
  805. { "ar_KW", BASE_iso14651_t1 },
  806. { "ar_LB", BASE_iso14651_t1 },
  807. { "ar_LY", BASE_iso14651_t1 },
  808. { "ar_MA", BASE_iso14651_t1 },
  809. { "ar_OM", BASE_iso14651_t1 },
  810. { "ar_QA", BASE_iso14651_t1 },
  811. { "ar_SA", BASE_ar_SA },
  812. { "ar_SD", BASE_iso14651_t1 },
  813. { "ar_SY", BASE_iso14651_t1 },
  814. { "ar_TN", BASE_iso14651_t1 },
  815. { "ar_YE", BASE_iso14651_t1 },
  816. { "az_AZ", BASE_iso14651_t1 },
  817. { "be_BY", BASE_iso14651_t1 },
  818. { "bg_BG", BASE_iso14651_t1 },
  819. { "bn_BD", BASE_iso14651_t1 },
  820. { "bn_IN", BASE_iso14651_t1 },
  821. { "br_FR", BASE_iso14651_t1 },
  822. { "bs_BA", BASE_iso14651_t1 },
  823. { "ca_ES", BASE_comm },
  824. { "cs_CZ", BASE_cs_CZ },
  825. { "cy_GB", BASE_iso14651_t1 },
  826. { "da_DK", BASE_comm },
  827. { "de_AT", BASE_iso14651_t1 },
  828. { "de_BE", BASE_iso14651_t1 },
  829. { "de_CH", BASE_iso14651_t1 },
  830. { "de_DE", BASE_iso14651_t1 },
  831. { "de_LU", BASE_iso14651_t1 },
  832. { "el_GR", BASE_iso14651_t1 },
  833. { "en_AU", BASE_iso14651_t1 },
  834. { "en_BW", BASE_iso14651_t1 },
  835. { "en_CA", BASE_comm },
  836. { "en_DK", BASE_iso14651_t1 },
  837. { "en_GB", BASE_iso14651_t1 },
  838. { "en_HK", BASE_iso14651_t1 },
  839. { "en_IE", BASE_iso14651_t1 },
  840. { "en_IN", BASE_iso14651_t1 },
  841. { "en_NZ", BASE_iso14651_t1 },
  842. { "en_PH", BASE_iso14651_t1 },
  843. { "en_SG", BASE_iso14651_t1 },
  844. { "en_US", BASE_iso14651_t1 },
  845. { "en_ZA", BASE_iso14651_t1 },
  846. { "en_ZW", BASE_iso14651_t1 },
  847. { "eo_EO", BASE_iso14651_t1 },
  848. { "es_AR", BASE_comm },
  849. { "es_BO", BASE_comm },
  850. { "es_CL", BASE_comm },
  851. { "es_CO", BASE_comm },
  852. { "es_CR", BASE_comm },
  853. { "es_DO", BASE_comm },
  854. { "es_EC", BASE_comm },
  855. { "es_ES", BASE_comm },
  856. { "es_GT", BASE_comm },
  857. { "es_HN", BASE_comm },
  858. { "es_MX", BASE_comm },
  859. { "es_NI", BASE_comm },
  860. { "es_PA", BASE_comm },
  861. { "es_PE", BASE_comm },
  862. { "es_PR", BASE_comm },
  863. { "es_PY", BASE_comm },
  864. { "es_SV", BASE_comm },
  865. { "es_US", BASE_comm },
  866. { "es_UY", BASE_comm },
  867. { "es_VE", BASE_comm },
  868. { "et_EE", BASE_comm },
  869. { "eu_ES", BASE_iso14651_t1 },
  870. { "fa_IR", BASE_iso14651_t1 },
  871. { "fi_FI", BASE_comm },
  872. { "fo_FO", BASE_comm },
  873. { "fr_BE", BASE_iso14651_t1 },
  874. { "fr_CA", BASE_comm },
  875. { "fr_CH", BASE_iso14651_t1 },
  876. { "fr_FR", BASE_iso14651_t1 },
  877. { "fr_LU", BASE_iso14651_t1 },
  878. { "ga_IE", BASE_iso14651_t1 },
  879. { "gd_GB", BASE_iso14651_t1 },
  880. { "gl_ES", BASE_comm },
  881. { "gv_GB", BASE_iso14651_t1 },
  882. { "he_IL", BASE_iso14651_t1 },
  883. { "hi_IN", BASE_iso14651_t1 },
  884. { "hr_HR", BASE_comm },
  885. { "hu_HU", BASE_iso14651_t1 },
  886. { "hy_AM", BASE_iso14651_t1 },
  887. { "id_ID", BASE_iso14651_t1 },
  888. { "is_IS", BASE_comm },
  889. { "it_CH", BASE_iso14651_t1 },
  890. { "it_IT", BASE_iso14651_t1 },
  891. { "iw_IL", BASE_iso14651_t1 },
  892. { "ja_JP", BASE_ja_JP },
  893. { "ka_GE", BASE_iso14651_t1 },
  894. { "kl_GL", BASE_comm },
  895. { "ko_KR", BASE_ko_KR },
  896. { "kw_GB", BASE_iso14651_t1 },
  897. { "lt_LT", BASE_comm },
  898. { "lv_LV", BASE_comm },
  899. { "mi_NZ", BASE_iso14651_t1 },
  900. { "mk_MK", BASE_iso14651_t1 },
  901. { "mr_IN", BASE_iso14651_t1 },
  902. { "ms_MY", BASE_iso14651_t1 },
  903. { "mt_MT", BASE_iso14651_t1 },
  904. { "nl_BE", BASE_iso14651_t1 },
  905. { "nl_NL", BASE_iso14651_t1 },
  906. { "nn_NO", BASE_iso14651_t1 },
  907. { "no_NO", BASE_comm },
  908. { "oc_FR", BASE_iso14651_t1 },
  909. { "pl_PL", BASE_comm },
  910. { "pt_BR", BASE_iso14651_t1 },
  911. { "pt_PT", BASE_iso14651_t1 },
  912. { "ro_RO", BASE_iso14651_t1 },
  913. { "ru_RU", BASE_iso14651_t1 },
  914. { "ru_UA", BASE_iso14651_t1 },
  915. { "se_NO", BASE_iso14651_t1 },
  916. { "sk_SK", BASE_cs_CZ },
  917. { "sl_SI", BASE_comm },
  918. { "sq_AL", BASE_iso14651_t1 },
  919. { "sr_YU", BASE_iso14651_t1 },
  920. { "sv_FI", BASE_comm },
  921. { "sv_SE", BASE_iso14651_t1 },
  922. { "ta_IN", BASE_iso14651_t1 },
  923. { "te_IN", BASE_iso14651_t1 },
  924. { "tg_TJ", BASE_iso14651_t1 },
  925. { "th_TH", BASE_th_TH },
  926. { "ti_ER", BASE_iso14651_t1 },
  927. { "ti_ET", BASE_iso14651_t1 },
  928. { "tl_PH", BASE_iso14651_t1 },
  929. { "tr_TR", BASE_comm },
  930. { "tt_RU", BASE_iso14651_t1 },
  931. { "uk_UA", BASE_iso14651_t1 },
  932. { "ur_PK", BASE_iso14651_t1 },
  933. { "uz_UZ", BASE_iso14651_t1 },
  934. { "vi_VN", BASE_iso14651_t1 },
  935. { "wa_BE", BASE_iso14651_t1 },
  936. { "yi_US", BASE_iso14651_t1 },
  937. { "zh_CN", BASE_iso14651_t1 },
  938. { "zh_HK", BASE_iso14651_t1 },
  939. { "zh_SG", BASE_iso14651_t1 },
  940. { "zh_TW", BASE_iso14651_t1 },
  941. };
  942. static int der_count[BASE_MAX];
  943. static const char *new_args[500];
  944. static int new_arg_count;
  945. static int dep_cmp(const void *s1, const void *s2)
  946. {
  947. return strcmp( (const char *) s1, ((const deps_t *) s2)->der_name);
  948. }
  949. static int old_main(int argc, char **argv);
  950. int main(int argc, char **argv)
  951. {
  952. const deps_t *p;
  953. ll_item_t *lli;
  954. int i;
  955. int total;
  956. if (argc < 2) {
  957. return EXIT_FAILURE;
  958. }
  959. init_locale_list();
  960. while (--argc) {
  961. p = (const deps_t *) bsearch(*++argv, deps, sizeof(deps)/sizeof(deps[0]), sizeof(deps[0]), dep_cmp);
  962. if (!p) {
  963. if (!strcmp("C", *argv)) {
  964. printf("ignoring C locale\n");
  965. continue;
  966. } else {
  967. printf("%s not found\n", *argv);
  968. return EXIT_FAILURE;
  969. }
  970. }
  971. i = p->base_locale;
  972. ++der_count[i];
  973. if (!strcmp(base_name[i], *argv)) {
  974. /* same name as base, so skip after count incremented */
  975. continue;
  976. }
  977. /* add it to the list. the main body will catch duplicates */
  978. lli = (ll_item_t *) xmalloc(sizeof(ll_item_t));
  979. lli->prev = lli->next = NULL;
  980. lli->data = (void *) *argv;
  981. insque(lli, locale_list[i]);
  982. }
  983. total = 0;
  984. for (i=0 ; i < BASE_MAX ; i++) {
  985. /* printf("der_count[%2d] = %3d\n", i, der_count[i]); */
  986. total += der_count[i];
  987. }
  988. /* printf("total = %d\n", total); */
  989. new_args[new_arg_count++] = "dummyprogramname";
  990. for (i=0 ; i < BASE_MAX ; i++) {
  991. if (!der_count[i]) {
  992. continue;
  993. }
  994. new_args[new_arg_count++] = (i == BASE_comm) ? "-c" : "-b";
  995. lli = locale_list[i];
  996. do {
  997. new_args[new_arg_count++] = (const char *) (lli->data);
  998. lli = lli->next;
  999. } while (lli != locale_list[i]);
  1000. new_args[new_arg_count++] = "-f";
  1001. }
  1002. /* for (i=0 ; i < new_arg_count ; i++) { */
  1003. /* printf("%3d: %s\n", i, new_args[i]); */
  1004. /* } */
  1005. return old_main(new_arg_count, (char **) new_args);
  1006. }
  1007. /* usage... prog -b basefile derived {derived} -s single {single} */
  1008. static int old_main(int argc, char **argv)
  1009. {
  1010. int next_is_base = 0;
  1011. int next_is_subset = 0;
  1012. superset = 0;
  1013. while (--argc) {
  1014. ++argv;
  1015. if (**argv == '-') {
  1016. if ((*argv)[1] == 'd') {
  1017. dump_weights((*argv) + 2);
  1018. } else if ((*argv)[1] == 'f') { /* dump all weight rules */
  1019. finalize_base();
  1020. } else if ((*argv)[1] == 'R') { /* dump all weight rules */
  1021. twalk(root_weight, print_weight_node);
  1022. } else if (((*argv)[1] == 'c') && !(*argv)[2]) { /* new common subset */
  1023. cur_base = cur_derived = NULL;
  1024. next_is_subset = 1;
  1025. next_is_base = 1;
  1026. superset = 0;
  1027. } else if (((*argv)[1] == 'b') && !(*argv)[2]) { /* new base locale */
  1028. cur_base = cur_derived = NULL;
  1029. next_is_subset = 0;
  1030. next_is_base = 1;
  1031. superset = 0;
  1032. } else if (((*argv)[1] == 's') && !(*argv)[2]) { /* single locales follow */
  1033. cur_base = cur_derived = NULL;
  1034. next_is_subset = 0;
  1035. next_is_base = 2;
  1036. superset = 0;
  1037. } else {
  1038. error_msg("unrecognized option %s", *argv);
  1039. }
  1040. continue;
  1041. }
  1042. /* new file */
  1043. new_col_locale(*argv); /* automaticly sets cur_col */
  1044. if (next_is_base) {
  1045. cur_base = cur_col;
  1046. } else {
  1047. cur_derived = cur_col;
  1048. }
  1049. pushfile(*argv);
  1050. /* fprintf(stderr, "processing file %s\n", *argv); */
  1051. processfile(); /* this does a popfile */
  1052. /* twalk(cur_col->root_colitem, print_colnode); */
  1053. if (next_is_base == 1) {
  1054. next_is_base = 0;
  1055. }
  1056. if (next_is_subset) {
  1057. next_is_subset = 0;
  1058. superset = 1;
  1059. }
  1060. }
  1061. fprintf(stderr, "success!\n");
  1062. fprintf(stderr,
  1063. /* "num_sym=%zu mem_sym=%zu unique_weights=%zu\n", */
  1064. "num_sym=%u mem_sym=%u unique_weights=%u\n",
  1065. num_sym, mem_sym, unique_weights);
  1066. /* twalk(root_weight, print_weight_node); */
  1067. fprintf(stderr, "num base locales = %d num derived locales = %d\n",
  1068. base_locale_len, der_locale_len);
  1069. fprintf(stderr,
  1070. "override_len = %d multistart_len = %d weightstr_len = %d\n"
  1071. "wcs2colidt_len = %d index2weight_len = %d index2ruleidx_len = %d\n"
  1072. "ruletable_len = %d\n"
  1073. "total size is %d bytes or %d kB\n",
  1074. override_len, multistart_len, weightstr_len,
  1075. wcs2colidt_len, index2weight_len, index2ruleidx_len,
  1076. ruletable_len,
  1077. #warning mult by 2 for rule indecies
  1078. (override_len + multistart_len + weightstr_len
  1079. + wcs2colidt_len + index2weight_len + index2ruleidx_len + ruletable_len) * 2,
  1080. (override_len + multistart_len + weightstr_len
  1081. + wcs2colidt_len + index2weight_len + index2ruleidx_len + ruletable_len + 511) / 512);
  1082. #if 0
  1083. {
  1084. int i;
  1085. for (i=0 ; i < base_locale_len ; i++) {
  1086. dump_base_locale(i);
  1087. }
  1088. for (i=0 ; i < der_locale_len ; i++) {
  1089. dump_der_locale(i);
  1090. }
  1091. }
  1092. #endif
  1093. {
  1094. FILE *fp = fopen("locale_collate.h", "w");
  1095. if (!fp) {
  1096. error_msg("couldn't open output file!");
  1097. }
  1098. dump_collate(fp);
  1099. if (ferror(fp) || fclose(fp)) {
  1100. error_msg("write error or close error for output file!\n");
  1101. }
  1102. }
  1103. return EXIT_SUCCESS;
  1104. }
  1105. static void error_msg(const char *fmt, ...)
  1106. {
  1107. va_list arg;
  1108. fprintf(stderr, "Error: ");
  1109. if (fno >= 0) {
  1110. fprintf(stderr, "file %s (%d): ", fname[fno], lineno[fno]);
  1111. }
  1112. va_start(arg, fmt);
  1113. vfprintf(stderr, fmt, arg);
  1114. va_end(arg);
  1115. fprintf(stderr, "\n");
  1116. exit(EXIT_FAILURE);
  1117. }
  1118. static void pushfile(char *filename)
  1119. {
  1120. static char fbuf[PATH_MAX];
  1121. snprintf(fbuf, PATH_MAX, "collation/%s", filename);
  1122. if (fno >= MAX_FNO) {
  1123. error_msg("file stack size exceeded");
  1124. }
  1125. if (!(fstack[++fno] = fopen(fbuf, "r"))) {
  1126. --fno; /* oops */
  1127. error_msg("cannot open file %s", fbuf);
  1128. }
  1129. fname[fno] = xsymdup(filename);
  1130. lineno[fno] = 0;
  1131. }
  1132. static void popfile(void)
  1133. {
  1134. if (fno < 0) {
  1135. error_msg("pop on empty file stack");
  1136. }
  1137. /* free(fname[fno]); */
  1138. fclose(fstack[fno]);
  1139. --fno;
  1140. }
  1141. static void eatwhitespace(void)
  1142. {
  1143. while (isspace(*pos)) {
  1144. ++pos;
  1145. }
  1146. }
  1147. static int iscommentchar(int c)
  1148. {
  1149. return ((c == '#') || (c == '%'));
  1150. }
  1151. static int next_line(void)
  1152. {
  1153. size_t n;
  1154. char *s = linebuf;
  1155. assert(fno >= 0);
  1156. pos_e = NULL;
  1157. do {
  1158. if (fgets(s, sizeof(linebuf), fstack[fno]) != NULL) {
  1159. ++lineno[fno];
  1160. n = strlen(linebuf);
  1161. if ((n == sizeof(linebuf) - 1) && (linebuf[n-1] != '\n')) {
  1162. /* Either line is too long or last line is very long with
  1163. * no trailing newline. But we'll always treat it as an
  1164. * errro. */
  1165. error_msg("line too long?");
  1166. }
  1167. --n;
  1168. /* Be careful... last line doesn't need a newline. */
  1169. if (linebuf[n] == '\n') {
  1170. linebuf[n--] = 0; /* trim trailing newline */
  1171. }
  1172. pos = linebuf;
  1173. eatwhitespace();
  1174. if (*pos && !iscommentchar(*pos)) { /* not empty or comment line */
  1175. return 1; /* got a line */
  1176. }
  1177. } else { /* eof */
  1178. popfile();
  1179. }
  1180. } while (fno >= 0);
  1181. return 0;
  1182. }
  1183. static char *next_token(void)
  1184. {
  1185. char *p;
  1186. #if 0
  1187. if (pos_e == NULL) {
  1188. return NULL
  1189. pos = pos_e;
  1190. *pos = end_of_token;
  1191. end_of_token = 0;
  1192. }
  1193. #else
  1194. if (pos_e != NULL) {
  1195. pos = pos_e;
  1196. *pos = end_of_token;
  1197. end_of_token = 0;
  1198. }
  1199. #endif
  1200. eatwhitespace();
  1201. p = pos;
  1202. if (!*p || iscommentchar(*p)) { /* end of line or start of comment */
  1203. pos = pos_e = NULL;
  1204. *p = 0; /* treat comment as end of line */
  1205. /* fprintf(stdout, "returning NUL token |%s|\n", pos); */
  1206. return NULL;
  1207. #if 1
  1208. } else if (*p == '<') { /* collating symbol, element, or value */
  1209. while (*++p) {
  1210. if ((*p == '/') && p[1]) {
  1211. ++p;
  1212. continue;
  1213. }
  1214. if (*p == '>') {
  1215. pos_e = ++p;
  1216. end_of_token = *p;
  1217. *p = 0;
  1218. /* fprintf(stdout, "returning col token |%s|\n", pos); */
  1219. return pos;
  1220. }
  1221. }
  1222. } else if (*p == '"') { /* collating element value? */
  1223. while (*++p) {
  1224. if (*p == '"') { /* found the end of the quoted string */
  1225. pos_e = ++p;
  1226. end_of_token = *p;
  1227. *p = 0;
  1228. /* fprintf(stdout, "returning quote token |%s|\n", pos); */
  1229. return pos;
  1230. }
  1231. }
  1232. #endif
  1233. } else { /* some kind of keyword */
  1234. while (*++p) {
  1235. if (isspace(*p) || (*p == ';')) {
  1236. break;
  1237. }
  1238. }
  1239. pos_e = p;
  1240. end_of_token = *p;
  1241. *p = 0;
  1242. /* fprintf(stdout, "returning key token |%s|\n", pos); */
  1243. return pos;
  1244. }
  1245. error_msg("illegal token |%s|", pos);
  1246. }
  1247. static void *xmalloc(size_t n)
  1248. {
  1249. void *p;
  1250. if (!(p = malloc(n))) {
  1251. error_msg("OUT OF MEMORY");
  1252. }
  1253. return p;
  1254. }
  1255. static void do_copy(void)
  1256. {
  1257. char *s;
  1258. char *e;
  1259. if ((s = next_token()) != NULL) {
  1260. e = strchr(s + 1, '"');
  1261. if ((*s == '"') && e && (*e == '"') && !e[1]) {
  1262. if (next_token() != NULL) {
  1263. error_msg("illegal trailing text: %s", pos);
  1264. }
  1265. *e = 0;
  1266. ++s;
  1267. if (cur_base && !strcmp(cur_base->name,s)) {
  1268. /* fprintf(stderr, "skipping copy of base file %s\n", s); */
  1269. #warning need to update last in order and position or check
  1270. return;
  1271. }
  1272. /* fprintf(stderr, "full copy of %s\n", s); */
  1273. pushfile(s);
  1274. return;
  1275. }
  1276. }
  1277. error_msg("illegal or missing arg for copy: %s", s);
  1278. }
  1279. static void do_colsym(void)
  1280. {
  1281. char *s;
  1282. char *e;
  1283. if ((s = next_token()) != NULL) {
  1284. e = strrchr(s,'>');
  1285. if ((*s == '<') && e && (*e == '>') && !e[1]) {
  1286. if (next_token() != NULL) {
  1287. error_msg("illegal trailing text: %s", pos);
  1288. }
  1289. e[1] = 0; /* cleanup in case next_token stored something */
  1290. add_colitem(s,NULL);
  1291. return;
  1292. }
  1293. }
  1294. error_msg("illegal or missing arg for collating-symbol: %s", s);
  1295. }
  1296. static void do_colele(void)
  1297. {
  1298. char *s;
  1299. char *e;
  1300. char *s1;
  1301. char *e1;
  1302. int n;
  1303. if ((s = next_token()) != NULL) {
  1304. e = strrchr(s,'>');
  1305. if ((*s == '<') && e && (*e == '>') && !e[1]) {
  1306. if (((s1 = next_token()) == NULL)
  1307. || (strcmp(s1,"from") != 0)
  1308. || ((s1 = next_token()) == NULL)
  1309. || (*s1 != '\"')
  1310. ) {
  1311. error_msg("illegal format for collating-element spec");
  1312. }
  1313. e1 = strchr(s1 + 1, '"');
  1314. if ((*s1 != '"') || !e1 || (*e1 != '"') || (e1[1] != 0)) {
  1315. error_msg("illegal definition for collating-element: %s", s1);
  1316. }
  1317. if (next_token() != NULL) {
  1318. error_msg("illegal trailing text: %s", pos);
  1319. }
  1320. e[1] = 0; /* cleanup in case next_token stored something */
  1321. e1[1] = 0;
  1322. add_colitem(s,s1);
  1323. ++s1;
  1324. if (!(n = is_ucode(s1))) {
  1325. error_msg("starting char must be a <U####> code: %s", s1);
  1326. }
  1327. assert(s1[n] == '<');
  1328. s1[n] = 0;
  1329. s = xsymdup(s1);
  1330. if (!(tsearch(s, &cur_base->root_starter_char, sym_cmp))) {
  1331. error_msg("OUT OF MEMORY");
  1332. }
  1333. return;
  1334. }
  1335. }
  1336. error_msg("illegal or missing arg for collating-element: %s", s);
  1337. }
  1338. static ll_item_t *find_section_list_item(const char *name, col_locale_t *loc)
  1339. {
  1340. ll_item_t *p;
  1341. if (!loc) {
  1342. return NULL;
  1343. }
  1344. p = loc->section_list;
  1345. while (p) {
  1346. #warning devel code
  1347. /* if (!((p->data_type == DT_SECTION) || (p->data_type == DT_REORDER))) { */
  1348. /* fprintf(stderr, "fsli = %d\n", p->data_type); */
  1349. /* } */
  1350. assert((p->data_type == DT_SECTION) || (p->data_type == DT_REORDER));
  1351. if (!strcmp(name, ((section_t *)(p->data))->name)) {
  1352. break;
  1353. }
  1354. p = p->next;
  1355. }
  1356. return p;
  1357. }
  1358. static ll_item_t *find_ll_last(ll_item_t *p)
  1359. {
  1360. assert(p);
  1361. while (p->next) {
  1362. p = p->next;
  1363. }
  1364. return p;
  1365. }
  1366. static void do_script(void)
  1367. {
  1368. char *s;
  1369. char *e;
  1370. if ((s = next_token()) != NULL) {
  1371. e = strrchr(s,'>');
  1372. if ((*s == '<') && e && (*e == '>') && !e[1]) {
  1373. if (next_token() != NULL) {
  1374. error_msg("illegal trailing text: %s", pos);
  1375. }
  1376. e[1] = 0; /* cleanup in case next_token stored something */
  1377. add_script(s);
  1378. return;
  1379. }
  1380. }
  1381. error_msg("illegal or missing arg for script: %s", s);
  1382. }
  1383. static col_locale_t *new_col_locale(char *name)
  1384. {
  1385. ll_item_t *lli;
  1386. ll_item_t *lli2;
  1387. cur_col = (col_locale_t *) xmalloc(sizeof(col_locale_t));
  1388. cur_col->name = name;
  1389. cur_col->root_colitem = NULL;
  1390. cur_col->root_element = NULL;
  1391. cur_col->root_scripts = NULL;
  1392. cur_col->base_locale = NULL;
  1393. if (!superset) {
  1394. /* start with an anonymous section */
  1395. cur_section = new_section(NULL);
  1396. cur_col->section_list = new_ll_item(DT_SECTION, cur_section);
  1397. } else {
  1398. /* start with a reorder section */
  1399. cur_section = new_section("R");
  1400. cur_num_weights = cur_section->num_rules
  1401. = ((section_t *)(cur_base->section_list->data))->num_rules;
  1402. memcpy(cur_rule,
  1403. ((section_t *)(cur_base->section_list->data))->rules,
  1404. MAX_COLLATION_WEIGHTS);
  1405. memcpy(cur_section->rules,
  1406. ((section_t *)(cur_base->section_list->data))->rules,
  1407. MAX_COLLATION_WEIGHTS);
  1408. cur_col->section_list = new_ll_item(DT_REORDER, cur_section);
  1409. assert(cur_base->section_list->next == NULL); /* currently only one section allowed */
  1410. lli = ((section_t *)(cur_base->section_list->data))->itm_list;
  1411. assert(lli);
  1412. lli2 = new_ll_item(DT_REORDER, cur_section);
  1413. lli2->prev = lli2->next = lli2;
  1414. insque(lli2, lli->prev);
  1415. ((section_t *)(cur_base->section_list->data))->itm_list = lli2;
  1416. }
  1417. /* cur_col->section_list = NULL; */
  1418. /* add_script(((section_t *)(cur_col->section_list->data))->name); */
  1419. cur_col->root_wi_index = NULL;
  1420. cur_col->root_wi_index_reordered = NULL;
  1421. cur_col->root_derived_wi = NULL;
  1422. cur_col->derived_list = NULL;
  1423. cur_col->root_starter_char = NULL;
  1424. cur_col->root_starter_all = NULL;
  1425. cur_col->undefined_idx = NULL;
  1426. return cur_col;
  1427. }
  1428. static int colitem_cmp(const void *n1, const void *n2)
  1429. {
  1430. return strcmp(((colitem_t *)n1)->string, ((colitem_t *)n2)->string);
  1431. }
  1432. static int colelement_cmp(const void *n1, const void *n2)
  1433. {
  1434. int r;
  1435. r = strcmp(((colitem_t *)n1)->string, ((colitem_t *)n2)->string);
  1436. if (!r) {
  1437. if (((colitem_t *)n1)->element && ((colitem_t *)n2)->element) {
  1438. r = strcmp(((colitem_t *)n1)->element, ((colitem_t *)n2)->element);
  1439. } else if (((colitem_t *)n1)->element == ((colitem_t *)n2)->element) {
  1440. r = 0; /* both null */
  1441. } else {
  1442. r = (((colitem_t *)n1)->element == NULL) ? -1 : 1;
  1443. }
  1444. }
  1445. return r;
  1446. }
  1447. static void del_colitem(colitem_t *p)
  1448. {
  1449. /* free((void *) p->element); */
  1450. /* free((void *) p->string); */
  1451. free(p);
  1452. }
  1453. static colitem_t *new_colitem(char *item, char *def)
  1454. {
  1455. colitem_t *p;
  1456. p = xmalloc(sizeof(colitem_t));
  1457. p->string = xsymdup(item);
  1458. p->element = (!def) ? def : xsymdup(def);
  1459. return p;
  1460. }
  1461. static void add_colitem(char *item, char *def)
  1462. {
  1463. colitem_t *p;
  1464. #if 0
  1465. printf("adding collation item %s", item);
  1466. if (def) {
  1467. printf(" with definition %s", def);
  1468. }
  1469. printf("\n");
  1470. #endif
  1471. p = new_colitem(item, def);
  1472. #warning devel code
  1473. if (superset) {
  1474. if (tfind(p, &cur_base->root_colitem, colitem_cmp)) {
  1475. /* fprintf(stderr, "skipping superset duplicate collating item \"%s\"\n", p->string); */
  1476. del_colitem(p);
  1477. return;
  1478. /* } else { */
  1479. /* fprintf(stderr, "superset: new collating item \"%s\" = %s\n", p->string, p->element); */
  1480. }
  1481. }
  1482. if (cur_col == cur_derived) {
  1483. if (!tfind(p, &cur_base->root_colitem, colitem_cmp)) {
  1484. /* not in current but could be in base */
  1485. if (!tsearch(p, &cur_base->root_colitem, colitem_cmp)) {
  1486. error_msg("OUT OF MEMORY!");
  1487. }
  1488. } else if (!tfind(p, &cur_base->root_colitem, colelement_cmp)) {
  1489. error_msg("collating element/symbol mismatch: item=%s def=%s", item, def);
  1490. }
  1491. }
  1492. if (!tfind(p, &cur_col->root_colitem, colitem_cmp)) {
  1493. /* not in current but could be in base */
  1494. if (!tsearch(p, &cur_col->root_colitem, colitem_cmp)) {
  1495. error_msg("OUT OF MEMORY!");
  1496. }
  1497. } else if (!tfind(p, &cur_col->root_colitem, colelement_cmp)) {
  1498. error_msg("collating element/symbol mismatch");
  1499. } else { /* already there */
  1500. fprintf(stderr, "duplicate collating item \"%s\"\n", p->string);
  1501. del_colitem(p);
  1502. }
  1503. }
  1504. /* add a script (section) to the current locale */
  1505. static void add_script(const char *s)
  1506. {
  1507. ll_item_t *l;
  1508. /* make sure it isn't in base if working with derived */
  1509. if (cur_base != cur_col) {
  1510. if (find_section_list_item(s, cur_base)) {
  1511. error_msg("attempt to add script %s for derived when already in base", s);
  1512. }
  1513. }
  1514. if (find_section_list_item(s, cur_col)) {
  1515. error_msg("attempt to readd script %s", s);
  1516. }
  1517. l = find_ll_last(cur_col->section_list);
  1518. insque(new_ll_item(DT_SECTION, new_section(s)), l);
  1519. }
  1520. static const char str_forward[] = "forward";
  1521. static const char str_backward[] = "backward";
  1522. static const char str_position[] = "position";
  1523. static void do_order_start(void)
  1524. {
  1525. const char *s;
  1526. char *e;
  1527. ll_item_t *l;
  1528. section_t *sect;
  1529. int rule;
  1530. if (order_state & ~IN_ORDER) {
  1531. error_msg("order_start following reorder{_sections}_after");
  1532. }
  1533. order_state |= IN_ORDER;
  1534. if (superset) {
  1535. if (++superset_order_start_cnt > 1) {
  1536. error_msg("currently only a common order_start is supported in superset");
  1537. }
  1538. return;
  1539. }
  1540. if (!(s = next_token())) {
  1541. s = str_forward; /* if no args */
  1542. }
  1543. if (*s == '<') { /* section (script) */
  1544. e = strrchr(s,'>');
  1545. if ((*s == '<') && e && (*e == '>') && !e[1]) {
  1546. e[1] = 0; /* cleanup in case next_token stored something */
  1547. if (!(l = find_section_list_item(s, cur_col))) {
  1548. error_msg("ref of undefined sections: %s", s);
  1549. }
  1550. sect = (section_t *)(l->data);
  1551. if (sect->num_rules) {
  1552. error_msg("sections already defined: %s", s);
  1553. }
  1554. } else {
  1555. error_msg("illegal section ref: %s", s);
  1556. }
  1557. if (!(s = next_token())) {
  1558. s = str_forward; /* if no args */
  1559. } else if (*s != ';') {
  1560. error_msg("missing seperator!");
  1561. }
  1562. } else { /* need an anonymous section */
  1563. if ((*cur_section->name != '<') && (cur_section->num_items == 0)) { /* already in an empty anonymous section */
  1564. sect = cur_section;
  1565. /* fprintf(stdout, "using empty anon section %s\n", sect->name); */
  1566. } else {
  1567. sect = new_section(NULL);
  1568. l = find_ll_last(cur_col->section_list);
  1569. insque(new_ll_item(DT_SECTION, sect), l);
  1570. /* fprintf(stdout, "adding order section after section %s\n", ((section_t *)(l->data))->name); */
  1571. /* fprintf(stdout, " last section is %s\n", ((section_t *)(l->next->data))->name); */
  1572. }
  1573. sect->num_rules = 0; /* setting this below so nix default */
  1574. }
  1575. cur_section = sect;
  1576. /* fprintf(stdout, "cur_section now %s\n", cur_section->name); */
  1577. #warning need to add section to weight list?
  1578. /* now do rules */
  1579. do {
  1580. rule = 0;
  1581. if (*s == ';') {
  1582. ++s;
  1583. }
  1584. while (*s) {
  1585. if (!strncmp(str_forward, s, 7)) {
  1586. rule |= R_FORWARD;
  1587. s += 7;
  1588. } else if (!strncmp(str_backward, s, 8)) {
  1589. rule |= R_BACKWARD;
  1590. s += 8;
  1591. } else if (!strncmp(str_position, s, 8)) {
  1592. rule |= R_POSITION;
  1593. s += 8;
  1594. }
  1595. if (*s == ',') {
  1596. ++s;
  1597. continue;
  1598. }
  1599. if (!*s || (*s == ';')) {
  1600. if (sect->num_rules >= MAX_COLLATION_WEIGHTS) {
  1601. error_msg("more than %d weight rules!", MAX_COLLATION_WEIGHTS);
  1602. }
  1603. if (!rule) {
  1604. error_msg("missing weight rule!");
  1605. }
  1606. if ((rule & (R_FORWARD|R_BACKWARD|R_POSITION)) > R_BACKWARD) {
  1607. error_msg("backward paired with forward and/or position!");
  1608. }
  1609. sect->rules[sect->num_rules++] = rule;
  1610. rule = 0;
  1611. continue;
  1612. }
  1613. error_msg("illegal weight rule: %s", s);
  1614. }
  1615. } while ((s = next_token()) != NULL);
  1616. cur_section = sect;
  1617. /* fprintf(stderr, "setting cur_num_weights to %d for %s\n", sect->num_rules, sect->name); */
  1618. cur_num_weights = sect->num_rules;
  1619. memcpy(cur_rule, sect->rules, MAX_COLLATION_WEIGHTS);
  1620. }
  1621. static void do_order_end(void)
  1622. {
  1623. if (!(order_state & IN_ORDER)) {
  1624. error_msg("order_end with no matching order_start");
  1625. }
  1626. order_state &= ~IN_ORDER;
  1627. cur_section = new_section(NULL);
  1628. }
  1629. static void do_reorder_after(void)
  1630. {
  1631. char *t;
  1632. ll_item_t *lli;
  1633. const weight_t *w;
  1634. int save_cur_num_weights;
  1635. char save_cur_rule[MAX_COLLATION_WEIGHTS];
  1636. if (order_state & ~IN_REORDER) {
  1637. error_msg("reorder_after following order_start or reorder_sections_after");
  1638. }
  1639. order_state |= IN_REORDER;
  1640. if (superset) {
  1641. error_msg("currently reorder_after is not supported in supersets");
  1642. }
  1643. #warning have to use rule for current section!!!
  1644. if (!(t = next_token())) {
  1645. error_msg("missing arg for reorder_after");
  1646. }
  1647. t = xsymdup(t);
  1648. if (next_token() != NULL) {
  1649. error_msg("trailing text reorder_after: %s", pos);
  1650. }
  1651. if (cur_col == cur_base) {
  1652. error_msg("sorry.. reorder_after in base locale is not currently supported");
  1653. }
  1654. if (!(lli = find_wi_index(t, cur_base))) {
  1655. error_msg("reorder_after for non-base item currently not supported: %s", t);
  1656. }
  1657. w = ((weighted_item_t *)(lli->data))->weight;
  1658. save_cur_num_weights = cur_num_weights;
  1659. memcpy(save_cur_rule, cur_rule, MAX_COLLATION_WEIGHTS);
  1660. cur_section = new_section("R");
  1661. insque(new_ll_item(DT_REORDER, cur_section), lli);
  1662. #if 0
  1663. {
  1664. ll_item_t *l1;
  1665. ll_item_t *l2;
  1666. ll_item_t *l3;
  1667. l1 = new_ll_item(DT_REORDER, cur_section);
  1668. l2 = find_ll_last(cur_col->section_list);
  1669. insque(l1, l2);
  1670. l3 = find_ll_last(cur_col->section_list);
  1671. fprintf(stderr, "reorder_after %p %p %p %s\n", l1, l2, l3, cur_section->name);
  1672. }
  1673. #else
  1674. insque(new_ll_item(DT_REORDER, cur_section), find_ll_last(cur_col->section_list));
  1675. #endif
  1676. cur_num_weights = cur_section->num_rules = save_cur_num_weights;
  1677. memcpy(cur_rule, save_cur_rule, MAX_COLLATION_WEIGHTS);
  1678. memcpy(cur_section->rules, save_cur_rule, MAX_COLLATION_WEIGHTS);
  1679. #warning devel code
  1680. /* fprintf(stderr, "reorder -- %s %d\n", ((weighted_item_t *)(lli->data))->symbol, w->num_weights); */
  1681. #warning hack to get around hu_HU reorder-after problem
  1682. /* if (!w->num_weights) { */
  1683. /* } else { */
  1684. /* cur_num_weights = w->num_weights; */
  1685. /* memcpy(cur_rule, w->rule, MAX_COLLATION_WEIGHTS); */
  1686. /* } */
  1687. /* fprintf(stderr, "reorder_after succeeded for %s\n", t); */
  1688. }
  1689. static void do_reorder_end(void)
  1690. {
  1691. if (!(order_state & IN_REORDER)) {
  1692. error_msg("reorder_end with no matching reorder_after");
  1693. }
  1694. order_state &= ~IN_REORDER;
  1695. }
  1696. static void do_reorder_sections_after(void)
  1697. {
  1698. const char *t;
  1699. ll_item_t *lli;
  1700. if (order_state & ~IN_REORDER_SECTIONS) {
  1701. error_msg("reorder_sections_after following order_start or reorder_after");
  1702. }
  1703. order_state |= IN_REORDER_SECTIONS;
  1704. if (superset) {
  1705. error_msg("currently reorder_sections_after is not supported in supersets");
  1706. }
  1707. if (!(t = next_token())) {
  1708. error_msg("missing arg for reorder_sections_after");
  1709. }
  1710. t = xsymdup(t);
  1711. if (next_token() != NULL) {
  1712. error_msg("trailing text reorder_sections_after: %s", pos);
  1713. }
  1714. if (cur_col == cur_base) {
  1715. error_msg("sorry.. reorder_sections_after in base locale is not currently supported");
  1716. }
  1717. lli = cur_base->section_list;
  1718. do {
  1719. /* fprintf(stderr, "hmm -- |%s|%d|\n", ((section_t *)(lli->data))->name, lli->data_type); */
  1720. if (lli->data_type & DT_SECTION) {
  1721. /* fprintf(stderr, "checking |%s|%s|\n", ((section_t *)(lli->data))->name, t); */
  1722. if (!strcmp(((section_t *)(lli->data))->name, t)) {
  1723. reorder_section_ptr = lli;
  1724. return;
  1725. }
  1726. }
  1727. lli = lli->next;
  1728. } while (lli);
  1729. error_msg("reorder_sections_after for non-base item currently not supported: %s", t);
  1730. }
  1731. static void do_reorder_sections_end(void)
  1732. {
  1733. if (!(order_state & IN_REORDER_SECTIONS)) {
  1734. error_msg("reorder_sections_end with no matching reorder_sections_after");
  1735. }
  1736. order_state &= ~IN_REORDER_SECTIONS;
  1737. reorder_section_ptr = NULL;
  1738. }
  1739. static ll_item_t *new_ll_item(int data_type, void *data)
  1740. {
  1741. ll_item_t *p;
  1742. p = xmalloc(sizeof(ll_item_t));
  1743. p->next = p->prev = NULL;
  1744. p->data_type = data_type;
  1745. p->data = data;
  1746. p->idx = INT_MIN;
  1747. return p;
  1748. }
  1749. static int sym_cmp(const void *n1, const void *n2)
  1750. {
  1751. /* fprintf(stderr, "sym_cmp: |%s| |%s|\n", (const char *)n1, (const char *)n2); */
  1752. return strcmp((const char *) n1, (const char *) n2);
  1753. }
  1754. static char *xsymdup(const char *s)
  1755. {
  1756. void *p;
  1757. if (!(p = tfind(s, &root_sym, sym_cmp))) { /* not a currently known symbol */
  1758. if (!(s = strdup(s)) || !(p = tsearch(s, &root_sym, sym_cmp))) {
  1759. error_msg("OUT OF MEMORY!");
  1760. }
  1761. ++num_sym;
  1762. mem_sym += strlen(s) + 1;
  1763. /* fprintf(stderr, "xsymdup: alloc |%s| %p |%s| %p\n", *(char **)p, p, s, s); */
  1764. /* } else { */
  1765. /* fprintf(stderr, "xsymdup: found |%s| %p\n", *(char **)p, p); */
  1766. }
  1767. return *(char **) p;
  1768. }
  1769. static int weight_cmp(const void *n1, const void *n2)
  1770. {
  1771. const weight_t *w1 = (const weight_t *) n1;
  1772. const weight_t *w2 = (const weight_t *) n2;
  1773. int i, r;
  1774. if (w1->num_weights != w2->num_weights) {
  1775. return w1->num_weights - w2->num_weights;
  1776. }
  1777. for (i=0 ; i < w1->num_weights ; i++) {
  1778. if (w1->rule[i] != w2->rule[i]) {
  1779. return w1->rule[i] - w2->rule[i];
  1780. }
  1781. if ((r = strcmp(w1->colitem[i], w2->colitem[i])) != 0) {
  1782. return r;
  1783. }
  1784. }
  1785. return 0;
  1786. }
  1787. static weight_t *register_weight(weight_t *w)
  1788. {
  1789. void *p;
  1790. if (!(p = tfind(w, &root_weight, weight_cmp))) { /* new weight */
  1791. p = xmalloc(sizeof(weight_t));
  1792. memcpy(p, w, sizeof(weight_t));
  1793. if (!(p = tsearch(p, &root_weight, weight_cmp))) {
  1794. error_msg("OUT OF MEMORY!");
  1795. }
  1796. ++unique_weights;
  1797. /* } else { */
  1798. /* fprintf(stderr, "rw: found\n"); */
  1799. }
  1800. return *(weight_t **)p;
  1801. }
  1802. static size_t ll_len(ll_item_t *l)
  1803. {
  1804. size_t n = 0;
  1805. ll_item_t *p = l;
  1806. while (p) {
  1807. ++n;
  1808. p = p->next;
  1809. if (p == l) { /* work for circular too */
  1810. break;
  1811. }
  1812. }
  1813. return n;
  1814. }
  1815. static size_t ll_count(ll_item_t *l, int mask)
  1816. {
  1817. size_t n = 0;
  1818. ll_item_t *p = l;
  1819. while (p) {
  1820. if (p->data_type & mask) {
  1821. ++n;
  1822. }
  1823. p = p->next;
  1824. if (p == l) { /* work for circular too */
  1825. break;
  1826. }
  1827. }
  1828. return n;
  1829. }
  1830. static int wi_index_cmp(const void *n1, const void *n2)
  1831. {
  1832. const char *s1 = ((weighted_item_t *)(((ll_item_t *) n1)->data))->symbol;
  1833. const char *s2 = ((weighted_item_t *)(((ll_item_t *) n2)->data))->symbol;
  1834. return strcmp(s1, s2);
  1835. }
  1836. static void add_wi_index(ll_item_t *l)
  1837. {
  1838. assert(l->data_type == DT_WEIGHTED);
  1839. if (!strcmp(((weighted_item_t *)(l->data))->symbol, "UNDEFINED")) {
  1840. cur_col->undefined_idx = l;
  1841. }
  1842. if (!tfind(l, &cur_col->root_wi_index, wi_index_cmp)) { /* new wi_index */
  1843. if (!tsearch(l, &cur_col->root_wi_index, wi_index_cmp)) {
  1844. error_msg("OUT OF MEMORY!");
  1845. }
  1846. }
  1847. if (cur_base != cur_col) {
  1848. if (!tfind(l, &cur_base->root_wi_index, wi_index_cmp)) {/* not a base val */
  1849. /* printf("derived: %s\n", ((weighted_item_t *)(l->data))->symbol); */
  1850. if (!tfind(l, &cur_base->root_derived_wi, wi_index_cmp)) { /* new derived */
  1851. if (!tsearch(l, &cur_base->root_derived_wi, wi_index_cmp)) {
  1852. error_msg("OUT OF MEMORY!");
  1853. }
  1854. }
  1855. }
  1856. }
  1857. }
  1858. static int final_index;
  1859. static int is_ucode(const char *s)
  1860. {
  1861. if ((s[0] == '<')
  1862. && (s[1] == 'U')
  1863. && isxdigit(s[2])
  1864. && isxdigit(s[3])
  1865. && isxdigit(s[4])
  1866. && isxdigit(s[5])
  1867. && (s[6] == '>')
  1868. ) {
  1869. return 7;
  1870. } else {
  1871. return 0;
  1872. }
  1873. }
  1874. static void add_final_col_index(const char *s)
  1875. {
  1876. ENTRY e;
  1877. e.key = (char *) s;
  1878. e.data = (void *)(final_index);
  1879. if (!hsearch(e, FIND)) { /* not in the table */
  1880. if (!hsearch(e, ENTER)) {
  1881. error_msg("OUT OF MEMORY! (hsearch)");
  1882. }
  1883. #if 0
  1884. {
  1885. int n;
  1886. void *v;
  1887. colitem_t ci;
  1888. colitem_t *p;
  1889. const char *t;
  1890. if (!strcmp(s, "UNDEFINED")) {
  1891. printf("%6d: %s\n", final_index, s);
  1892. } else {
  1893. assert(*s == '<');
  1894. if ((n = is_ucode(s)) != 0) {
  1895. assert(!s[n]);
  1896. printf("%6d: %s\n", final_index, s);
  1897. } else {
  1898. ci.string = (char *) s;
  1899. ci.element = NULL; /* don't care */
  1900. v = tfind(&ci, &cur_base->root_colitem, colitem_cmp);
  1901. if (!v) {
  1902. fprintf(stderr, "%s NOT DEFINED!!!\n", s);
  1903. } else {
  1904. p = *((colitem_t **) v);
  1905. if (p->element != NULL) {
  1906. t = p->element;
  1907. assert(*t == '"');
  1908. ++t;
  1909. n = is_ucode(t);
  1910. assert(n);
  1911. printf("%6d: %.*s | ", final_index, n, t);
  1912. do {
  1913. t += n;
  1914. assert(*t);
  1915. if (*t == '"') {
  1916. assert(!t[1]);
  1917. break;
  1918. }
  1919. n = is_ucode(t);
  1920. assert(n);
  1921. printf("%.*s", n, t);
  1922. } while (1);
  1923. printf(" collating-element %s\n", s);
  1924. } else {
  1925. printf("%6d: %s (collating-symbol)\n", final_index, s);
  1926. }
  1927. }
  1928. }
  1929. }
  1930. }
  1931. #endif
  1932. ++final_index;
  1933. }
  1934. }
  1935. static int final_index_val0(const char *s)
  1936. {
  1937. ENTRY *p;
  1938. ENTRY e;
  1939. e.key = (char *) s;
  1940. if (!(p = hsearch(e, FIND))) { /* not in the table */
  1941. return 0;
  1942. }
  1943. return (int)(p->data);
  1944. }
  1945. static int final_index_val(const char *s)
  1946. {
  1947. ENTRY *p;
  1948. ENTRY e;
  1949. e.key = (char *) s;
  1950. if (!(p = hsearch(e, FIND))) { /* not in the table */
  1951. error_msg("can't find final index: %s", s);
  1952. }
  1953. return (int)(p->data);
  1954. }
  1955. static size_t num_tree_nodes;
  1956. static void count_nodes(const void *ptr, VISIT order, int level)
  1957. {
  1958. if ((order == postorder) || (order == leaf)) {
  1959. ++num_tree_nodes;
  1960. }
  1961. }
  1962. static size_t tnumnodes(const void *root)
  1963. {
  1964. num_tree_nodes = 0;
  1965. twalk(root, count_nodes);
  1966. return num_tree_nodes;
  1967. }
  1968. static ll_item_t *find_wi_index(const char *sym, col_locale_t *cl)
  1969. {
  1970. weighted_item_t w;
  1971. ll_item_t l;
  1972. void *p;
  1973. w.symbol = sym;
  1974. l.data = &w;
  1975. l.data_type = DT_WEIGHTED;
  1976. p = tfind(&l, &cl->root_wi_index, wi_index_cmp);
  1977. if (p) {
  1978. p = *(ll_item_t **)p;
  1979. }
  1980. return (ll_item_t *) p;
  1981. }
  1982. static void mark_reordered(const char *sym)
  1983. {
  1984. ll_item_t *lli;
  1985. lli = find_wi_index(sym, cur_base);
  1986. if (lli) {
  1987. if (!tsearch(lli, &cur_base->root_wi_index_reordered, wi_index_cmp)) {
  1988. error_msg("OUT OF MEMORY!");
  1989. }
  1990. }
  1991. }
  1992. static ll_item_t *find_wi_index_reordered(const char *sym)
  1993. {
  1994. weighted_item_t w;
  1995. ll_item_t l;
  1996. void *p;
  1997. w.symbol = sym;
  1998. l.data = &w;
  1999. l.data_type = DT_WEIGHTED;
  2000. p = tfind(&l, &cur_base->root_wi_index_reordered, wi_index_cmp);
  2001. if (p) {
  2002. p = *(ll_item_t **)p;
  2003. }
  2004. return (ll_item_t *) p;
  2005. }
  2006. static ll_item_t *init_comm_ptr(void)
  2007. {
  2008. assert(cur_base);
  2009. assert(cur_base->section_list);
  2010. /* at the moment, only support one section in comm */
  2011. assert(cur_base->section_list->next == NULL);
  2012. comm_cur_ptr = ((section_t *)(cur_base->section_list->data))->itm_list;
  2013. while (comm_cur_ptr && (comm_cur_ptr->data_type & DT_REORDER)) {
  2014. comm_cur_ptr = comm_cur_ptr->next;
  2015. }
  2016. #warning devel code
  2017. /* { */
  2018. /* ll_item_t *p = comm_cur_ptr; */
  2019. /* fprintf(stderr, "init_comm_ptr\n"); */
  2020. /* while (p != comm_cur_ptr) { */
  2021. /* if (p->data_type & DT_WEIGHTED) { */
  2022. /* fprintf(stderr, "%s", ((weighted_item_t *)p)->symbol); */
  2023. /* } */
  2024. /* p = p->next; */
  2025. /* } */
  2026. /* } */
  2027. assert(comm_cur_ptr);
  2028. /* fprintf(stderr, "init_comm_ptr -- %s %p %p %p %d\n", */
  2029. /* ((weighted_item_t *)(comm_cur_ptr->data))->symbol, */
  2030. /* comm_cur_ptr, comm_cur_ptr->prev, comm_cur_ptr->next, */
  2031. /* ll_len(comm_cur_ptr)); */
  2032. comm_prev_ptr = NULL;
  2033. return comm_cur_ptr;
  2034. }
  2035. static ll_item_t *next_comm_ptr(void)
  2036. {
  2037. /* at the moment, only support one section in comm */
  2038. assert(cur_base->section_list->next == NULL);
  2039. comm_prev_ptr = comm_cur_ptr;
  2040. while (comm_cur_ptr && ((comm_cur_ptr = comm_cur_ptr->next) != NULL)) {
  2041. if (!(comm_cur_ptr->data_type & DT_REORDER)) {
  2042. break;
  2043. }
  2044. }
  2045. return comm_cur_ptr;
  2046. }
  2047. static int dump_count;
  2048. #if 0
  2049. static void dump_section(section_t *s, int mask, col_locale_t *der)
  2050. {
  2051. ll_item_t *lli;
  2052. ll_item_t *lli0;
  2053. weighted_item_t *w;
  2054. weight_t *p;
  2055. int i;
  2056. lli0 = lli = s->itm_list;
  2057. if (!lli0) {
  2058. return;
  2059. }
  2060. do {
  2061. if (!(lli->data_type & mask)) {
  2062. lli = lli->next;
  2063. continue;
  2064. }
  2065. if (lli->data_type & DT_WEIGHTED) {
  2066. ++dump_count;
  2067. w = (weighted_item_t *)(lli->data);
  2068. p = w->weight;
  2069. printf("%6d: %s (%d) ", dump_count, w->symbol, p->num_weights);
  2070. for (i = 0 ; i < p->num_weights ; i++) {
  2071. if (p->rule[i] & R_FORWARD) {
  2072. printf("F");
  2073. }
  2074. if (p->rule[i] & R_BACKWARD) {
  2075. printf("B");
  2076. }
  2077. if (p->rule[i] & R_POSITION) {
  2078. printf("P");
  2079. }
  2080. printf(",");
  2081. }
  2082. for (i = 0 ; i < p->num_weights ; i++) {
  2083. printf(" %s", p->colitem[i]);
  2084. }
  2085. printf("\n");
  2086. } else if (lli->data_type & (DT_SECTION|DT_REORDER)) {
  2087. if (lli->data_type == DT_REORDER) {
  2088. assert(der);
  2089. if (strncmp(((section_t *)(lli->data))->name, der->name, strlen(der->name))) {
  2090. lli = lli->next;
  2091. continue;
  2092. }
  2093. }
  2094. if (lli->data_type & DT_SECTION) {
  2095. printf("SECTION -----------------\n");
  2096. } else {
  2097. printf("REORDER -----------------\n");
  2098. }
  2099. dump_section((section_t *)(lli->data), mask, der);
  2100. printf("DONE --------------------\n");
  2101. }
  2102. lli = lli->next;
  2103. } while (lli != lli0);
  2104. }
  2105. #else
  2106. static int in_reorder_section = 0;
  2107. static void dump_section(section_t *s, int mask, col_locale_t *der)
  2108. {
  2109. ll_item_t *lli;
  2110. ll_item_t *lli0;
  2111. weighted_item_t *w;
  2112. weight_t *p;
  2113. int i;
  2114. lli0 = lli = s->itm_list;
  2115. if (!lli0) {
  2116. return;
  2117. }
  2118. do {
  2119. if (!(lli->data_type & mask)) {
  2120. lli = lli->next;
  2121. continue;
  2122. }
  2123. if (lli->data_type & DT_WEIGHTED) {
  2124. ++dump_count;
  2125. w = (weighted_item_t *)(lli->data);
  2126. p = w->weight;
  2127. #if 1
  2128. if (in_reorder_section) {
  2129. printf(" %p", w);
  2130. }
  2131. #else
  2132. printf("%6d: %s (%d) ", dump_count, w->symbol, p->num_weights);
  2133. for (i = 0 ; i < p->num_weights ; i++) {
  2134. if (p->rule[i] & R_FORWARD) {
  2135. printf("F");
  2136. }
  2137. if (p->rule[i] & R_BACKWARD) {
  2138. printf("B");
  2139. }
  2140. if (p->rule[i] & R_POSITION) {
  2141. printf("P");
  2142. }
  2143. printf(",");
  2144. }
  2145. for (i = 0 ; i < p->num_weights ; i++) {
  2146. printf(" %s", p->colitem[i]);
  2147. }
  2148. printf("\n");
  2149. #endif
  2150. } else if (lli->data_type & (DT_SECTION|DT_REORDER)) {
  2151. if (lli->data_type == DT_REORDER) {
  2152. assert(der);
  2153. if (strncmp(((section_t *)(lli->data))->name, der->name, strlen(der->name))) {
  2154. lli = lli->next;
  2155. continue;
  2156. }
  2157. }
  2158. if (lli->data_type & DT_SECTION) {
  2159. /* printf("SECTION -----------------\n"); */
  2160. assert(0);
  2161. } else {
  2162. /* printf("REORDER -----------------\n"); */
  2163. in_reorder_section = 1;
  2164. }
  2165. dump_section((section_t *)(lli->data), mask, der);
  2166. /* printf("DONE --------------------\n"); */
  2167. printf("\n");
  2168. in_reorder_section = 0;
  2169. }
  2170. lli = lli->next;
  2171. } while (lli != lli0);
  2172. }
  2173. #endif
  2174. static void dump_weights(const char *name)
  2175. {
  2176. ll_item_t *lli;
  2177. col_locale_t *base;
  2178. col_locale_t *der;
  2179. col_locale_t cl;
  2180. void *p;
  2181. assert(name);
  2182. if (!*name) { /* use last */
  2183. base = cur_base;
  2184. der = cur_derived;
  2185. } else {
  2186. cl.name = (char *) name;
  2187. if (!(p = tfind(&cl, &root_col_locale, col_locale_cmp))) {
  2188. error_msg("unknown locale: %s", name);
  2189. }
  2190. base = *((col_locale_t **) p);
  2191. der = NULL;
  2192. if (base->base_locale) { /* oops... really derived */
  2193. der = base;
  2194. base = der->base_locale;
  2195. }
  2196. }
  2197. dump_count = 0;
  2198. if (base) {
  2199. /* printf("BASE - %s\n", base->name); */
  2200. for (lli = base->section_list ; lli ; lli = lli->next) {
  2201. /* printf("SECTION %s\n", ((section_t *)(lli->data))->name); */
  2202. dump_section((section_t *)(lli->data), ~0, der);
  2203. }
  2204. }
  2205. assert(der != base);
  2206. if (der) {
  2207. /* printf("DERIVED - %s\n", der->name); */
  2208. for (lli = der->section_list ; lli ; lli = lli->next) {
  2209. if (lli->data_type == DT_SECTION) {
  2210. dump_section((section_t *)(lli->data), DT_WEIGHTED, der);
  2211. }
  2212. }
  2213. }
  2214. /* printf("DONE\n"); */
  2215. }
  2216. static void print_starter_node(const void *ptr, VISIT order, int level)
  2217. {
  2218. if (order == postorder || order == leaf) {
  2219. fprintf(stderr, " %s\n", *(const char **) ptr);
  2220. }
  2221. }
  2222. static void finalize_base(void)
  2223. {
  2224. ll_item_t *s;
  2225. ll_item_t *h;
  2226. ll_item_t *lli;
  2227. ll_item_t *h2;
  2228. ll_item_t *l2;
  2229. ll_item_t *cli;
  2230. ll_item_t *rli = NULL;
  2231. weighted_item_t *w;
  2232. weight_t *p;
  2233. int i, n, mr, r, mi;
  2234. col_locale_t *cl;
  2235. void *mm;
  2236. int num_invariant = 0;
  2237. int num_varying = 0;
  2238. int max_weight;
  2239. int index2weight_len_inc = 1;
  2240. assert(cur_base);
  2241. assert(base_locale_len+1 < BASE_LOCALE_LEN);
  2242. base_locale_array[base_locale_len].name = cur_base->name;
  2243. base_locale_array[base_locale_len].num_weights = 1;
  2244. base_locale_array[base_locale_len].index2weight_offset = index2weight_len;
  2245. base_locale_array[base_locale_len].index2ruleidx_offset = index2ruleidx_len;
  2246. if (!strcmp(cur_base->name,"ja_JP") || !strcmp(cur_base->name,"ko_KR")) {
  2247. #warning fix the index2weight check!!
  2248. index2weight_len_inc = 0;
  2249. }
  2250. /* printf("%s -- index2weight_len = %d\n", cur_base->name, index2weight_len); */
  2251. if (!hcreate(30000)) {
  2252. error_msg("OUT OF MEMORY!");
  2253. }
  2254. /* first pass ... set the fixed indexes */
  2255. final_index = i = 1;
  2256. mr = 0;
  2257. for (s = cur_base->section_list ; s ; s = s->next) {
  2258. #if 1
  2259. if (s->data_type & DT_REORDER) { /* a reordered section */
  2260. fprintf(stderr, "pass1: reordered section %s - xxx\n", ((section_t *)(s->data))->name);
  2261. lli = ((section_t *)(s->data))->itm_list;
  2262. r = 0;
  2263. if (lli) {
  2264. /* r = ll_len( ((section_t *)(lli->data))->itm_list ); */
  2265. r = ll_len(lli) + 1;
  2266. }
  2267. if (r > mr) {
  2268. mr = r;
  2269. }
  2270. fprintf(stderr, "pass1: reordered section %s - %d\n", ((section_t *)(s->data))->name, r);
  2271. continue;
  2272. }
  2273. #endif
  2274. h = lli = ((section_t *)(s->data))->itm_list;
  2275. if (!lli) {
  2276. continue;
  2277. }
  2278. do {
  2279. if (lli->data_type & DT_RANGE) {
  2280. i += mr;
  2281. mr = 0;
  2282. #warning check ko_kR and 9
  2283. /* ++i; */
  2284. lli->idx = i;
  2285. assert(!rli);
  2286. rli = lli;
  2287. fprintf(stderr, "range pre = %d after = ", i);
  2288. i += ((range_item_t *)(lli->data))->length + 1;
  2289. #warning check ko_kR and 9
  2290. /* ++i; */
  2291. fprintf(stderr, "%d\n", i);
  2292. if (!index2weight_len_inc) { /* ko_KR hack */
  2293. final_index += ((range_item_t *)(lli->data))->length + 1;
  2294. }
  2295. /* add_final_col_index("RANGE"); */
  2296. } else if (lli->data_type & DT_WEIGHTED) {
  2297. i += mr;
  2298. mr = 0;
  2299. w = (weighted_item_t *)(lli->data);
  2300. if (find_wi_index_reordered(w->symbol)) { /* reordered symbol so skip on first pass */
  2301. ++num_varying;
  2302. ++i;
  2303. continue;
  2304. }
  2305. ++num_invariant;
  2306. index2weight_buffer[index2weight_len] = lli->idx = i++;
  2307. index2weight_len += index2weight_len_inc;
  2308. add_final_col_index(w->symbol);
  2309. } else {
  2310. assert(lli->data_type & DT_REORDER);
  2311. r = ll_len( ((section_t *)(lli->data))->itm_list );
  2312. #warning check ko_kR and 9
  2313. if (r > mr) {
  2314. mr = r;
  2315. }
  2316. /* r = 0; */
  2317. }
  2318. } while ((lli = lli->next) != h);
  2319. }
  2320. /* second pass ... set the reordered indexes */
  2321. mi = i + mr;
  2322. mr = i = 0;
  2323. for (s = cur_base->section_list ; s ; s = s->next) {
  2324. h = lli = ((section_t *)(s->data))->itm_list;
  2325. if (!lli) {
  2326. continue;
  2327. }
  2328. do {
  2329. if (lli->data_type & DT_RANGE) {
  2330. i += mr;
  2331. mr = 0;
  2332. i = lli->idx + ((range_item_t *)(lli->data))->length + 1;
  2333. #warning check
  2334. } else if ((lli->data_type & DT_WEIGHTED) && !(s->data_type & DT_REORDER)) {
  2335. i += mr;
  2336. mr = 0;
  2337. w = (weighted_item_t *)(lli->data);
  2338. if (find_wi_index_reordered(w->symbol) /* reordered symbol skipped on first pass */
  2339. #if 0
  2340. || (s->data_type & DT_REORDER) /* or in a reordered section */
  2341. #endif
  2342. ) {
  2343. assert(!(s->data_type & DT_REORDER));
  2344. index2weight_buffer[index2weight_len] = lli->idx = ++i;
  2345. index2weight_len += index2weight_len_inc;
  2346. add_final_col_index(w->symbol);
  2347. /* fprintf(stdout, "%11s: r %6d %6d %s\n", */
  2348. /* cur_base->name, lli->idx, final_index_val(w->symbol), w->symbol); */
  2349. continue;
  2350. }
  2351. i = lli->idx;
  2352. /* fprintf(stdout, "%11s: w %6d %6d %s\n", */
  2353. /* cur_base->name, lli->idx, final_index_val(w->symbol), w->symbol); */
  2354. } else {
  2355. /* fprintf(stderr, "section: %s %d %d\n", ((section_t *)(s->data))->name, */
  2356. /* s->data_type, lli->data_type); */
  2357. /* assert(!(s->data_type & DT_REORDER)); */
  2358. /* assert(lli->data_type & DT_REORDER); */
  2359. #if 1
  2360. if (s->data_type & DT_REORDER) {
  2361. h2 = l2 = lli;
  2362. if (!h2) {
  2363. continue;
  2364. }
  2365. } else {
  2366. assert(s->data_type & DT_SECTION);
  2367. h2 = l2 = ((section_t *)(lli->data))->itm_list;
  2368. if (!h2) {
  2369. continue;
  2370. }
  2371. }
  2372. #else
  2373. h2 = l2 = ((section_t *)(lli->data))->itm_list;
  2374. if (!h2) {
  2375. continue;
  2376. }
  2377. #endif
  2378. r = 0;
  2379. do {
  2380. assert(l2->data_type & DT_WEIGHTED);
  2381. ++r;
  2382. l2->idx = i + r;
  2383. /* fprintf(stdout, "%s: R %6d %s\n", */
  2384. /* ((section_t *)(lli->data))->name, l2->idx, ((weighted_item_t *)(l2->data))->symbol); */
  2385. } while ((l2 = l2->next) != h2);
  2386. if (r > mr) {
  2387. mr = r;
  2388. }
  2389. }
  2390. } while ((lli = lli->next) != h);
  2391. }
  2392. /* finally, walk through all derived locales and set non-reordered section items */
  2393. mr = mi;
  2394. for (cli = cur_base->derived_list ; cli ; cli = cli->next) {
  2395. cl = (col_locale_t *)(cli->data);
  2396. /* fprintf(stderr, "pass3: %d %s\n", cli->data_type, cl->name); */
  2397. /* fprintf(stdout, "pass3: %d %s\n", cli->data_type, cl->name); */
  2398. assert(cli->data_type == DT_COL_LOCALE);
  2399. i = mi;
  2400. for (s = cl->section_list ; s ; s = s->next) {
  2401. /* if (s->data_type & DT_REORDER) { */
  2402. /* continue; */
  2403. /* } */
  2404. h = lli = ((section_t *)(s->data))->itm_list;
  2405. if (!lli) {
  2406. continue;
  2407. }
  2408. do {
  2409. assert(!(lli->data_type & DT_RANGE));
  2410. if (lli->data_type & DT_WEIGHTED) {
  2411. /* fprintf(stderr, " %d %d %s\n", lli->data_type, lli->idx, ((weighted_item_t *)(lli->data))->symbol); */
  2412. add_final_col_index(((weighted_item_t *)(lli->data))->symbol);
  2413. if (s->data_type & DT_REORDER) {
  2414. continue;
  2415. }
  2416. assert(lli->idx == INT_MIN);
  2417. lli->idx = ++i;
  2418. /* fprintf(stdout, "%11s: S %6d %6d %s\n", */
  2419. /* cl->name, lli->idx, */
  2420. /* final_index_val(((weighted_item_t *)(lli->data))->symbol), */
  2421. /* ((weighted_item_t *)(lli->data))->symbol); */
  2422. } else {
  2423. assert(0);
  2424. assert(lli->data_type & DT_SECTION);
  2425. h2 = l2 = ((section_t *)(lli->data))->itm_list;
  2426. if (!h2) {
  2427. continue;
  2428. }
  2429. do {
  2430. assert(l2->data_type & DT_WEIGHTED);
  2431. assert(l2->idx == INT_MIN);
  2432. l2->idx = ++i;
  2433. add_final_col_index(((weighted_item_t *)(l2->data))->symbol);
  2434. } while ((l2 = l2->next) != h2);
  2435. }
  2436. } while ((lli = lli->next) != h);
  2437. }
  2438. if (i > mr) {
  2439. mr = i;
  2440. }
  2441. }
  2442. max_weight = mr;
  2443. assert(num_varying == tnumnodes(cur_base->root_wi_index_reordered));
  2444. /* we can now initialize the wcs2index array */
  2445. {
  2446. ENTRY *p;
  2447. ENTRY e;
  2448. char buf[8];
  2449. static const char xd[] = "0123456789ABCDEF";
  2450. int starter_index = final_index;
  2451. int wcs2index_count = 0;
  2452. strcpy(buf, "<U....>");
  2453. memset(wcs2index, 0, sizeof(wcs2index));
  2454. e.key = (char *) buf;
  2455. for (i=1 ; i <= 0xffff ; i++) {
  2456. buf[5] = xd[ i & 0xf ];
  2457. buf[4] = xd[ (i >> 4) & 0xf ];
  2458. buf[3] = xd[ (i >> 8) & 0xf ];
  2459. buf[2] = xd[ (i >> 12) & 0xf ];
  2460. if ((p = hsearch(e, FIND)) != NULL) {
  2461. ++wcs2index_count;
  2462. if ((tfind(buf, &cur_base->root_starter_char, sym_cmp)) != NULL) {
  2463. wcs2index[i] = ++starter_index;
  2464. /* fprintf(stderr, "wcs2index[ %#06x ] = %d (starter)\n", i, wcs2index[i]); */
  2465. } else {
  2466. wcs2index[i] = (int)(p->data);
  2467. /* fprintf(stderr, "wcs2index[ %#06x ] = %d\n", i, wcs2index[i]); */
  2468. }
  2469. } else {
  2470. if ((tfind(buf, &cur_base->root_starter_char, sym_cmp)) != NULL) {
  2471. error_msg("marked starter but not in hash: %s", buf);
  2472. }
  2473. }
  2474. }
  2475. /* ---------------------------------------------------------------------- */
  2476. {
  2477. int i, n;
  2478. table_data table;
  2479. size_t t, smallest;
  2480. n = 0;
  2481. smallest = SIZE_MAX;
  2482. table.ii = NULL;
  2483. for (i=0 ; i < 14 ; i++) {
  2484. if ((RANGE >> i) < 4) {
  2485. break;
  2486. }
  2487. t = newopt(wcs2index, RANGE, i, &table);
  2488. if (smallest >= t) {
  2489. n = i;
  2490. smallest = t;
  2491. /* } else { */
  2492. /* break; */
  2493. }
  2494. }
  2495. /* printf("smallest = %u for range %#x (%u)\n", smallest, RANGE, RANGE); */
  2496. assert(smallest != SIZE_MAX);
  2497. if (smallest + wcs2colidt_len >= WCS2COLIDT_LEN) {
  2498. error_msg("WCS2COLIDT_LEN too small");
  2499. }
  2500. base_locale_array[base_locale_len].wcs2colidt_offset = wcs2colidt_len;
  2501. table.ii = wcs2colidt_buffer + wcs2colidt_len;
  2502. t = smallest;
  2503. smallest = SIZE_MAX;
  2504. smallest = newopt(wcs2index, RANGE, n, &table);
  2505. assert(t == smallest);
  2506. wcs2colidt_len += smallest;
  2507. /* fprintf(stderr, "smallest = %d wcs2colidt_len = %d\n", smallest, wcs2colidt_len); */
  2508. #if 0
  2509. {
  2510. unsigned int sc, n, i0, i1;
  2511. unsigned int u = 0xe40;
  2512. table_data *tbl = &table;
  2513. #define __LOCALE_DATA_WCctype_TI_MASK ((1 << tbl->ti_shift)-1)
  2514. #define __LOCALE_DATA_WCctype_TI_SHIFT (tbl->ti_shift)
  2515. #define __LOCALE_DATA_WCctype_TI_LEN (tbl->ti_len)
  2516. #define __LOCALE_DATA_WCctype_II_MASK ((1 << tbl->ii_shift)-1)
  2517. #define __LOCALE_DATA_WCctype_II_SHIFT (tbl->ii_shift)
  2518. #define __LOCALE_DATA_WCctype_II_LEN (tbl->ii_len)
  2519. sc = u & __LOCALE_DATA_WCctype_TI_MASK;
  2520. u >>= __LOCALE_DATA_WCctype_TI_SHIFT;
  2521. n = u & __LOCALE_DATA_WCctype_II_MASK;
  2522. u >>= __LOCALE_DATA_WCctype_II_SHIFT;
  2523. i0 = tbl->ii[u];
  2524. fprintf(stderr, "i0 = %d\n", i0);
  2525. i0 <<= __LOCALE_DATA_WCctype_II_SHIFT;
  2526. i1 = tbl->ii[__LOCALE_DATA_WCctype_II_LEN + i0 + n];
  2527. /* i1 = tbl->ti[i0 + n]; */
  2528. fprintf(stderr, "i1 = %d\n", i1);
  2529. i1 <<= __LOCALE_DATA_WCctype_TI_SHIFT;
  2530. /* return *(uint16_t *)(&(tbl->ii[__LOCALE_DATA_WCctype_II_LEN + __LOCALE_DATA_WCctype_TI_LEN + i1 + sc])); */
  2531. fprintf(stderr, "i2 = %d\n", __LOCALE_DATA_WCctype_II_LEN + __LOCALE_DATA_WCctype_TI_LEN + i1 + sc);
  2532. fprintf(stderr, "val = %d\n", tbl->ii[__LOCALE_DATA_WCctype_II_LEN + __LOCALE_DATA_WCctype_TI_LEN + i1 + sc]);
  2533. /* return tbl->ut[i1 + sc]; */
  2534. }
  2535. #endif
  2536. base_locale_array[base_locale_len].ii_shift = table.ii_shift;
  2537. base_locale_array[base_locale_len].ti_shift = table.ti_shift;
  2538. base_locale_array[base_locale_len].ii_len = table.ii_len;
  2539. base_locale_array[base_locale_len].ti_len = table.ti_len;
  2540. }
  2541. /* ---------------------------------------------------------------------- */
  2542. base_locale_array[base_locale_len].num_col_base = num_invariant + num_varying;
  2543. base_locale_array[base_locale_len].max_col_index = final_index;
  2544. base_locale_array[base_locale_len].max_weight = max_weight;
  2545. fprintf(stderr, "%s: %6u invariant %6u varying %6u derived %6u total %6u max weight %6u wcs2\n",
  2546. cur_base->name, num_invariant, num_varying,
  2547. tnumnodes(cur_base->root_derived_wi), final_index, max_weight,
  2548. wcs2index_count);
  2549. }
  2550. #if 1
  2551. /* ok, now we need to dump out the base and derived tables... */
  2552. /* don't forget to break up collating elements!!! */
  2553. /* fprintf(stdout, "**************************************************\n"); */
  2554. /* first pass ... set the invariants */
  2555. for (s = cur_base->section_list ; s ; s = s->next) {
  2556. #if 1
  2557. if (s->data_type & DT_REORDER) {
  2558. fprintf(stderr, "1: skipping reordered section %s\n", ((section_t *)(s->data))->name);
  2559. continue;
  2560. }
  2561. #endif
  2562. h = lli = ((section_t *)(s->data))->itm_list;
  2563. if (!lli) {
  2564. continue;
  2565. }
  2566. do {
  2567. if (lli->data_type & DT_WEIGHTED) {
  2568. w = (weighted_item_t *)(lli->data);
  2569. if (find_wi_index_reordered(w->symbol)) { /* reordered symbol so skip on first pass */
  2570. continue;
  2571. }
  2572. if (index2weight_len_inc) {
  2573. index2ruleidx_buffer[index2ruleidx_len++] =
  2574. add_rule((weighted_item_t *)(lli->data));
  2575. }
  2576. /* fprintf(stdout, "%11s: w %6d %6d %s\n", */
  2577. /* cur_base->name, lli->idx, final_index_val(w->symbol), w->symbol); */
  2578. }
  2579. } while ((lli = lli->next) != h);
  2580. }
  2581. /* second pass ... set varying */
  2582. for (s = cur_base->section_list ; s ; s = s->next) {
  2583. #if 1
  2584. if (s->data_type & DT_REORDER) {
  2585. fprintf(stderr, "2: skipping reordered section %s\n", ((section_t *)(s->data))->name);
  2586. continue;
  2587. }
  2588. #endif
  2589. h = lli = ((section_t *)(s->data))->itm_list;
  2590. if (!lli) {
  2591. continue;
  2592. }
  2593. do {
  2594. if (lli->data_type & DT_WEIGHTED) {
  2595. w = (weighted_item_t *)(lli->data);
  2596. if (find_wi_index_reordered(w->symbol)) { /* reordered symbol so skip on first pass */
  2597. if (index2weight_len_inc) {
  2598. index2ruleidx_buffer[index2ruleidx_len++] =
  2599. add_rule((weighted_item_t *)(lli->data));
  2600. }
  2601. /* fprintf(stdout, "%11s: r %6d %6d %s\n", */
  2602. /* cur_base->name, lli->idx, final_index_val(w->symbol), w->symbol); */
  2603. continue;
  2604. }
  2605. }
  2606. } while ((lli = lli->next) != h);
  2607. }
  2608. do_starter_lists(cur_base);
  2609. /* fprintf(stderr,"updated final_index = %d\n", final_index); */
  2610. if (rli) {
  2611. base_locale_array[base_locale_len].range_low
  2612. = strtoul(((range_item_t *)(rli->data))->symbol1 + 2, NULL, 16);
  2613. base_locale_array[base_locale_len].range_count
  2614. = ((range_item_t *)(rli->data))->length;
  2615. base_locale_array[base_locale_len].range_base_weight = rli->idx;
  2616. base_locale_array[base_locale_len].range_rule_offset = add_range_rule((range_item_t *)(rli->data));
  2617. /* fprintf(stdout, "%11s: %6d %6d %s %s (%d)\n", */
  2618. /* "RANGE", rli->idx, -1, */
  2619. /* ((range_item_t *)(rli->data))->symbol1, */
  2620. /* ((range_item_t *)(rli->data))->symbol2, */
  2621. /* ((range_item_t *)(rli->data))->length); */
  2622. }
  2623. /* fprintf(stdout,"\nDerived\n\n"); */
  2624. /* first, if base name is of the form ll_CC, add a derived locale for it */
  2625. if ((strlen(cur_base->name) == 5)
  2626. && islower(cur_base->name[0])
  2627. && islower(cur_base->name[1])
  2628. && (cur_base->name[2] == '_')
  2629. && isupper(cur_base->name[3])
  2630. && isupper(cur_base->name[4])
  2631. ) {
  2632. fprintf(stderr, "adding special derived for %s\n", cur_base->name);
  2633. /* fprintf(stderr,"updated final_index = %d\n", final_index); */
  2634. assert(der_locale_len+1 < DER_LOCALE_LEN);
  2635. der_locale_array[der_locale_len].name = cur_base->name;
  2636. der_locale_array[der_locale_len].base_idx = base_locale_len;
  2637. u16_buf[0] = 1;
  2638. u16_buf[1] = 0;
  2639. u16_buf_len = 2;
  2640. mm = NULL;
  2641. if ((u16_buf_len > override_len) ||
  2642. !(mm = memmem(override_buffer, override_len*sizeof(override_buffer[0]),
  2643. u16_buf, u16_buf_len*sizeof(u16_buf[0])))
  2644. ) {
  2645. assert(override_len + u16_buf_len < OVERRIDE_LEN);
  2646. memcpy(override_buffer + override_len, u16_buf, u16_buf_len*sizeof(u16_buf[0]));
  2647. der_locale_array[der_locale_len].overrides_offset = override_len;
  2648. override_len += u16_buf_len;
  2649. /* printf("%s: override_len = %d u16_buf_len = %d\n", cl->name, override_len, u16_buf_len); */
  2650. } else if (!(u16_buf_len > override_len)) {
  2651. assert(mm);
  2652. der_locale_array[der_locale_len].overrides_offset = ((uint16_t *)(mm)) - override_buffer;
  2653. /* printf("%s: memmem found a match with u16_buf_len = %d\n", cl->name, u16_buf_len); */
  2654. }
  2655. der_locale_array[der_locale_len].multistart_offset
  2656. = base_locale_array[base_locale_len].multistart_offset;
  2657. der_locale_array[der_locale_len].undefined_idx = final_index_val0("UNDEFINED");
  2658. if (!der_locale_array[der_locale_len].undefined_idx) {
  2659. error_msg("no UNDEFINED definition for %s", cur_base->name);
  2660. }
  2661. ++der_locale_len;
  2662. } else {
  2663. fprintf(stderr, "NOT adding special derived for %s\n", cur_base->name);
  2664. }
  2665. /* now all the derived... */
  2666. for (cli = cur_base->derived_list ; cli ; cli = cli->next) {
  2667. cl = (col_locale_t *)(cli->data);
  2668. assert(cli->data_type == DT_COL_LOCALE);
  2669. assert(der_locale_len+1 < DER_LOCALE_LEN);
  2670. der_locale_array[der_locale_len].name = cl->name;
  2671. der_locale_array[der_locale_len].base_idx = base_locale_len;
  2672. u16_buf_len = 0;
  2673. for (i = 0 ; i < 2 ; i++) {
  2674. if (i) {
  2675. /* fprintf(stdout, " section --- (singles)\n"); */
  2676. u16_buf[u16_buf_len++] = 1; /* single */
  2677. }
  2678. /* we do this in two passes... first all sequences, then all single reorders */
  2679. for (s = cl->section_list ; s ; s = s->next) {
  2680. /* fprintf(stderr, "doing section %s\n", ((section_t *)(s->data))->name); */
  2681. h = lli = ((section_t *)(s->data))->itm_list;
  2682. if (!lli) {
  2683. /* fprintf(stdout, "EMPTY ITEM LIST IN SECTION %s\n", ((section_t *)(s->data))->name ); */
  2684. continue;
  2685. }
  2686. assert(u16_buf_len +4 < sizeof(u16_buf)/sizeof(u16_buf[0]));
  2687. if ((!i && (ll_len(h) > 1) ) || (ll_len(h) == i)) {
  2688. if (!i) {
  2689. /* fprintf(stdout, " section ----------------- %d %d\n", i, ll_len(h)); */
  2690. u16_buf[u16_buf_len++] = ll_len(h); /* multi */
  2691. assert(lli->data_type & DT_WEIGHTED);
  2692. #if 0
  2693. u16_buf[u16_buf_len++] = final_index_val(((weighted_item_t *)(lli->data))->symbol); /* start index */
  2694. #endif
  2695. u16_buf[u16_buf_len++] = lli->idx; /* start weight */
  2696. }
  2697. do {
  2698. assert(lli->data_type & DT_WEIGHTED);
  2699. if (lli->data_type & DT_WEIGHTED) {
  2700. /* fprintf(stdout, "%11s: S %6d %6d %s\n", */
  2701. /* cl->name, lli->idx, */
  2702. /* final_index_val(((weighted_item_t *)(lli->data))->symbol), */
  2703. /* ((weighted_item_t *)(lli->data))->symbol); */
  2704. #if 0
  2705. if (i) {
  2706. assert(u16_buf_len +4 < sizeof(u16_buf)/sizeof(u16_buf[0]));
  2707. u16_buf[u16_buf_len++] = final_index_val(((weighted_item_t *)(lli->data))->symbol);
  2708. assert(u16_buf[u16_buf_len-1]);
  2709. u16_buf[u16_buf_len++] = lli->idx; /* weight */
  2710. }
  2711. #else
  2712. assert(u16_buf_len +4 < sizeof(u16_buf)/sizeof(u16_buf[0]));
  2713. u16_buf[u16_buf_len++] = final_index_val(((weighted_item_t *)(lli->data))->symbol);
  2714. assert(u16_buf[u16_buf_len-1]);
  2715. if (i) {
  2716. u16_buf[u16_buf_len++] = lli->idx; /* weight */
  2717. }
  2718. #endif
  2719. u16_buf[u16_buf_len++] = add_rule((weighted_item_t *)(lli->data));
  2720. }
  2721. } while ((lli = lli->next) != h);
  2722. }
  2723. }
  2724. }
  2725. u16_buf[u16_buf_len++] = 0;
  2726. mm = NULL;
  2727. if ((u16_buf_len > override_len) ||
  2728. !(mm = memmem(override_buffer, override_len*sizeof(override_buffer[0]),
  2729. u16_buf, u16_buf_len*sizeof(u16_buf[0])))
  2730. ) {
  2731. assert(override_len + u16_buf_len < OVERRIDE_LEN);
  2732. memcpy(override_buffer + override_len, u16_buf, u16_buf_len*sizeof(u16_buf[0]));
  2733. der_locale_array[der_locale_len].overrides_offset = override_len;
  2734. override_len += u16_buf_len;
  2735. /* printf("%s: override_len = %d u16_buf_len = %d\n", cl->name, override_len, u16_buf_len); */
  2736. } else if (!(u16_buf_len > override_len)) {
  2737. assert(mm);
  2738. der_locale_array[der_locale_len].overrides_offset = ((uint16_t *)(mm)) - override_buffer;
  2739. /* printf("%s: memmem found a match with u16_buf_len = %d\n", cl->name, u16_buf_len); */
  2740. }
  2741. do_starter_lists(cl);
  2742. der_locale_array[der_locale_len].undefined_idx = final_index_val0("UNDEFINED");
  2743. #if 0
  2744. assert(der_locale_array[der_locale_len].undefined_idx);
  2745. if (!der_locale_array[der_locale_len].undefined_idx) {
  2746. der_locale_array[der_locale_len].undefined_idx = base_locale_array[base_locale_len].undefined_idx;
  2747. }
  2748. #endif
  2749. if (!der_locale_array[der_locale_len].undefined_idx) {
  2750. error_msg("no UNDEFINED definition for %s", cl->name);
  2751. }
  2752. ++der_locale_len;
  2753. }
  2754. #endif
  2755. #warning handle UNDEFINED idx specially? what if in only some of derived?
  2756. /* base_locale_array[base_locale_len].undefined_idx = final_index_val0("UNDEFINED"); */
  2757. base_locale_array[base_locale_len].undefined_idx = 0;
  2758. hdestroy();
  2759. ++base_locale_len;
  2760. /* if (tnumnodes(cur_base->root_starter_char)) { */
  2761. /* fprintf(stderr, "starter nodes\n"); */
  2762. /* twalk(cur_base->root_starter_char, print_starter_node); */
  2763. /* } */
  2764. }
  2765. static int starter_all_cmp(const void *n1, const void *n2)
  2766. {
  2767. const char *s1 = ((weighted_item_t *) n1)->symbol;
  2768. const char *s2 = ((weighted_item_t *) n2)->symbol;
  2769. colitem_t x;
  2770. colitem_t *p;
  2771. int n;
  2772. /* sort by 1st char ... then inverse for string */
  2773. x.element = NULL;
  2774. if (!is_ucode(s1)) {
  2775. x.string = s1;
  2776. p = tfind(&x, &cur_base->root_colitem, colitem_cmp);
  2777. s1 = (*((colitem_t **) p))->element + 1;
  2778. }
  2779. if (!is_ucode(s2)) {
  2780. x.string = s2;
  2781. p = tfind(&x, &cur_base->root_colitem, colitem_cmp);
  2782. s2 = (*((colitem_t **) p))->element + 1;
  2783. }
  2784. /* <U####>< */
  2785. /* 01234567 */
  2786. assert(is_ucode(s1));
  2787. assert(is_ucode(s2));
  2788. n = strncmp(s1+2, s2+2, 4);
  2789. if (n) {
  2790. return n;
  2791. }
  2792. s1 += 7;
  2793. s2 += 7;
  2794. return strcmp(s2, s1);
  2795. }
  2796. static void print_starter_all_node(const void *ptr, VISIT order, int level)
  2797. {
  2798. const weighted_item_t *w = *(const weighted_item_t **) ptr;
  2799. colitem_t *ci;
  2800. void *p;
  2801. int n;
  2802. colitem_t x;
  2803. if (order == postorder || order == leaf) {
  2804. #if 0
  2805. if ((n = is_ucode(w->symbol)) != 0) {
  2806. printf(" %s\n", w->symbol);
  2807. } else {
  2808. x.string = w->symbol;
  2809. x.element = NULL;
  2810. p = tfind(&x, &cur_base->root_colitem, colitem_cmp);
  2811. assert(p);
  2812. ci = *((colitem_t **) p);
  2813. printf("%s = %s\n", ci->element, w->symbol);
  2814. }
  2815. #else
  2816. printf("%s|", w->symbol);
  2817. /* if ((n = is_ucode(w->symbol)) != 0) { */
  2818. /* printf("\n"); */
  2819. /* } */
  2820. #endif
  2821. }
  2822. }
  2823. static void process_starter_node(const void *ptr, VISIT order, int level)
  2824. {
  2825. const weighted_item_t *w = *(const weighted_item_t **) ptr;
  2826. colitem_t *ci;
  2827. void *p;
  2828. int n;
  2829. colitem_t x;
  2830. const char *s;
  2831. char buf[32];
  2832. /* store index of collation item followed by (unprefixed) nul-terminated string */
  2833. if (order == postorder || order == leaf) {
  2834. if ((n = is_ucode(w->symbol)) != 0) {
  2835. u16_buf[u16_buf_len++] = final_index_val(w->symbol);
  2836. assert(u16_buf[u16_buf_len-1]);
  2837. u16_buf[u16_buf_len++] = 0;
  2838. if (++u16_starter < base_locale_array[base_locale_len].num_starters) {
  2839. u16_buf[u16_starter] = u16_buf_len;
  2840. }
  2841. /* fprintf(stderr, "ucode - %d %d\n", u16_buf[u16_starter-1], u16_buf_len); */
  2842. } else {
  2843. x.string = w->symbol;
  2844. x.element = NULL;
  2845. p = tfind(&x, &cur_base->root_colitem, colitem_cmp);
  2846. assert(p);
  2847. ci = *((colitem_t **) p);
  2848. s = ci->element;
  2849. u16_buf[u16_buf_len++] = final_index_val(w->symbol);
  2850. assert(u16_buf[u16_buf_len-1]);
  2851. assert(*s == '"');
  2852. n = is_ucode(++s);
  2853. /* fprintf(stderr, "s is |%s| with len %d (%d)\n", s, strlen(s), n); */
  2854. assert(n);
  2855. s += n;
  2856. while (*s != '"') {
  2857. n = is_ucode(s);
  2858. assert(n);
  2859. strncpy(buf, s, n+1);
  2860. buf[n] = 0;
  2861. /* fprintf(stderr, "buf is |%s| with len %d (%d)\n", buf, strlen(buf), n); */
  2862. u16_buf[u16_buf_len++] = final_index_val(buf);
  2863. assert(u16_buf[u16_buf_len-1]);
  2864. s += n;
  2865. }
  2866. u16_buf[u16_buf_len++] = 0;
  2867. }
  2868. }
  2869. }
  2870. static void **p_cl_root_starter_all;
  2871. static void complete_starter_node(const void *ptr, VISIT order, int level)
  2872. {
  2873. weighted_item_t w;
  2874. weighted_item_t *p;
  2875. if (order == postorder || order == leaf) {
  2876. w.symbol = *(const char **) ptr;
  2877. w.weight = NULL;
  2878. if (!tfind(&w, p_cl_root_starter_all, starter_all_cmp)) {
  2879. p = xmalloc(sizeof(weighted_item_t));
  2880. p->symbol = w.symbol;
  2881. p->weight = NULL;
  2882. /* fprintf(stderr, "complete_starter_node: %s\n", *(const char **) ptr); */
  2883. if (!tsearch(p, p_cl_root_starter_all, starter_all_cmp)) {
  2884. error_msg("OUT OF MEMORY");
  2885. }
  2886. }
  2887. }
  2888. }
  2889. static void do_starter_lists(col_locale_t *cl)
  2890. {
  2891. ll_item_t *s;
  2892. ll_item_t *h;
  2893. ll_item_t *lli;
  2894. col_locale_t *c;
  2895. colitem_t *ci;
  2896. weighted_item_t *w;
  2897. void *p;
  2898. char buf[32];
  2899. int n;
  2900. colitem_t x;
  2901. void *mm;
  2902. c = cl;
  2903. if (c != cur_base) {
  2904. c = cur_base;
  2905. }
  2906. /* printf("STARTERS %s --------------------\n", cl->name); */
  2907. LOOP:
  2908. for (s = c->section_list ; s ; s = s->next) {
  2909. h = lli = ((section_t *)(s->data))->itm_list;
  2910. if (!lli) {
  2911. continue;
  2912. }
  2913. do {
  2914. if (lli->data_type & DT_WEIGHTED) {
  2915. w = (weighted_item_t *)(lli->data);
  2916. ci = NULL;
  2917. if ((n = is_ucode(w->symbol)) != 0) {
  2918. strcpy(buf, w->symbol);
  2919. } else {
  2920. /* fprintf(stdout, "looking for |%s|\n", w->symbol); */
  2921. x.string = w->symbol;
  2922. x.element = NULL;
  2923. p = tfind(&x, &cur_base->root_colitem, colitem_cmp);
  2924. if (!p) {
  2925. /* fprintf(stderr, "Whoa... processing starters for %s and couldn't find %s\n", */
  2926. /* cl->name, w->symbol); */
  2927. continue;
  2928. }
  2929. ci = *((colitem_t **) p);
  2930. if (!ci->element) { /* just a collating symbol */
  2931. continue;
  2932. }
  2933. assert(ci->element[0] == '"');
  2934. n = is_ucode(ci->element + 1);
  2935. assert(n);
  2936. strncpy(buf, ci->element + 1, n);
  2937. }
  2938. if ((tfind(buf, &cur_base->root_starter_char, sym_cmp)) != NULL) {
  2939. /* fprintf(stdout, "adding from %s: %s", c->name, w->symbol); */
  2940. /* if (ci) { */
  2941. /* fprintf(stdout, " = %s", ci->element); */
  2942. /* } */
  2943. /* fprintf(stdout, "\n"); */
  2944. if (!tsearch(w, &cl->root_starter_all, starter_all_cmp)) {
  2945. error_msg("OUT OF MEMORY");
  2946. }
  2947. }
  2948. }
  2949. } while ((lli = lli->next) != h);
  2950. }
  2951. if (c != cl) {
  2952. c = cl;
  2953. goto LOOP;
  2954. }
  2955. p_cl_root_starter_all = &cl->root_starter_all;
  2956. twalk(cur_base->root_starter_char, complete_starter_node);
  2957. if (cl == cur_base) {
  2958. base_locale_array[base_locale_len].num_starters = tnumnodes(cur_base->root_starter_char);
  2959. }
  2960. #if 0
  2961. printf("\nNow walking tree...\n\n");
  2962. twalk(cl->root_starter_all, print_starter_all_node);
  2963. printf("\n\n");
  2964. #endif
  2965. u16_starter = 0;
  2966. u16_buf[0] = u16_buf_len = base_locale_array[base_locale_len].num_starters;
  2967. twalk(cl->root_starter_all, process_starter_node);
  2968. /* fprintf(stderr, "s=%d n=%d\n", u16_starter, base_locale_array[base_locale_len].num_starters); */
  2969. assert(u16_starter == base_locale_array[base_locale_len].num_starters);
  2970. #if 0
  2971. { int i;
  2972. for (i=0 ; i < u16_buf_len ; i++) {
  2973. fprintf(stderr, "starter %2d: %d - %#06x\n", i, u16_buf[i], u16_buf[i]);
  2974. }}
  2975. #endif
  2976. mm = NULL;
  2977. if (u16_buf_len) {
  2978. /* assert(base_locale_array[base_locale_len].num_starters); */
  2979. if ((u16_buf_len > multistart_len) ||
  2980. !(mm = memmem(multistart_buffer, multistart_len*sizeof(multistart_buffer[0]),
  2981. u16_buf, u16_buf_len*sizeof(u16_buf[0])))
  2982. ) {
  2983. assert(multistart_len + u16_buf_len < MULTISTART_LEN);
  2984. memcpy(multistart_buffer + multistart_len, u16_buf, u16_buf_len*sizeof(u16_buf[0]));
  2985. if (cl == cur_base) {
  2986. base_locale_array[base_locale_len].multistart_offset = multistart_len;
  2987. } else {
  2988. der_locale_array[der_locale_len].multistart_offset = multistart_len;
  2989. }
  2990. multistart_len += u16_buf_len;
  2991. /* fprintf(stderr, "%s: multistart_len = %d u16_buf_len = %d\n", cl->name, multistart_len, u16_buf_len); */
  2992. } else if (!(u16_buf_len > multistart_len)) {
  2993. assert(mm);
  2994. if (cl == cur_base) {
  2995. base_locale_array[base_locale_len].multistart_offset = ((uint16_t *)(mm)) - multistart_buffer;
  2996. } else {
  2997. der_locale_array[der_locale_len].multistart_offset = ((uint16_t *)(mm)) - multistart_buffer;
  2998. }
  2999. /* fprintf(stderr, "%s: memmem found a match with u16_buf_len = %d\n", cl->name, u16_buf_len); */
  3000. }
  3001. } else {
  3002. assert(!base_locale_array[base_locale_len].num_starters);
  3003. }
  3004. /* printf("u16_buf_len = %d\n", u16_buf_len); */
  3005. /* printf("STARTERS %s DONE ---------------\n", cl->name); */
  3006. }
  3007. /* For sorting the blocks of unsigned chars. */
  3008. static size_t nu_val;
  3009. int nu_memcmp(const void *a, const void *b)
  3010. {
  3011. return memcmp(*(unsigned char**)a, *(unsigned char**)b, nu_val * sizeof(tbl_item));
  3012. }
  3013. size_t newopt(tbl_item *ut, size_t usize, int shift, table_data *tbl)
  3014. {
  3015. static int recurse = 0;
  3016. tbl_item *ti[RANGE]; /* table index */
  3017. size_t numblocks;
  3018. size_t blocksize;
  3019. size_t uniq;
  3020. size_t i, j;
  3021. size_t smallest, t;
  3022. tbl_item *ii_save;
  3023. int uniqblock[1 << (8*sizeof(tbl_item) - 1)];
  3024. tbl_item uit[RANGE];
  3025. int shift2;
  3026. if (shift > 15) {
  3027. return SIZE_MAX;
  3028. }
  3029. ii_save = NULL;
  3030. blocksize = 1 << shift;
  3031. numblocks = usize >> shift;
  3032. /* init table index */
  3033. for (i=j=0 ; i < numblocks ; i++) {
  3034. ti[i] = ut + j;
  3035. j += blocksize;
  3036. }
  3037. /* sort */
  3038. nu_val = blocksize;
  3039. qsort(ti, numblocks, sizeof(unsigned char *), nu_memcmp);
  3040. uniq = 1;
  3041. uit[(ti[0]-ut)/blocksize] = 0;
  3042. for (i=1 ; i < numblocks ; i++) {
  3043. if (memcmp(ti[i-1], ti[i], blocksize*sizeof(tbl_item)) < 0) {
  3044. if (++uniq > (1 << (8*sizeof(tbl_item) - 1))) {
  3045. break;
  3046. }
  3047. uniqblock[uniq - 1] = i;
  3048. }
  3049. #if 1
  3050. else if (memcmp(ti[i-1], ti[i], blocksize*sizeof(tbl_item)) > 0) {
  3051. printf("bad sort %i!\n", i);
  3052. abort();
  3053. }
  3054. #endif
  3055. uit[(ti[i]-ut)/blocksize] = uniq - 1;
  3056. }
  3057. smallest = SIZE_MAX;
  3058. shift2 = -1;
  3059. if (uniq <= (1 << (8*sizeof(tbl_item) - 1))) {
  3060. smallest = numblocks + uniq * blocksize;
  3061. if (!recurse) {
  3062. ++recurse;
  3063. for (j=1 ; j < 14 ; j++) {
  3064. if ((numblocks >> j) < 2) break;
  3065. if (tbl) {
  3066. ii_save = tbl->ii;
  3067. tbl->ii = NULL;
  3068. }
  3069. if ((t = newopt(uit, numblocks, j, tbl)) < SIZE_MAX) {
  3070. t += uniq * blocksize;
  3071. }
  3072. if (tbl) {
  3073. tbl->ii = ii_save;
  3074. }
  3075. if (smallest >= t) {
  3076. shift2 = j;
  3077. smallest = t;
  3078. /* if (!tbl->ii) { */
  3079. /* printf("ishift %u tshift %u size %u\n", */
  3080. /* shift2, shift, t); */
  3081. /* } */
  3082. /* } else { */
  3083. /* break; */
  3084. }
  3085. }
  3086. --recurse;
  3087. }
  3088. } else {
  3089. return SIZE_MAX;
  3090. }
  3091. if (tbl->ii) {
  3092. if (recurse) {
  3093. tbl->ii_shift = shift;
  3094. tbl->ii_len = numblocks;
  3095. memcpy(tbl->ii, uit, numblocks*sizeof(tbl_item));
  3096. tbl->ti = tbl->ii + tbl->ii_len;
  3097. tbl->ti_len = uniq * blocksize;
  3098. for (i=0 ; i < uniq ; i++) {
  3099. memcpy(tbl->ti + i * blocksize, ti[uniqblock[i]], blocksize*sizeof(tbl_item));
  3100. }
  3101. } else {
  3102. ++recurse;
  3103. /* printf("setting ishift %u tshift %u\n", shift2, shift); */
  3104. newopt(uit, numblocks, shift2, tbl);
  3105. --recurse;
  3106. tbl->ti_shift = shift;
  3107. tbl->ut_len = uniq * blocksize;
  3108. tbl->ut = tbl->ti + tbl->ti_len;
  3109. for (i=0 ; i < uniq ; i++) {
  3110. memcpy(tbl->ut + i * blocksize, ti[uniqblock[i]], blocksize*sizeof(tbl_item));
  3111. }
  3112. }
  3113. }
  3114. return smallest;
  3115. }
  3116. static const int rule2val[8] = {
  3117. -1,
  3118. (1 << 14), /* forward */
  3119. (2 << 14), /* position */
  3120. (3 << 14), /* forward,position */
  3121. 0, /* backward */
  3122. -1,
  3123. -1,
  3124. -1,
  3125. };
  3126. static int final_index_val_x(const char *s, const char *sym)
  3127. {
  3128. int r;
  3129. if (!(r = final_index_val0(s))) {
  3130. if (!strcmp(s, "IGNORE")) {
  3131. r = 0;
  3132. } else if (!strcmp(s, "..") || !strcmp(sym, "RANGE")) {
  3133. if (*sym == '.') {
  3134. final_index_val(sym); /* make sure it's known */
  3135. }
  3136. r = 0x3fff;
  3137. } else if (!strcmp(s, ".")) {
  3138. r = 0x3ffe;
  3139. } else {
  3140. error_msg("can't find final index: %s", s);
  3141. }
  3142. }
  3143. return r;
  3144. }
  3145. /* store rule2val in 2 high bits and collation index in lower.
  3146. * for sort strings, store (offset from base) + max colindex as index.
  3147. */
  3148. static unsigned int add_rule(weighted_item_t *wi)
  3149. {
  3150. weight_t *w = wi->weight;
  3151. int i, j, r, n;
  3152. uint16_t rbuf[MAX_COLLATION_WEIGHTS];
  3153. uint16_t ws_buf[32];
  3154. void *mm;
  3155. char buf[32];
  3156. const char *s;
  3157. const char *e;
  3158. for (i=0 ; i < MAX_COLLATION_WEIGHTS ; i++) {
  3159. rbuf[i] = rule2val[R_FORWARD]; /* set a default to forward-ignore */
  3160. }
  3161. if (base_locale_array[base_locale_len].num_weights < w->num_weights) {
  3162. base_locale_array[base_locale_len].num_weights = w->num_weights;
  3163. }
  3164. for (i=0 ; i < w->num_weights ; i++) {
  3165. assert(rule2val[(int)(w->rule[i])] >= 0);
  3166. assert(w->colitem[i] && *w->colitem[i]);
  3167. if (*w->colitem[i] == '"') { /* string... */
  3168. s = w->colitem[i] + 1;
  3169. assert(*s == '<');
  3170. n = 0;
  3171. do {
  3172. e = s;
  3173. do {
  3174. if (*e == '/') {
  3175. e += 2;
  3176. continue;
  3177. }
  3178. } while (*e++ != '>');
  3179. assert(((size_t)(e-s) < sizeof(buf)));
  3180. memcpy(buf, s, (size_t)(e-s));
  3181. buf[(size_t)(e-s)] = 0;
  3182. r = final_index_val_x(buf, wi->symbol);
  3183. assert(n + 1 < sizeof(ws_buf)/sizeof(ws_buf[0]));
  3184. ws_buf[n++] = r | rule2val[(int)(w->rule[i])];
  3185. s = e;
  3186. } while (*s != '"');
  3187. ws_buf[n++] = 0; /* terminator */
  3188. mm = memmem(weightstr_buffer, weightstr_len*sizeof(weightstr_buffer[0]),
  3189. ws_buf, n*sizeof(ws_buf[0]));
  3190. if (!mm) {
  3191. assert(weightstr_len + n < WEIGHTSTR_LEN);
  3192. memcpy(weightstr_buffer + weightstr_len, ws_buf, n*sizeof(ws_buf[0]));
  3193. mm = weightstr_buffer + weightstr_len;
  3194. weightstr_len += n;
  3195. }
  3196. r = (((uint16_t *)(mm)) - weightstr_buffer)
  3197. + base_locale_array[base_locale_len].max_col_index + 2;
  3198. assert(r < (1 << 14));
  3199. rbuf[i] = r | rule2val[(int)(w->rule[i])];
  3200. } else { /* item */
  3201. r = final_index_val_x(w->colitem[i], wi->symbol);
  3202. rbuf[i] = r | rule2val[(int)(w->rule[i])];
  3203. }
  3204. }
  3205. for (i=0 ; i < ruletable_len ; i += MAX_COLLATION_WEIGHTS) {
  3206. if (!memcmp(ruletable_buffer + i, rbuf, MAX_COLLATION_WEIGHTS*sizeof(ruletable_buffer[0]))) {
  3207. return i/MAX_COLLATION_WEIGHTS;
  3208. }
  3209. }
  3210. memcpy(ruletable_buffer + ruletable_len, rbuf, MAX_COLLATION_WEIGHTS*sizeof(ruletable_buffer[0]));
  3211. ruletable_len += MAX_COLLATION_WEIGHTS;
  3212. return (ruletable_len / MAX_COLLATION_WEIGHTS)-1;
  3213. }
  3214. static unsigned int add_range_rule(range_item_t *ri)
  3215. {
  3216. weight_t *w = ri->weight;
  3217. int i, j, r, n;
  3218. uint16_t rbuf[MAX_COLLATION_WEIGHTS];
  3219. uint16_t ws_buf[32];
  3220. void *mm;
  3221. char buf[32];
  3222. const char *s;
  3223. const char *e;
  3224. for (i=0 ; i < MAX_COLLATION_WEIGHTS ; i++) {
  3225. rbuf[i] = rule2val[R_FORWARD]; /* set a default to forward-ignore */
  3226. }
  3227. if (base_locale_array[base_locale_len].num_weights < w->num_weights) {
  3228. base_locale_array[base_locale_len].num_weights = w->num_weights;
  3229. }
  3230. for (i=0 ; i < w->num_weights ; i++) {
  3231. assert(rule2val[(int)(w->rule[i])] >= 0);
  3232. assert(w->colitem[i] && *w->colitem[i]);
  3233. if (*w->colitem[i] == '"') { /* string... */
  3234. s = w->colitem[i] + 1;
  3235. assert(*s == '<');
  3236. n = 0;
  3237. do {
  3238. e = s;
  3239. do {
  3240. if (*e == '/') {
  3241. e += 2;
  3242. continue;
  3243. }
  3244. } while (*e++ != '>');
  3245. assert(((size_t)(e-s) < sizeof(buf)));
  3246. memcpy(buf, s, (size_t)(e-s));
  3247. buf[(size_t)(e-s)] = 0;
  3248. r = final_index_val_x(buf, "RANGE");
  3249. assert(n + 1 < sizeof(ws_buf)/sizeof(ws_buf[0]));
  3250. ws_buf[n++] = r | rule2val[(int)(w->rule[i])];
  3251. s = e;
  3252. } while (*s != '"');
  3253. ws_buf[n++] = 0; /* terminator */
  3254. mm = memmem(weightstr_buffer, weightstr_len*sizeof(weightstr_buffer[0]),
  3255. ws_buf, n*sizeof(ws_buf[0]));
  3256. if (!mm) {
  3257. assert(weightstr_len + n < WEIGHTSTR_LEN);
  3258. memcpy(weightstr_buffer + weightstr_len, ws_buf, n*sizeof(ws_buf[0]));
  3259. mm = weightstr_buffer + weightstr_len;
  3260. weightstr_len += n;
  3261. }
  3262. r = (((uint16_t *)(mm)) - weightstr_buffer)
  3263. + base_locale_array[base_locale_len].max_col_index + 2;
  3264. assert(r < (1 << 14));
  3265. rbuf[i] = r | rule2val[(int)(w->rule[i])];
  3266. } else { /* item */
  3267. r = final_index_val_x(w->colitem[i], "RANGE");
  3268. rbuf[i] = r | rule2val[(int)(w->rule[i])];
  3269. }
  3270. }
  3271. for (i=0 ; i < ruletable_len ; i += MAX_COLLATION_WEIGHTS) {
  3272. if (!memcmp(ruletable_buffer + i, rbuf, MAX_COLLATION_WEIGHTS*sizeof(ruletable_buffer[0]))) {
  3273. return i/MAX_COLLATION_WEIGHTS;
  3274. }
  3275. }
  3276. memcpy(ruletable_buffer + ruletable_len, rbuf, MAX_COLLATION_WEIGHTS*sizeof(ruletable_buffer[0]));
  3277. ruletable_len += MAX_COLLATION_WEIGHTS;
  3278. return (ruletable_len / MAX_COLLATION_WEIGHTS)-1;
  3279. }
  3280. #define DUMPn(X) fprintf(stderr, "%10d-%-.20s", base_locale_array[n]. X, #X);
  3281. static void dump_base_locale(int n)
  3282. {
  3283. assert(n < base_locale_len);
  3284. fprintf(stderr, "Base Locale: %s\n", base_locale_array[n].name);
  3285. DUMPn(num_weights);
  3286. DUMPn(ii_shift);
  3287. DUMPn(ti_shift);
  3288. DUMPn(ii_len);
  3289. DUMPn(ti_len);
  3290. DUMPn(max_weight);
  3291. fprintf(stderr, "\n");
  3292. DUMPn(num_col_base);
  3293. DUMPn(max_col_index);
  3294. DUMPn(undefined_idx);
  3295. DUMPn(range_low);
  3296. DUMPn(range_count);
  3297. fprintf(stderr, "\n");
  3298. DUMPn(range_base_weight);
  3299. DUMPn(num_starters);
  3300. fprintf(stderr, "\n");
  3301. DUMPn(range_rule_offset);
  3302. DUMPn(wcs2colidt_offset);
  3303. DUMPn(index2weight_offset);
  3304. fprintf(stderr, "\n");
  3305. DUMPn(index2ruleidx_offset);
  3306. DUMPn(multistart_offset);
  3307. fprintf(stderr, "\n");
  3308. }
  3309. #undef DUMPn
  3310. #define DUMPn(X) fprintf(stderr, "%10d-%s", der_locale_array[n]. X, #X);
  3311. static void dump_der_locale(int n)
  3312. {
  3313. assert(n < der_locale_len);
  3314. fprintf(stderr, "Derived Locale: %s (%.12s)",
  3315. der_locale_array[n].name,
  3316. base_locale_array[der_locale_array[n].base_idx].name);
  3317. DUMPn(base_idx);
  3318. DUMPn(undefined_idx);
  3319. DUMPn(overrides_offset);
  3320. DUMPn(multistart_offset);
  3321. fprintf(stderr, "\n");
  3322. }
  3323. static unsigned long collate_pos;
  3324. static void dump_u16_array(FILE *fp, uint16_t *u, int len, const char *name)
  3325. {
  3326. int i;
  3327. fprintf(fp, "\t/* %8lu %s */\n", collate_pos, name);
  3328. for (i=0 ; i < len ; i++) {
  3329. if (!(i & 7)) {
  3330. fprintf(fp, "\n\t");
  3331. }
  3332. fprintf(fp," %#06x,", (unsigned int)(u[i]));
  3333. }
  3334. fprintf(fp,"\n");
  3335. collate_pos += len;
  3336. }
  3337. #define OUT_U16C(X,N) fprintf(fp,"\t%10d, /* %8lu %s */\n", X, collate_pos++, N);
  3338. static void dump_collate(FILE *fp)
  3339. {
  3340. int n;
  3341. fprintf(fp, "const uint16_t __locale_collate_tbl[] = {\n");
  3342. OUT_U16C(base_locale_len, "numbef of base locales");
  3343. OUT_U16C(der_locale_len, "number of derived locales");
  3344. OUT_U16C(MAX_COLLATION_WEIGHTS, "max collation weights");
  3345. OUT_U16C(index2weight_len, "number of index2{weight|ruleidx} elements");
  3346. OUT_U16C(weightstr_len, "number of weightstr elements");
  3347. OUT_U16C(multistart_len, "number of multistart elements");
  3348. OUT_U16C(override_len, "number of override elements");
  3349. OUT_U16C(ruletable_len, "number of ruletable elements");
  3350. #undef DUMPn
  3351. #define DUMPn(X) fprintf(fp, "\t%10d, /* %8lu %s */\n", base_locale_array[n]. X, collate_pos++, #X);
  3352. for (n=0 ; n < base_locale_len ; n++) {
  3353. unsigned wcs2colidt_offset_low = base_locale_array[n].wcs2colidt_offset & 0xffffU;
  3354. unsigned wcs2colidt_offset_hi = base_locale_array[n].wcs2colidt_offset >> 16;
  3355. fprintf(fp, "\t/* Base Locale %2d: %s */\n", n, base_locale_array[n].name);
  3356. DUMPn(num_weights);
  3357. DUMPn(num_starters);
  3358. DUMPn(ii_shift);
  3359. DUMPn(ti_shift);
  3360. DUMPn(ii_len);
  3361. DUMPn(ti_len);
  3362. DUMPn(max_weight);
  3363. DUMPn(num_col_base);
  3364. DUMPn(max_col_index);
  3365. DUMPn(undefined_idx);
  3366. DUMPn(range_low);
  3367. DUMPn(range_count);
  3368. DUMPn(range_base_weight);
  3369. DUMPn(range_rule_offset);
  3370. DUMPn(index2weight_offset);
  3371. DUMPn(index2ruleidx_offset);
  3372. DUMPn(multistart_offset);
  3373. #undef DUMPn
  3374. #define DUMPn(X) fprintf(fp, "\t%10d, /* %8lu %s */\n", X, collate_pos++, #X);
  3375. DUMPn(wcs2colidt_offset_low);
  3376. DUMPn(wcs2colidt_offset_hi);
  3377. }
  3378. #undef DUMPn
  3379. fprintf(fp, "#define COL_IDX_C %5d\n", 0);
  3380. #define DUMPn(X) fprintf(fp, "\t%10d, /* %8lu %s */\n", der_locale_array[n]. X, collate_pos++, #X);
  3381. for (n=0 ; n < der_locale_len ; n++) {
  3382. fprintf(fp, "#define COL_IDX_%s %5d\n", der_locale_array[n].name, n+1);
  3383. fprintf(fp, "\t/* Derived Locale %4d: %s (%.12s) */\n",
  3384. n, der_locale_array[n].name,
  3385. base_locale_array[der_locale_array[n].base_idx].name);
  3386. DUMPn(base_idx);
  3387. DUMPn(undefined_idx);
  3388. DUMPn(overrides_offset);
  3389. DUMPn(multistart_offset);
  3390. }
  3391. #undef DUMPn
  3392. fprintf(fp, "\n");
  3393. dump_u16_array(fp, index2weight_buffer, index2weight_len, "index2weight");
  3394. dump_u16_array(fp, index2ruleidx_buffer, index2ruleidx_len, "index2ruleidx");
  3395. dump_u16_array(fp, multistart_buffer, multistart_len, "multistart");
  3396. dump_u16_array(fp, override_buffer, override_len, "override");
  3397. dump_u16_array(fp, ruletable_buffer, ruletable_len, "ruletable");
  3398. dump_u16_array(fp, weightstr_buffer, weightstr_len, "weightstr");
  3399. dump_u16_array(fp, wcs2colidt_buffer, wcs2colidt_len, "wcs2colidt");
  3400. fprintf(fp,"}; /* %8lu */\n", collate_pos);
  3401. fprintf(fp,"#define __lc_collate_data_LEN %d\n\n", collate_pos);
  3402. }