/* ** Unicode Bidirectional Algorithm ** Reference Implementation ** Copyright 2015, Unicode, Inc. All rights reserved. ** Unpublished rights reserved under U.S. copyright laws. */ /* * bidirefp.h * * Private types, constants, and declarations used by bidiref. */ /* * Change history: * * 2013-Jun-02 kenw Created. * 2013-Jun-05 kenw Added diretyBit to UBACONTEXT. * 2013-Jun-19 kenw Add declaration for br_ErrPrint. * 2013-Jul-08 kenw Move BR_MAXINPUTLEN to bidiref.h. * 2014-Apr-17 kenw Add accelerator flags to BIDIUNIT. * 2015-Jun-05 kenw Set MAXPAIRINGDEPTH to 63 for UBA 8.0. * 2015-Aug-19 kenw Add CJK Extension E range. * 2015-Dec-04 kenw Bumped up bracket stack size by one. */ #ifndef __BIDIREFP_LOADED #define __BIDIREFP_LOADED #ifdef __cplusplus extern "C" { #endif #include "bidiref.h" /* * Constants defining Unihan ranges. */ #define CJK_URO_FIRST (0x4E00) #define CJK_URO_LAST (0x9FCC) #define CJK_EXTA_FIRST (0x3400) #define CJK_EXTA_LAST (0x4DB5) #define CJK_EXTB_FIRST (0x20000) #define CJK_EXTB_LAST (0x2A6D6) #define CJK_EXTC_FIRST (0x2A700) #define CJK_EXTC_LAST (0x2B734) #define CJK_EXTD_FIRST (0x2B740) #define CJK_EXTD_LAST (0x2B81D) #define CJK_EXTE_FIRST (0x2B820) #define CJK_EXTE_LAST (0x2CEA1) /* * Enumerated types for relevant Unicode properties, * to make reference to them easier to understand in the algorithm. */ typedef enum { BIDI_None, BIDI_Unknown, BIDI_L, /* 0x00 strong: left-to-right (bc=L) */ BIDI_ON, /* 0x01 neutral: Other Neutral (bc=ON) */ BIDI_R, /* 0x02 strong: right-to-left (bc=R) */ BIDI_EN, /* 0x03 weak: European Number (bc=EN) */ BIDI_ES, /* 0x04 weak: European Number Separator (bc=ES) */ BIDI_ET, /* 0x05 weak: European Number Terminator (bc=ET) */ BIDI_AN, /* 0x06 weak: Arabic Number (bc=AN) */ BIDI_CS, /* 0x07 weak: Common Number Separator (bc=CS) */ BIDI_B, /* 0x08 neutral: Paragraph Separator (bc=B) */ BIDI_S, /* 0x09 neutral: Segment Separator (bc=S) */ BIDI_WS, /* 0x0A neutral: White Space (bc=WS) */ BIDI_AL, /* 0x0B strong: right-to-left Arabic (bc=AL) */ BIDI_NSM, /* 0x0C weak: non-spacing mark (bc=NSM) */ BIDI_BN, /* 0x0D weak: boundary neutral (bc=BN) */ BIDI_PDF, /* 0x0E format: pop directional formatting (bc=PDF) */ BIDI_LRE, /* format: left-to-right embedding */ BIDI_LRO, /* format: left-to-right override */ BIDI_RLE, /* format: right-to-left embedding */ BIDI_RLO, /* format: right-to-left override */ BIDI_LRI, /* format: left-to-right isolate */ BIDI_RLI, /* format: right-to-left isolate */ BIDI_FSI, /* format: first strong isolate */ BIDI_PDI, /* format: pop directional isolate */ BIDI_MAX } BIDIPROP; typedef enum { BPT_O, BPT_C, BPT_None } BPTPROP; /* * File format defines. * * These defines can be used to distinguish between formats * of data to be read in. They are used to branch the * parsing functions in brinput.c. */ #define FORMAT_A (0) /* Format used in BidiCharacterTest.txt */ #define FORMAT_B (1) /* Format used in BidiTest.txt */ /* * ALGORITHM_STATE is used to store how far along in * algorithm the data has been processed. This can be * used to help the debug display to show relevant * information. */ typedef enum { State_Unitialized, /* context allocated, but no data */ State_Initialized, /* test case data read in and parsed */ State_P3Done, /* rules done through P3 */ State_X8Done, /* rules done through X8 */ State_X9Done, /* rules done through X9 */ State_RunsDone, /* rules done through X10 part 1: runs are identified */ State_X10Done, /* rules done through X10: runs & seqs are identified */ State_W1Done, /* rules done through W1: combining marks are resolved */ State_W2Done, /* rules done through W2: */ State_W3Done, /* rules done through W3: */ State_W4Done, /* rules done through W4: */ State_W5Done, /* rules done through W5: */ State_W6Done, /* rules done through W6: */ State_W7Done, /* rules done through W7: weak types are resolved */ State_N0Done, /* rules done through N0: bracket pairs are resolved */ State_N1Done, /* rules done through N1: */ State_N2Done, /* rules done through N2: neutral types are resolved */ State_I2Done, /* rules done through I2: implicit levels are resolved */ State_L1Done, /* rules done through L1: trailing whitespace resolved */ State_L2Done, /* rules done through L2: reordering data available */ State_Complete /* finished application of rules: ready for checking */ } ALGORITHM_STATE; /* * BIDIPROPDATA * * This struct is used to store property information * about characters and code points. * * Only limited property data is stored -- just the data * needed for running the UBA algorithm. */ typedef struct { int han; BIDIPROP bidivalue; U_Int_32 bpb; BPTPROP bpt; } BIDIDATA; typedef BIDIDATA *BDPTR; #define BPB_None (0xFFFFFFFF) typedef enum { Dir_LTR, Dir_RTL, Dir_Auto, Dir_Unknown } Paragraph_Direction; typedef enum { Override_Neutral, Override_LTR, Override_RTL } D_Override_Status; /* * A single status stack definition is shared by both the * UBA62 and UBA63 reference implementations. The * UBA62 implementation makes no use of the isolate_status * field. * * The status stack is used by rules X1-X8. */ typedef struct { int embedding_level; D_Override_Status override_status; /* direction */ int isolate_status; /* boolean */ } STATUSSTACKELEMENT; typedef STATUSSTACKELEMENT *STACKPTR; /* * The maximum_depth for embedding for UBA62. */ #define MAX_DEPTH_62 61 /* * The maximum_depth for embedding for UBA63. */ #define MAX_DEPTH_63 125 /* * Typedefs used in the bracket pairing algorithm in rule N0. */ typedef struct { U_Int_32 bracket; int pos; } BRACKETSTACKELEMENT; typedef BRACKETSTACKELEMENT *BRACKETSTACKPTR; typedef struct Pairing_Element_Struct { struct Pairing_Element_Struct *next; int openingpos; int closingpos; } PAIRINGELEMENT; typedef PAIRINGELEMENT *PAIRINGPTR; /* * MAXPAIRINGDEPTH is used to declare the bracket pairing stack. * * For UBA 8.0 and subsequent versions, this depth is specified * to be exactly 63. (See brrule.c, where the stack allocation is * made as MAXPAIRINGDEPTH +1.) * * For UBA 6.3 (and 7.0), these depth was implementation-dependent, * and not specified by UAX #9. For the 6.3 and 7.0 versions of * bidiref it was set to half the MAXINPUTLEN, because in principle an * input string could consist *only* of brackets to pair up: "(({{[]}}))", * etc., in which case the maximum possible set of pairs would be * half the text length. * * Starting with the 8.0 version of bidiref, however, 63 is used * also for testing the 6.3 and 7.0 versions of UBA, and for * consistency, the stack full error handling is also based * on the 8.0 specification. */ #define MAXPAIRINGDEPTH (63) /* * Provide a reasonable length for input-handling buffers, based * on MAXINPUTLEN, which limits how many code points the * implementation will handle for an input string. */ #define BUFFERLEN (5 * BR_MAXINPUTLEN) /* for data chunks parsed from input */ #define RAWBUFFERLEN (2 * BUFFERLEN) /* full unparsed input line */ /* * BIDIUNIT * * This struct is the primitive manipulated by * the UBA reference implementation. It is used * to construct a vector in the UBACONTEXT, * containing the character data (stored as UTF-32), * the looked-up property data, so that the UBA, * which works primarily on the basis of the Bidi_Class * values, has the correct input, along with the * level and order information. * * Starting with Version 2.0 of bidiref, a number of * accelerator flags are added, to speed up checking of * common combinations of Bidi_Class values. Each flag * is conceptually a Boolean, and could be stored as a bit * in a bitfield, but in this reference implementation * is simply stored as another int field in the BIDIUNIT. * These values are all initialized when the BIDIUNIT * vector is initialized for a testcase string. Some need * to be reset, whenever a rule changes the current bc * value for a character. */ typedef struct { U_Int_32 c; /* character value, stored as UTF-32 */ BIDIPROP bc; /* current Bidi_Class value */ BIDIPROP orig_bc; /* store original value for Bidi_Class */ int bc_numeric; /* accelerator for bc = (AN or EN) */ int bc_isoinit; /* accelerator for bc = (LRI or RLI or FSI) */ U_Int_32 bpb; /* bidi paired bracket property */ BPTPROP bpt; /* bidi paired bracket type property */ int level; /* current embedding level */ int expLevel; /* store expected level for test case */ int order; /* store position for reordering */ int order2; /* spare order array used in reversal algorithm */ } BIDIUNIT; typedef BIDIUNIT *BIDIUNITPTR; #define NOLEVEL (-1) /* * A text chain is simply an array of pointers to BIDIUNITs. * * This data structure is introduced to help in the abstraction * of rule application between UBA62 (and earlier), where * rules apply to level runs, and UBA63 (and later), where * rules apply to isolating run sequences. Because * isolating run sequences may contain indefinite lists * of discontiguous level runs, it is difficult to * specify rule application directly to the isolating * run sequences -- the concept of previous character * and next character in the sequence gets rather complex * and makes the implementation difficult. * * One possible approach to the isolating run sequences * would be to simply clone the entire text into * allocated contiguous BIDIUNIT vectors for each * isolating run sequence identified. However, that makes * bookkeeping in the original input structure a bit more * complex. Instead, whenever an isolating run sequence * is identified, a contiguous text chain is allocated * and initialized instead. This keeps all the data * in the original input structure, and simply embodies * the traversal order (and prior and next relation) * needed for rule application. * * So that the rule application can be uniform, a * text chain is also allocated for level runs, as well. * * The text chain is hung off the "textChain" field in * both the BIDIRUN and the ISOLATING_RUN_SEQUENCE * struct definitions. The "len" field defines the * length of that array. */ /* * BIDIRUN * * Once embedding levels are determined, the UBA * treats each contiguous sequence at the same level * as a distinct run. * * To simplify the processing of runs, a list of runs * is constructed as a linked list, which hangs off * the UBACONTEXT. * * Each BIDIRUN consists of pointers to the first * and last BIDIUNIT of the run, and then additional * information calculated during the X10 rule, when * the runs are identified. * * This may not be the most efficient approach to * an implementation, but it makes it much easier * to express the X10 rule and subsequent rules which * process the runs individually. * * The seqID is only used in processing run lists * into isolating run sequence lists in UBA63. * It is initialized to zero. During processing * to identify isolating run sequences, it is set * to the sequence id of the isolating run sequence * that a run is assigned to. */ typedef struct BidiRunStruct { struct BidiRunStruct *next; /* next run */ BIDIUNITPTR first; /* first element of this run */ BIDIUNITPTR last; /* last element of this run */ int len; /* explicit len, to simplify processing */ int runID; /* stamp run with id for debugging */ int seqID; /* isolating run sequence id */ int level; /* embedding level of this run */ BIDIPROP sor; /* direction of start of run: L or R */ BIDIPROP eor; /* direction of end of run: L or R */ BIDIUNITPTR *textChain; /* constructed text chain */ } BIDIRUN; typedef BIDIRUN *BIDIRUNPTR; /* * The BidiRunListStruct abstracts the creation of * a list of bidi runs for attachment to the isolating * run lists. Instead of duplicating all the run information * into that list, the list consists just of pointers to * the already allocated basic list of runs. */ typedef struct BidiRunListStruct { struct BidiRunListStruct *next; BIDIRUNPTR run; } BIDIRUNLISTELEMENT; typedef BIDIRUNLISTELEMENT *BIDIRUNLISTPTR; /* * ISOLATING_RUN_SEQUENCE * * This is a concept introduced in UBA63. * * Essentially it consists of an ordered sequence of * bidi runs, as defined in BD13. * * It is implemented here by attaching the list of runs * associated with this particular isolating run sequence. * The attached list just contains pointers to each of * the relevant BIDIRUN structs in the already constructed * sequential list of runs attached to the UBACONTEXT. * * All runs associated with a single isolating run sequence * are at the same level, so that level can be stored * in the isolating run sequence struct for ease of access. * * Each isolating run sequence has a start of sequence (sos) * and end of sequence (eos) directional value assigned. * These are calculated based on the sor and eor values * for the associated list of runs, but again, are stored * in the isolating run sequence struct for ease of access. */ typedef struct IRSequenceStruct { struct IRSequenceStruct *next; /* next sequence */ BIDIRUNLISTPTR theRuns; /* list of runs in this sequence */ BIDIRUNLISTPTR lastRun; /* for list appending */ int len; /* explicit len, to simplify processing */ int seqID; /* stamp seq with id for debugging */ int level; /* embedding level of this seq */ BIDIPROP sos; /* direction of start of seq: L or R */ BIDIPROP eos; /* directino of end of seq: L or R */ BIDIUNITPTR *textChain; /* constructed text chain */ } ISOLATING_RUN_SEQUENCE; typedef ISOLATING_RUN_SEQUENCE *IRSEQPTR; /* * UBACONTEXT * * This struct is used to store all context associated * with the bidi reference UBA processing, including * input, expected test output, * and the constructed runs and other intermediate data. * * theText, testLen, paragraphDirection are input. * theRuns and paragraphEmbeddingLevel are calculated. * expEmbeddingLevel, expOrder are parsed * from the testcase data and checked against * calculated values. * For simplicity, the expected levels data parsed * from the test case are stored with theText. * * Starting from Version 2.0, theText pointer is * allocated statically, simply pointing to a static * buffer of BR_MAXINPUTLEN length, to cut down on * repeated dynamic allocations during long testcase * runs. */ typedef struct { ALGORITHM_STATE state; /* track state */ int dirtyBit; /* used for debug output control */ int testId; /* used for tagging trace output */ Paragraph_Direction paragraphDirection; /* input */ int paragraphEmbeddingLevel; /* calculated */ int textLen; /* input */ BIDIUNITPTR theText; /* input */ BIDIRUNPTR theRuns; /* calculated */ BIDIRUNPTR lastRun; /* for list appending */ IRSEQPTR theSequences; /* calculated: UBA63 only */ IRSEQPTR lastSequence; /* for list appending */ int expEmbeddingLevel; /* expected test result */ char *expOrder; /* expected test result */ } UBACONTEXT; typedef UBACONTEXT *UBACTXTPTR; /* * BIDI_RULE_TYPE * * An enumerated type which facilitates lookup of * rule callbacks and other data by rule type. */ typedef enum { UBA_RULE_W1, UBA_RULE_W2, UBA_RULE_W3, UBA_RULE_W4, UBA_RULE_W5, UBA_RULE_W6, UBA_RULE_W7, UBA_RULE_N0, UBA_RULE_N1, UBA_RULE_N2, UBA_RULE_Last } BIDI_RULE_TYPE; /* * RULE_CONTEXT * * A struct used to package up parameter information * for a class of rules, to make parameter passing * and function prototype neater. */ typedef struct { BIDIUNITPTR *textChain; int len; int level; BIDIPROP sot; BIDIPROP eot; } BIDI_RULE_CONTEXT; typedef BIDI_RULE_CONTEXT *BIDIRULECTXTPTR; /* * BIDIRUN_RULE_FPTR * * Function type of a UBA rule which operates on * a single text chain. * * This type is used to better abstract the bidi rule * dispatch mechanism, to distinguish version-specific * operations to extract level runs from the logic * which then applies to that run. */ typedef int (*BIDIRUN_RULE_FPTR)( BIDIRULECTXTPTR brp ); typedef struct { BIDIRUN_RULE_FPTR ruleMethod; /* actual function callback for rule */ char *ruleLabel; /* printable label for the function */ char *ruleNumber; /* W1, W2, ... */ char *ruleError; } RULE_TBL_ELEMENT; typedef RULE_TBL_ELEMENT *RTEPTR; /* * Private Function Declarations */ /* * Exported from libbidir library */ extern int Trace ( U_Int_32 traceValue ); extern void TraceOffAll ( void ); extern int GetFileFormat ( void ); extern void SetFileFormat ( int format ); extern int GetUBAVersion ( void ); extern void SetUBAVersion ( int version ); extern int convertHexToInt ( U_Int_32 *dest, char *source ); extern int br_Evaluate ( char *data, int *outval ); extern char *copyField ( char *dest, char *src ); extern char *copySubField ( char *dest, char *src ); extern void br_ErrPrint ( char *errString ); extern int br_GetBCFromLabel ( char* src ); extern BIDIPROP br_GetBC ( U_Int_32 c ); extern BPTPROP br_GetBPT ( U_Int_32 c ); extern U_Int_32 br_GetBPB ( U_Int_32 c ); extern int br_InitTable ( int version ); extern int br_UBA ( UBACTXTPTR ctxt ); extern int br_Check ( UBACTXTPTR ctxt ); /* * Exported from brinput.c for use in bidiref.c */ extern int br_RunStaticTest ( void ); extern int br_RunFileTests ( int version, char *input ); /* * Exported from brtable.c for use in brtest.c */ extern int br_GetInitDone ( void ); #ifdef __cplusplus } #endif #endif /* __BIDIREFP_LOADED */