HTML Tidy  5.6.0
The HTACG Tidy HTML Project
lexer.h
Go to the documentation of this file.
1 #ifndef __LEXER_H__
2 #define __LEXER_H__
3 
4 /* lexer.h -- Lexer for html parser
5 
6  (c) 1998-2008 (W3C) MIT, ERCIM, Keio University
7  See tidy.h for the copyright notice.
8 
9  Given an input source, it returns a sequence of tokens.
10 
11  GetToken(source) gets the next token
12  UngetToken(source) provides one level undo
13 
14  The tags include an attribute list:
15 
16  - linked list of attribute/value nodes
17  - each node has 2 NULL-terminated strings.
18  - entities are replaced in attribute values
19 
20  white space is compacted if not in preformatted mode
21  If not in preformatted mode then leading white space
22  is discarded and subsequent white space sequences
23  compacted to single space characters.
24 
25  If XmlTags is no then Tag names are folded to upper
26  case and attribute names to lower case.
27 
28  Not yet done:
29  - Doctype subset and marked sections
30 */
31 
32 #ifdef __cplusplus
33 extern "C" {
34 #endif
35 
36 #include "forward.h"
37 
38 /* lexer character types
39 */
40 #define digit 1u
41 #define letter 2u
42 #define namechar 4u
43 #define white 8u
44 #define newline 16u
45 #define lowercase 32u
46 #define uppercase 64u
47 #define digithex 128u
48 
49 
50 /* node->type is one of these values
51 */
52 typedef enum
53 {
68 } NodeType;
69 
70 
71 
72 /* lexer GetToken states
73 */
74 typedef enum
75 {
89 } LexerState;
90 
91 /* ParseDocTypeDecl state constants */
92 typedef enum
93 {
100 
101 /* content model shortcut encoding
102 
103  Descriptions are tentative.
104 */
105 #define CM_UNKNOWN 0
106 /* Elements with no content. Map to HTML specification. */
107 #define CM_EMPTY (1 << 0)
108 /* Elements that appear outside of "BODY". */
109 #define CM_HTML (1 << 1)
110 /* Elements that can appear within HEAD. */
111 #define CM_HEAD (1 << 2)
112 /* HTML "block" elements. */
113 #define CM_BLOCK (1 << 3)
114 /* HTML "inline" elements. */
115 #define CM_INLINE (1 << 4)
116 /* Elements that mark list item ("LI"). */
117 #define CM_LIST (1 << 5)
118 /* Elements that mark definition list item ("DL", "DT"). */
119 #define CM_DEFLIST (1 << 6)
120 /* Elements that can appear inside TABLE. */
121 #define CM_TABLE (1 << 7)
122 /* Used for "THEAD", "TFOOT" or "TBODY". */
123 #define CM_ROWGRP (1 << 8)
124 /* Used for "TD", "TH" */
125 #define CM_ROW (1 << 9)
126 /* Elements whose content must be protected against white space movement.
127  Includes some elements that can found in forms. */
128 #define CM_FIELD (1 << 10)
129 /* Used to avoid propagating inline emphasis inside some elements
130  such as OBJECT or APPLET. */
131 #define CM_OBJECT (1 << 11)
132 /* Elements that allows "PARAM". */
133 #define CM_PARAM (1 << 12)
134 /* "FRAME", "FRAMESET", "NOFRAMES". Used in ParseFrameSet. */
135 #define CM_FRAMES (1 << 13)
136 /* Heading elements (h1, h2, ...). */
137 #define CM_HEADING (1 << 14)
138 /* Elements with an optional end tag. */
139 #define CM_OPT (1 << 15)
140 /* Elements that use "align" attribute for vertical position. */
141 #define CM_IMG (1 << 16)
142 /* Elements with inline and block model. Used to avoid calling InlineDup. */
143 #define CM_MIXED (1 << 17)
144 /* Elements whose content needs to be indented only if containing one
145  CM_BLOCK element. */
146 #define CM_NO_INDENT (1 << 18)
147 /* Elements that are obsolete (such as "dir", "menu"). */
148 #define CM_OBSOLETE (1 << 19)
149 /* User defined elements. Used to determine how attributes wihout value
150  should be printed. */
151 #define CM_NEW (1 << 20)
152 /* Elements that cannot be omitted. */
153 #define CM_OMITST (1 << 21)
154 
155 /* If the document uses just HTML 2.0 tags and attributes described
156 ** it as HTML 2.0 Similarly for HTML 3.2 and the 3 flavors of HTML 4.0.
157 ** If there are proprietary tags and attributes then describe it as
158 ** HTML Proprietary. If it includes the xml-lang or xmlns attributes
159 ** but is otherwise HTML 2.0, 3.2 or 4.0 then describe it as one of the
160 ** flavors of Voyager (strict, loose or frameset).
161 */
162 
163 /* unknown */
164 #define xxxx 0u
165 
166 /* W3C defined HTML/XHTML family document types */
167 #define HT20 1u
168 #define HT32 2u
169 #define H40S 4u
170 #define H40T 8u
171 #define H40F 16u
172 #define H41S 32u
173 #define H41T 64u
174 #define H41F 128u
175 #define X10S 256u
176 #define X10T 512u
177 #define X10F 1024u
178 #define XH11 2048u
179 #define XB10 4096u
180 
181 /* proprietary stuff */
182 #define VERS_SUN 8192u
183 #define VERS_NETSCAPE 16384u
184 #define VERS_MICROSOFT 32768u
185 
186 /* special flag */
187 #define VERS_XML 65536u
188 
189 /* HTML5 */
190 #define HT50 131072u
191 #define XH50 262144u
192 
193 /* compatibility symbols */
194 #define VERS_UNKNOWN (xxxx)
195 #define VERS_HTML20 (HT20)
196 #define VERS_HTML32 (HT32)
197 #define VERS_HTML40_STRICT (H40S|H41S|X10S)
198 #define VERS_HTML40_LOOSE (H40T|H41T|X10T)
199 #define VERS_FRAMESET (H40F|H41F|X10F)
200 #define VERS_XHTML11 (XH11)
201 #define VERS_BASIC (XB10)
202 /* HTML5 */
203 #define VERS_HTML5 (HT50|XH50)
204 
205 /* meta symbols */
206 #define VERS_HTML40 (VERS_HTML40_STRICT|VERS_HTML40_LOOSE|VERS_FRAMESET)
207 #define VERS_IFRAME (VERS_HTML40_LOOSE|VERS_FRAMESET)
208 #define VERS_LOOSE (VERS_HTML20|VERS_HTML32|VERS_IFRAME)
209 #define VERS_EVENTS (VERS_HTML40|VERS_XHTML11)
210 #define VERS_FROM32 (VERS_HTML32|VERS_HTML40|HT50)
211 #define VERS_FROM40 (VERS_HTML40|VERS_XHTML11|VERS_BASIC|VERS_HTML5)
212 #define VERS_XHTML (X10S|X10T|X10F|XH11|XB10|XH50)
213 
214 /* strict */
215 #define VERS_STRICT (VERS_HTML5|VERS_HTML40_STRICT)
216 
217 /* all W3C defined document types */
218 #define VERS_ALL (VERS_HTML20|VERS_HTML32|VERS_FROM40|XH50|HT50)
219 
220 /* all proprietary types */
221 #define VERS_PROPRIETARY (VERS_NETSCAPE|VERS_MICROSOFT|VERS_SUN)
222 
223 /* Linked list of class names and styles
224 */
225 struct _Style;
226 typedef struct _Style TagStyle;
227 
228 struct _Style
229 {
233  TagStyle *next;
234 };
235 
236 
237 /* Linked list of style properties
238 */
239 struct _StyleProp;
240 typedef struct _StyleProp StyleProp;
241 
243 {
246  StyleProp *next;
247 };
248 
249 
250 
251 
252 /* Attribute/Value linked list node
253 */
254 
255 struct _AttVal
256 {
257  AttVal* next;
258  const Attribute* dict;
259  Node* asp;
260  Node* php;
261  int delim;
264 };
265 
266 
267 
268 /*
269  Mosaic handles inlines via a separate stack from other elements
270  We duplicate this to recover from inline markup errors such as:
271 
272  <i>italic text
273  <p>more italic text</b> normal text
274 
275  which for compatibility with Mosaic is mapped to:
276 
277  <i>italic text</i>
278  <p><i>more italic text</i> normal text
279 
280  Note that any inline end tag pop's the effect of the current
281  inline start tag, so that </b> pop's <i> in the above example.
282 */
283 struct _IStack
284 {
285  IStack* next;
286  const Dict* tag; /* tag's dictionary definition */
287  tmbstr element; /* name (NULL for text nodes) */
288  AttVal* attributes;
289 };
290 
291 
292 /* HTML/XHTML/XML Element, Comment, PI, DOCTYPE, XML Decl,
293 ** etc. etc.
294 */
295 
296 struct _Node
297 {
298  Node* parent; /* tree structure */
299  Node* prev;
300  Node* next;
301  Node* content;
302  Node* last;
303 
304  AttVal* attributes;
305  const Dict* was; /* old tag when it was changed */
306  const Dict* tag; /* tag's dictionary definition */
307 
308  tmbstr element; /* name (NULL for text nodes) */
309 
310  uint start; /* start of span onto text array */
311  uint end; /* end of span onto text array */
312  NodeType type; /* TextNode, StartTag, EndTag etc. */
313 
314  uint line; /* current line of document */
315  uint column; /* current column of document */
316 
317  Bool closed; /* true if closed by explicit end tag */
318  Bool implicit; /* true if inferred */
319  Bool linebreak; /* true if followed by a line break */
320 };
321 
322 
323 /*
324  The following are private to the lexer
325  Use NewLexer() to create a lexer, and
326  FreeLexer() to free it.
327 */
328 
329 struct _Lexer
330 {
331  uint lines; /* lines seen */
332  uint columns; /* at start of current token */
333  Bool waswhite; /* used to collapse contiguous white space */
334  Bool pushed; /* true after token has been pushed back */
335  Bool insertspace; /* when space is moved after end tag */
336  Bool excludeBlocks; /* Netscape compatibility */
337  Bool exiled; /* true if moved out of table */
338  Bool isvoyager; /* true if xmlns attribute on html element */
339  uint versions; /* bit vector of HTML versions */
340  uint doctype; /* version as given by doctype (if any) */
341  uint versionEmitted; /* version of doctype emitted */
342  Bool bad_doctype; /* e.g. if html or PUBLIC is missing */
343  uint txtstart; /* start of current node */
344  uint txtend; /* end of current node */
345  LexerState state; /* state of lexer's finite state machine */
346 
347  Node* token; /* last token returned by GetToken() */
348  Node* itoken; /* last duplicate inline returned by GetToken() */
349  Node* root; /* remember root node of the document */
350  Node* parent; /* remember parent node for CDATA elements */
351 
352  Bool seenEndBody; /* true if a </body> tag has been encountered */
353  Bool seenEndHtml; /* true if a </html> tag has been encountered */
354 
355  /*
356  Lexer character buffer
357 
358  Parse tree nodes span onto this buffer
359  which contains the concatenated text
360  contents of all of the elements.
361 
362  lexsize must be reset for each file.
363  */
364  tmbstr lexbuf; /* MB character buffer */
365  uint lexlength; /* allocated */
366  uint lexsize; /* used */
367 
368  /* Inline stack for compatibility with Mosaic */
369  Node* inode; /* for deferring text node */
370  IStack* insert; /* for inferring inline tags */
371  IStack* istack;
372  uint istacklength; /* allocated */
373  uint istacksize; /* used */
374  uint istackbase; /* start of frame */
375 
376  TagStyle *styles; /* used for cleaning up presentation markup */
377 
378  TidyAllocator* allocator; /* allocator */
379 };
380 
381 
382 /* Lexer Functions
383 */
384 
385 /* choose what version to use for new doctype */
386 int TY_(HTMLVersion)( TidyDocImpl* doc );
387 
388 /* everything is allowed in proprietary version of HTML */
389 /* this is handled here rather than in the tag/attr dicts */
390 
391 void TY_(ConstrainVersion)( TidyDocImpl* doc, uint vers );
392 
393 Bool TY_(IsWhite)(uint c);
394 Bool TY_(IsDigit)(uint c);
395 Bool TY_(IsLetter)(uint c);
396 Bool TY_(IsHTMLSpace)(uint c);
397 Bool TY_(IsNewline)(uint c);
398 Bool TY_(IsNamechar)(uint c);
399 Bool TY_(IsXMLLetter)(uint c);
400 Bool TY_(IsXMLNamechar)(uint c);
401 
402 /* Bool IsLower(uint c); */
403 Bool TY_(IsUpper)(uint c);
404 uint TY_(ToLower)(uint c);
405 uint TY_(ToUpper)(uint c);
406 
407 Lexer* TY_(NewLexer)( TidyDocImpl* doc );
408 void TY_(FreeLexer)( TidyDocImpl* doc );
409 
410 /* store character c as UTF-8 encoded byte stream */
411 void TY_(AddCharToLexer)( Lexer *lexer, uint c );
412 
413 /*
414  Used for elements and text nodes
415  element name is NULL for text nodes
416  start and end are offsets into lexbuf
417  which contains the textual content of
418  all elements in the parse tree.
419 
420  parent and content allow traversal
421  of the parse tree in any direction.
422  attributes are represented as a linked
423  list of AttVal nodes which hold the
424  strings for attribute/value pairs.
425 */
426 Node* TY_(NewNode)( TidyAllocator* allocator, Lexer* lexer );
427 
428 
429 /* used to clone heading nodes when split by an <HR> */
430 Node* TY_(CloneNode)( TidyDocImpl* doc, Node *element );
431 
432 /* free node's attributes */
433 void TY_(FreeAttrs)( TidyDocImpl* doc, Node *node );
434 
435 /* doesn't repair attribute list linkage */
436 void TY_(FreeAttribute)( TidyDocImpl* doc, AttVal *av );
437 
438 /* detach attribute from node */
439 void TY_(DetachAttribute)( Node *node, AttVal *attr );
440 
441 /* detach attribute from node then free it
442 */
443 void TY_(RemoveAttribute)( TidyDocImpl* doc, Node *node, AttVal *attr );
444 
445 /*
446  Free document nodes by iterating through peers and recursing
447  through children. Set next to NULL before calling FreeNode()
448  to avoid freeing peer nodes. Doesn't patch up prev/next links.
449  */
450 void TY_(FreeNode)( TidyDocImpl* doc, Node *node );
451 
452 Node* TY_(TextToken)( Lexer *lexer );
453 
454 /* used for creating preformatted text from Word2000 */
455 Node* TY_(NewLineNode)( Lexer *lexer );
456 
457 /* used for adding a &nbsp; for Word2000 */
458 Node* TY_(NewLiteralTextNode)(Lexer *lexer, ctmbstr txt );
459 
460 void TY_(AddStringLiteral)( Lexer* lexer, ctmbstr str );
461 /* void AddStringLiteralLen( Lexer* lexer, ctmbstr str, int len ); */
462 
463 /* find element */
464 Node* TY_(FindDocType)( TidyDocImpl* doc );
465 Node* TY_(FindHTML)( TidyDocImpl* doc );
466 Node* TY_(FindHEAD)( TidyDocImpl* doc );
467 Node* TY_(FindTITLE)(TidyDocImpl* doc);
468 Node* TY_(FindBody)( TidyDocImpl* doc );
469 Node* TY_(FindXmlDecl)(TidyDocImpl* doc);
470 
471 /* Returns containing block element, if any */
472 Node* TY_(FindContainer)( Node* node );
473 
474 /* add meta element for Tidy */
475 Bool TY_(AddGenerator)( TidyDocImpl* doc );
476 
477 uint TY_(ApparentVersion)( TidyDocImpl* doc );
478 
479 ctmbstr TY_(HTMLVersionNameFromCode)( uint vers, Bool isXhtml );
480 
481 uint TY_(HTMLVersionNumberFromCode)( uint vers );
482 
483 Bool TY_(WarnMissingSIInEmittedDocType)( TidyDocImpl* doc );
484 
485 Bool TY_(SetXHTMLDocType)( TidyDocImpl* doc );
486 
487 
488 /* fixup doctype if missing */
489 Bool TY_(FixDocType)( TidyDocImpl* doc );
490 
491 /* ensure XML document starts with <?xml version="1.0"?> */
492 /* add encoding attribute if not using ASCII or UTF-8 output */
493 Bool TY_(FixXmlDecl)( TidyDocImpl* doc );
494 
495 Node* TY_(InferredTag)(TidyDocImpl* doc, TidyTagId id);
496 
497 void TY_(UngetToken)( TidyDocImpl* doc );
498 
499 
500 /*
501  modes for GetToken()
502 
503  MixedContent -- for elements which don't accept PCDATA
504  Preformatted -- white space preserved as is
505  IgnoreMarkup -- for CDATA elements such as script, style
506 */
507 typedef enum
508 {
515 } GetTokenMode;
516 
517 Node* TY_(GetToken)( TidyDocImpl* doc, GetTokenMode mode );
518 
519 void TY_(InitMap)(void);
520 
521 
522 /* create a new attribute */
523 AttVal* TY_(NewAttribute)( TidyDocImpl* doc );
524 
525 /* create a new attribute with given name and value */
526 AttVal* TY_(NewAttributeEx)( TidyDocImpl* doc, ctmbstr name, ctmbstr value,
527  int delim );
528 
529 /* insert attribute at the end of attribute list of a node */
530 void TY_(InsertAttributeAtEnd)( Node *node, AttVal *av );
531 
532 /* insert attribute at the start of attribute list of a node */
533 void TY_(InsertAttributeAtStart)( Node *node, AttVal *av );
534 
535 /*************************************
536  In-line Stack functions
537 *************************************/
538 
539 
540 /* duplicate attributes */
541 AttVal* TY_(DupAttrs)( TidyDocImpl* doc, AttVal* attrs );
542 
543 /*
544  push a copy of an inline node onto stack
545  but don't push if implicit or OBJECT or APPLET
546  (implicit tags are ones generated from the istack)
547 
548  One issue arises with pushing inlines when
549  the tag is already pushed. For instance:
550 
551  <p><em>text
552  <p><em>more text
553 
554  Shouldn't be mapped to
555 
556  <p><em>text</em></p>
557  <p><em><em>more text</em></em>
558 */
559 void TY_(PushInline)( TidyDocImpl* doc, Node* node );
560 
561 /* pop inline stack */
562 void TY_(PopInline)( TidyDocImpl* doc, Node* node );
563 
564 Bool TY_(IsPushed)( TidyDocImpl* doc, Node* node );
565 Bool TY_(IsPushedLast)( TidyDocImpl* doc, Node *element, Node *node );
566 
567 /*
568  This has the effect of inserting "missing" inline
569  elements around the contents of blocklevel elements
570  such as P, TD, TH, DIV, PRE etc. This procedure is
571  called at the start of ParseBlock. when the inline
572  stack is not empty, as will be the case in:
573 
574  <i><h1>italic heading</h1></i>
575 
576  which is then treated as equivalent to
577 
578  <h1><i>italic heading</i></h1>
579 
580  This is implemented by setting the lexer into a mode
581  where it gets tokens from the inline stack rather than
582  from the input stream.
583 */
584 int TY_(InlineDup)( TidyDocImpl* doc, Node *node );
585 
586 /*
587  defer duplicates when entering a table or other
588  element where the inlines shouldn't be duplicated
589 */
590 void TY_(DeferDup)( TidyDocImpl* doc );
591 Node* TY_(InsertedToken)( TidyDocImpl* doc );
592 
593 /* stack manipulation for inline elements */
594 Bool TY_(SwitchInline)( TidyDocImpl* doc, Node* element, Node* node );
595 Bool TY_(InlineDup1)( TidyDocImpl* doc, Node* node, Node* element );
596 
597 #ifdef __cplusplus
598 }
599 #endif
600 
601 
602 #endif /* __LEXER_H__ */
uint lexlength
Definition: lexer.h:365
Definition: lexer.h:59
uint doctype
Definition: lexer.h:340
Definition: lexer.h:63
Definition: lexer.h:77
uint end
Definition: lexer.h:311
uint lines
Definition: lexer.h:331
Bool closed
Definition: lexer.h:317
Bool seenEndBody
Definition: lexer.h:352
uint txtend
Definition: lexer.h:344
Definition: lexer.h:513
Definition: lexer.h:85
Node * next
Definition: lexer.h:300
Bool linebreak
Definition: lexer.h:319
tmbstr attribute
Definition: lexer.h:262
TidyAllocator * allocator
Definition: lexer.h:378
tmbstr tag
Definition: lexer.h:230
AttVal * attributes
Definition: lexer.h:288
tmbstr properties
Definition: lexer.h:232
NodeType
Definition: lexer.h:52
LexerState state
Definition: lexer.h:345
int delim
Definition: lexer.h:261
const tmbchar * ctmbstr
Definition: tidyplatform.h:594
StyleProp * next
Definition: lexer.h:246
AttVal * next
Definition: lexer.h:257
uint column
Definition: lexer.h:315
ParseDocTypeDeclState
Definition: lexer.h:92
Definition: lexer.h:97
Bool pushed
Definition: lexer.h:334
const Dict * was
Definition: lexer.h:305
NodeType type
Definition: lexer.h:312
Definition: lexer.h:88
Definition: lexer.h:55
uint line
Definition: lexer.h:314
Definition: lexer.h:509
Definition: lexer.h:242
tmbstr lexbuf
Definition: lexer.h:364
Node * parent
Definition: lexer.h:298
Definition: lexer.h:54
Node * content
Definition: lexer.h:301
Definition: lexer.h:67
Definition: lexer.h:81
tmbstr tag_class
Definition: lexer.h:231
Definition: lexer.h:329
Definition: lexer.h:83
Definition: lexer.h:87
Definition: lexer.h:296
TidyTagId
Known HTML element types.
Definition: tidyenum.h:852
uint start
Definition: lexer.h:310
Bool seenEndHtml
Definition: lexer.h:353
Bool bad_doctype
Definition: lexer.h:342
Node * token
Definition: lexer.h:347
uint istackbase
Definition: lexer.h:374
Node * inode
Definition: lexer.h:369
uint lexsize
Definition: lexer.h:366
Bool
Definition: tidyplatform.h:631
Definition: lexer.h:283
Definition: lexer.h:66
Definition: lexer.h:60
uint versions
Definition: lexer.h:339
Bool isvoyager
Definition: lexer.h:338
Bool implicit
Definition: lexer.h:318
Definition: lexer.h:76
Definition: lexer.h:78
IStack * insert
Definition: lexer.h:370
Definition: lexer.h:95
Bool exiled
Definition: lexer.h:337
tmbchar * tmbstr
Definition: tidyplatform.h:593
Definition: lexer.h:80
uint istacklength
Definition: lexer.h:372
const Dict * tag
Definition: lexer.h:286
Definition: lexer.h:56
TagStyle * next
Definition: lexer.h:233
Definition: lexer.h:514
uint txtstart
Definition: lexer.h:343
Node * prev
Definition: lexer.h:299
unsigned int uint
Definition: tidyplatform.h:554
Definition: lexer.h:96
LexerState
Definition: lexer.h:74
uint columns
Definition: lexer.h:332
uint versionEmitted
Definition: lexer.h:341
Definition: lexer.h:511
IStack * next
Definition: lexer.h:285
Definition: lexer.h:84
tmbstr name
Definition: lexer.h:244
Definition: lexer.h:65
Node * root
Definition: lexer.h:349
Definition: lexer.h:94
Definition: lexer.h:64
Definition: lexer.h:82
Node * asp
Definition: lexer.h:259
Definition: lexer.h:510
Definition: lexer.h:255
uint istacksize
Definition: lexer.h:373
TagStyle * styles
Definition: lexer.h:376
Definition: lexer.h:512
const Attribute * dict
Definition: lexer.h:258
tmbstr value
Definition: lexer.h:245
Definition: lexer.h:98
#define TY_(str)
Definition: forward.h:23
Definition: lexer.h:228
Bool insertspace
Definition: lexer.h:335
Bool waswhite
Definition: lexer.h:333
Definition: lexer.h:86
IStack * istack
Definition: lexer.h:371
Definition: lexer.h:62
Definition: lexer.h:57
Node * last
Definition: lexer.h:302
Definition: lexer.h:61
Definition: lexer.h:58
tmbstr element
Definition: lexer.h:287
Definition: lexer.h:79
Node * php
Definition: lexer.h:260
GetTokenMode
Definition: lexer.h:507
Node * itoken
Definition: lexer.h:348
Bool excludeBlocks
Definition: lexer.h:336