Plan 9 from Bell Labs’s /usr/web/sources/extra/i/lex.c

Copyright © 2021 Plan 9 Foundation.
Distributed under the MIT License.
Download the Plan 9 distribution.


#include "i.h"

enum {
	EOF = -2,
	EOB = -1
};

#define ISNAMCHAR(c)	((c)<256 && (isalpha(c) || isdigit(c) || (c) == '-' || (c) == '.'))

// HTML 4.0 tag names.
// Keep sorted, and in correspondence with enum in i.h.
Rune* tagnames[] = {
	L" ",
	L"!",
	L"a", 
	L"abbr",
	L"acronym",
	L"address",
	L"applet", 
	L"area",
	L"b",
	L"base",
	L"basefont",
	L"bdo",
	L"big",
	L"blink",
	L"blockquote",
	L"body",
	L"bq",
	L"br",
	L"button",
	L"caption",
	L"center",
	L"cite",
	L"code",
	L"col",
	L"colgroup",
	L"dd",
	L"del",
	L"dfn",
	L"dir",
	L"div",
	L"dl",
	L"dt",
	L"em",
	L"fieldset",
	L"font",
	L"form",
	L"frame",
	L"frameset",
	L"h1",
	L"h2",
	L"h3",
	L"h4",
	L"h5",
	L"h6",
	L"head",
	L"hr",
	L"html",
	L"i",
	L"iframe",
	L"img",
	L"input",
	L"ins",
	L"isindex",
	L"kbd",
	L"label",
	L"legend",
	L"li",
	L"link",
	L"map",
	L"menu",
	L"meta",
	L"nobr",
	L"noframes",
	L"noscript",
	L"object",
	L"ol",
	L"optgroup",
	L"option",
	L"p",
	L"param",
	L"pre",
	L"q",
	L"s",
	L"samp",
	L"script",
	L"select",
	L"small",
	L"span",
	L"strike",
	L"strong",
	L"style",
	L"sub",
	L"sup",
	L"table",
	L"tbody",
	L"td",
	L"textarea",
	L"tfoot",
	L"th",
	L"thead",
	L"title",
	L"tr",
	L"tt",
	L"u",
	L"ul",
	L"var"
};

// HTML 4.0 attribute names.
// Keep sorted, and in correspondence with enum in i.h.
Rune* attrnames[] = {
	L"abbr",
	L"accept-charset",
	L"access-key",
	L"action",
	L"align",
	L"alink",
	L"alt",
	L"archive",
	L"axis",
	L"background",
	L"bgcolor",
	L"border",
	L"cellpadding",
	L"cellspacing",
	L"char",
	L"charoff",
	L"charset",
	L"checked",
	L"cite",
	L"class",
	L"classid",
	L"clear",
	L"code",
	L"codebase",
	L"codetype",
	L"color",
	L"cols",
	L"colspan",
	L"compact",
	L"content",
	L"coords",
	L"data",
	L"datetime",
	L"declare",
	L"defer",
	L"dir",
	L"disabled",
	L"enctype",
	L"face",
	L"for",
	L"frame",
	L"frameborder",
	L"headers",
	L"height",
	L"href",
	L"hreflang",
	L"hspace",
	L"http-equiv",
	L"id",
	L"ismap",
	L"label",
	L"lang",
	L"link",
	L"longdesc",
	L"marginheight",
	L"marginwidth",
	L"maxlength",
	L"media",
	L"method",
	L"multiple",
	L"name",
	L"nohref",
	L"noresize",
	L"noshade",
	L"nowrap",
	L"object",
	L"onblur",
	L"onchange",
	L"onclick",
	L"ondblclick",
	L"onfocus",
	L"onkeypress",
	L"onkeyup",
	L"onload",
	L"onmousedown",
	L"onmousemove",
	L"onmouseout",
	L"onmouseover",
	L"onmouseup",
	L"onreset",
	L"onselect",
	L"onsubmit",
	L"onunload",
	L"profile",
	L"prompt",
	L"readonly",
	L"rel",
	L"rev",
	L"rows",
	L"rowspan",
	L"rules",
	L"scheme",
	L"scope",
	L"scrolling",
	L"selected",
	L"shape",
	L"size",
	L"span",
	L"src",
	L"standby",
	L"start",
	L"style",
	L"summary",
	L"tabindex",
	L"target",
	L"text",
	L"title",
	L"type",
	L"usemap",
	L"valign",
	L"value",
	L"valuetype",
	L"version",
	L"vlink",
	L"vspace",
	L"width"
};


// Character entity to unicode character number map.
// Keep sorted by name.
StringInt	chartab[142]= {
	{L"AElig", 198},
	{L"Aacute", 193},
	{L"Acirc", 194},
	{L"Agrave", 192},
	{L"Aring", 197},
	{L"Atilde", 195},
	{L"Auml", 196},
	{L"Ccedil", 199},
	{L"ETH", 208},
	{L"Eacute", 201},
	{L"Ecirc", 202},
	{L"Egrave", 200},
	{L"Euml", 203},
	{L"Iacute", 205},
	{L"Icirc", 206},
	{L"Igrave", 204},
	{L"Iuml", 207},
	{L"Ntilde", 209},
	{L"Oacute", 211},
	{L"Ocirc", 212},
	{L"Ograve", 210},
	{L"Oslash", 216},
	{L"Otilde", 213},
	{L"Ouml", 214},
	{L"THORN", 222},
	{L"Uacute", 218},
	{L"Ucirc", 219},
	{L"Ugrave", 217},
	{L"Uuml", 220},
	{L"Yacute", 221},
	{L"aacute", 225},
	{L"acirc", 226},
	{L"acute", 180},
	{L"aelig", 230},
	{L"agrave", 224},
	{L"alpha", 945},
	{L"amp", 38},
	{L"aring", 229},
	{L"atilde", 227},
	{L"auml", 228},
	{L"beta", 946},
	{L"brvbar", 166},
	{L"ccedil", 231},
	{L"cdots", 8943},
	{L"cedil", 184},
	{L"cent", 162},
	{L"chi", 967},
	{L"copy", 169},
	{L"curren", 164},
	{L"ddots", 8945},
	{L"deg", 176},
	{L"delta", 948},
	{L"divide", 247},
	{L"eacute", 233},
	{L"ecirc", 234},
	{L"egrave", 232},
	{L"emdash", 8212},
	{L"emsp", 8195},
	{L"endash", 8211},
	{L"ensp", 8194},
	{L"epsilon", 949},
	{L"eta", 951},
	{L"eth", 240},
	{L"euml", 235},
	{L"frac12", 189},
	{L"frac14", 188},
	{L"frac34", 190},
	{L"gamma", 947},
	{L"gt", 62},
	{L"iacute", 237},
	{L"icirc", 238},
	{L"iexcl", 161},
	{L"igrave", 236},
	{L"iota", 953},
	{L"iquest", 191},
	{L"iuml", 239},
	{L"kappa", 954},
	{L"lambda", 955},
	{L"laquo", 171},
	{L"ldots", 8230},
	{L"lt", 60},
	{L"macr", 175},
	{L"micro", 181},
	{L"middot", 183},
	{L"mu", 956},
	{L"nbsp", 160},
	{L"not", 172},
	{L"ntilde", 241},
	{L"nu", 957},
	{L"oacute", 243},
	{L"ocirc", 244},
	{L"ograve", 242},
	{L"omega", 969},
	{L"omicron", 959},
	{L"ordf", 170},
	{L"ordm", 186},
	{L"oslash", 248},
	{L"otilde", 245},
	{L"ouml", 246},
	{L"para", 182},
	{L"phi", 966},
	{L"pi", 960},
	{L"plusmn", 177},
	{L"pound", 163},
	{L"psi", 968},
	{L"quad", 8193},
	{L"quot", 34},
	{L"raquo", 187},
	{L"reg", 174},
	{L"rho", 961},
	{L"sect", 167},
	{L"shy", 173},
	{L"sigma", 963},
	{L"sp", 8194},
	{L"sup1", 185},
	{L"sup2", 178},
	{L"sup3", 179},
	{L"szlig", 223},
	{L"tau", 964},
	{L"theta", 952},
	{L"thinsp", 8201},
	{L"thorn", 254},
	{L"times", 215},
	{L"trade", 8482},
	{L"uacute", 250},
	{L"ucirc", 251},
	{L"ugrave", 249},
	{L"uml", 168},
	{L"upsilon", 965},
	{L"uuml", 252},
	{L"varepsilon", 8712},
	{L"varphi", 981},
	{L"varpi", 982},
	{L"varrho", 1009},
	{L"vdots", 8942},
	{L"vsigma", 962},
	{L"vtheta", 977},
	{L"xi", 958},
	{L"yacute", 253},
	{L"yen", 165},
	{L"yuml", 255},
	{L"zeta", 950}
};
#define NCHARTAB (sizeof(chartab)/sizeof(chartab[0]))

// Characters Winstart..Winend are those that Windows
// uses interpolated into the Latin1 set.
// They aren't supposed to appear in HTML, but they do....
enum {
	Winstart = 127,
	Winend = 159
};

static int	winchars[]= { 8226,	// 8226 is a bullet
	8226, 8226, 8218, 402, 8222, 8230, 8224, 8225,
	710, 8240, 352, 8249, 338, 8226, 8226, 8226,
	8226, 8216, 8217, 8220, 8221, 8226, 8211, 8212,
	732, 8482, 353, 8250, 339, 8226, 8226, 376};

static StringInt*	tagtable;		// initialized from tagnames
static StringInt*	attrtable;		// initialized from attrnames

static int		getplaindata(TokenSource* ts, Token* a, int* pai);
static int		getdata(TokenSource* ts, int firstc, int starti, Token* a, int* pai);
static int		getscriptdata(TokenSource* ts, int firstc, int starti, Token* a, int* pai);
static int		gettag(TokenSource* ts, int starti, Token* a, int* pai);
static Rune*	buftostr(Rune* s, Rune* buf, int j);
static int		comment(TokenSource* ts);
static int		findstr(TokenSource* ts, Rune* s);
static int		ampersand(TokenSource* ts);
static int		lowerc(int c);
static int		getchar(TokenSource* ts);
static void		ungetchar(TokenSource* ts, int c);
static void		backup(TokenSource* ts, int savei);
static void		freeinsidetoken(Token* t);
	int		Tfmt(Fmt *f);

int	dbglex = 0;

void
lexinit(void)
{
	tagtable = makestrinttab(tagnames, Numtags);
	attrtable = makestrinttab(attrnames, Numattrs);
	fmtinstall('T', Tfmt);
}

// Call this with a ByteSource that is just past the 'gethdr' stage.
TokenSource*
newtokensource(ByteSource* b, int chset, int mtype)
{
	TokenSource*	ans;

	assert(chset == US_Ascii || chset == ISO_8859_1 ||
			chset == UTF_8 || chset == Unicode);
	ans = (TokenSource*)emalloc(sizeof(TokenSource));
	ans->i = 0;
	ans->b = b;
	ans->chset = chset;
	ans->mtype = mtype;
	ans->ihigh = 0;
	dbglex = config.dbg['x'];
	return ans;
}

enum {
	ToksMax = 500
};

// Call this repeatedly to get another chunk of tokens
// (at most ToksMax of them), whenever the ByteSource buffer
// has more data.
//  The number of returned tokens is returned in *plen.
Token*
gettoks(TokenSource* ts, int* plen)
{
	Token*	a;
	int	ai;
	int	starti;
	int	c;
	int	i;
	int	tag;

	a = (Token*)emalloc(ToksMax * sizeof(Token));
	ai = 0;
	if(dbglex) {
		trace("gettoks starts, ts.i=%d, ts.b.edata=%d\n", ts->i, ts->b->edata);
		if(dbglex > 1 && ts->i < ts->b->edata) {
			trace("SOURCE from ts.i on...\n");
			write(1, ts->b->data+ts->i, ts->b->edata - ts->i);
			trace("\nSOURCE END\n\n");
		}
	}
	if(ts->mtype == TextHtml) {
		while(ai < ToksMax) {
			starti = ts->i;
			c = getchar(ts);
			if(c < 0)
				break;
			if(c == '<') {
				tag = gettag(ts, starti, a, &ai);
				if(tag == Tscript) {
					// special rules for getting Data after....
					starti = ts->i;
					c = getchar(ts);
					tag = getscriptdata(ts, c, starti, a, &ai);
				}
			}
			else
				tag = getdata(ts, c, starti, a, &ai);
			if(tag == -1)
				break;
			else if(dbglex > 1 && tag != Comment)
				trace("lex: got token %T\n", &a[ai-1]);
		}
		if(ai > 0) {
			// Several tags expect PCDATA after them,
			// which means that build needs to see another tag or eof
			// after any data in order to know that PCDATA is ended.
			// Backup if we haven't got to the following tag yet.
			i = ai - 1;
			while(i >= 0 && a[i].tag == Data)
				i--;
			if(i >= 0) {
				tag = a[i].tag;
				if(tag == Toption || tag == Tselect || tag == Ttextarea || tag == Ttitle || tag == Tscript) {
					if(ts->i != ts->b->dalloclen) {
						ts->i = a[i].starti;
						ai = i;
					}
				}
			}
		}
	}
	else {
		// plain text (non-html) tokens
		while(ai < ToksMax) {
			tag = getplaindata(ts, a, &ai);
			if(tag == -1)
				break;
			if(dbglex > 1)
				trace("lex: got token %T\n", &a[ai]);
		}
	}
	if(dbglex)
		trace("lex: returning %d tokens\n", ai);
	ts->b->lim = ts->ihigh;
	*plen = ai;
	if(ai == 0) {
		return nil;
	}
	return a;
}

// For case where source isn't HTML.
// Just make data tokens, one per line (or partial line,
// at end of buffer), ignoring non-whitespace control
// characters and dumping \r's.
// If find non-empty token, fill in a[*pai], bump *pai, and return Data.
// Otherwise return -1;
static int
getplaindata(TokenSource* ts, Token* a, int* pai)
{
	Rune*	s;
	int	j;
	int	starti;
	int	c;
	Token*	tok;
	Rune	buf[BIGBUFSIZE];

	s = nil;
	j = 0;
	starti = ts->i;
	for(c = getchar(ts); c >= 0; c = getchar(ts)) {
		if(c < ' ') {
			if(isspace(c)) {
				if(c == '\r') {
					// ignore it unless no following '\n',
					// in which case treat it like '\n'
					c = getchar(ts);
					if(c != '\n') {
						if(c >= 0)
							ungetchar(ts, c);
						c = '\n';
					}
				}
			}
			else
				c = 0;
		}
		if(c != 0) {
			buf[j++] = c;
			if(j == BIGBUFSIZE-1) {
				s = buftostr(s, buf, j);
				j = 0;
			}
		}
		if(c == '\n')
			break;
	}
	s = buftostr(s, buf, j);
	if(s == nil)
		return -1;
	tok = &a[(*pai)++];
	tok->tag = Data;
	tok->text = s;
	tok->attr = nil;
	tok->starti = starti;
	return Data;
}

// Return concatenation of s and buf[0:j]
static Rune*
buftostr(Rune* s, Rune* buf, int j)
{
	buf[j] = 0;
	if(s == nil)
		s = Strndup(buf, j);
	else 
		s = Strdup2(s, buf);
	return s;
}

// Gather data up to next start-of-tag or end-of-buffer.
// Translate entity references (&amp;).
// Ignore non-whitespace control characters and get rid of \r's.
// If find non-empty token, fill in a[*pai], bump *pai, and return Data.
// Otherwise return -1;
static int
getdata(TokenSource* ts, int firstc, int starti, Token* a, int* pai)
{
	Rune*	s;
	int	j;
	int	c;
	Token*	tok;
	Rune	buf[BIGBUFSIZE];

	s = nil;
	j = 0;
	c = firstc;
	while(c >= 0) {
		if(c == '&') {
			c = ampersand(ts);
			if(c < 0)
				break;
		}
		else if(c < ' ') {
			if(isspace(c)) {
				if(c == '\r') {
					// ignore it unless no following '\n',
					// in which case treat it like '\n'
					c = getchar(ts);
					if(c != '\n') {
						if(c >= 0)
							ungetchar(ts, c);
						c = '\n';
					}
				}
			}
			else {
				if(warn)
					trace("warning: non-whitespace control character %d ignored\n", c);
				c = 0;
			}
		}
		else if(c == '<') {
			ungetchar(ts, c);
			break;
		}
		if(c != 0) {
			buf[j++] = c;
			if(j == BIGBUFSIZE-1) {
				s = buftostr(s, buf, j);
				j = 0;
			}
		}
		c = getchar(ts);
	}
	s = buftostr(s, buf, j);
	if(s == nil)
		return -1;
	tok = &a[(*pai)++];
	tok->tag = Data;
	tok->text = s;
	tok->attr = nil;
	tok->starti = starti;
	return Data;
}

// The rules for lexing scripts are different (ugh).
// Gather up everything until see a </SCRIPT>.
static int
getscriptdata(TokenSource* ts, int firstc, int starti, Token* a, int* pai)
{
	Rune*	s;
	int	j;
	int	tstarti;
	int	savei;
	int	c;
	int	tag;
	int	done;
	Token*	tok;
	Rune	buf[BIGBUFSIZE];

	s = nil;
	j = 0;
	tstarti = starti;
	c = firstc;
	done = 0;
	while(c >= 0) {
		if(c == '<') {
			// other browsers ignore stuff to end of line after <!
			savei = ts->i;
			c = getchar(ts);
			if(c == '!') {
				while(c >= 0 && c != '\n' && c != '\r')
					c = getchar(ts);
				if(c == '\r')
					c = getchar(ts);
				if(c == '\n')
					c = getchar(ts);
			}
			else if(c >= 0) {
				backup(ts, savei);
				tag = gettag(ts, tstarti, a, pai);
				if(tag == -1)
					break;
				if(tag != Comment)
					(*pai)--;
				backup(ts, tstarti);
				if(tag == Tscript + RBRA) {
					done = 1;
					break;
				}
				// here tag was not </SCRIPT>, so take as regular data
				c = getchar(ts);
			}
		}
		if(c < 0)
			break;
		if(c != 0) {
			buf[j++] = c;
			if(j == BIGBUFSIZE-1) {
				s = buftostr(s, buf, j);
				j = 0;
			}
		}
		tstarti = ts->i;
		c = getchar(ts);
	}
	if(done || ts->i == ts->b->edata) {
		s = buftostr(s, buf, j);
		tok = &a[(*pai)++];
		tok->tag = Data;
		tok->text = s;
		tok->attr = nil;
		tok->starti = starti;
		return Data;
	}
	backup(ts, starti);
	return -1;
}

// We've just seen a '<'.  Gather up stuff to closing '>' (if buffer
// ends before then, return -1).
// If it's a tag, look up the name, gather the attributes, and return
// the appropriate token.
// Else it's either just plain data or some kind of ignorable stuff:
// return Data or Comment as appropriate.
// If it's not a Comment, put it in a[*pai] and bump *pai.
static int
gettag(TokenSource* ts, int starti, Token* a, int* pai)
{
	int	rbra;
	int	ans;
	Attr*	al;
	int	nexti;
	int	c;
	int	ti;
	int	afnd;
	int	attid;
	int	quote;
	Rune*	val;
	int	nv;
	int	i;
	int	tag;
	Token*	tok;
	Rune	buf[BIGBUFSIZE];

	rbra = 0;
	nexti = ts->i;
	tok = &a[*pai];
	tok->tag = Notfound;
	tok->text = nil;
	tok->attr = nil;
	tok->starti = starti;
	c = getchar(ts);
	if(c == '/') {
		rbra = RBRA;
		c = getchar(ts);
	}
	if(c < 0)
		goto eob_done;
	if(c >= 256 || !isalpha(c)) {
		// not a tag
		if(c == '!') {
			ans = comment(ts);
			if(ans != -1)
				return ans;
			goto eob_done;
		}
		else {
			backup(ts, nexti);
			tok->tag = Data;
			tok->text = L"<";
			(*pai)++;
			return Data;
		}
	}
	// c starts a tagname
	buf[0] = c;
	i = 1;
	while(1) {
		c = getchar(ts);
		if(c < 0)
			goto eob_done;
		if(!ISNAMCHAR(c))
			break;
		// if name is bigger than buf it won't be found anyway...
		if(i < BIGBUFSIZE)
			buf[i++] = c;
	}
	if(lookup(tagtable, Numtags, buf, i, &tag))
		tok->tag = tag + rbra;
	else
		tok->text = Strndup(buf, i);	// for warning print, in build

	// attribute gathering loop
	al = nil;
	while(1) {
		// look for "ws name" or "ws name ws = ws val"  (ws=whitespace)
		// skip whitespace
attrloop_continue:
		while(c < 256 && isspace(c)) {
			c = getchar(ts);
			if(c < 0)
				goto eob_done;
		}
		if(c == '>')
			goto attrloop_done;
		if(c == '<') {
			if(warn)
				trace("warning: unclosed tag\n");
			ungetchar(ts, c);
			goto attrloop_done;
		}
		if(c >= 256 || !isalpha(c)) {
			if(warn)
				trace("warning: expected attribute name\n");
			// skipt to next attribute name
			while(1) {
				c = getchar(ts);
				if(c < 0)
					goto eob_done;
				if(c < 256 && isalpha(c))
					goto attrloop_continue;
				if(c == '<') {
					if(warn)
						trace("warning: unclosed tag\n");
					ungetchar(ts, 60);
					goto attrloop_done;
				}
				if(c == '>')
					goto attrloop_done;
			}
		}
		// gather attribute name
		buf[0] = c;
		i = 1;
		while(1) {
			c = getchar(ts);
			if(c < 0)
				goto eob_done;
			if(!ISNAMCHAR(c))
				break;
			if(i < BIGBUFSIZE-1)
				buf[i++] = c;
		}
		afnd = lookup(attrtable, Numattrs, buf, i, &attid);
		if(warn && !afnd) {
			buf[i] = 0;
			trace("warning: unknown attribute name %S\n", buf);
		}
		// skip whitespace
		while(c < 256 && isspace(c)) {
			c = getchar(ts);
			if(c < 0)
				goto eob_done;
		}
		if(c != '=') {
			if(afnd)
				al = newattr(attid, nil, al);
			goto attrloop_continue;
		}
		//# c is '=' here;  skip whitespace
		while(1) {
			c = getchar(ts);
			if(c < 0)
				goto eob_done;
			if(c >= 256 || !isspace(c))
				break;
		}
		quote = 0;
		if(c == '\'' || c == '"') {
			quote = c;
			c = getchar(ts);
			if(c < 0)
				goto eob_done;
		}
		val = nil;
		nv = 0;
		while(1) {
valloop_continue:
			if(c < 0)
				goto eob_done;
			if(c == '>') {
				if(quote) {
					// c might be part of string (though not good style)
					// but if line ends before close quote, assume
					// there was an unmatched quote
					ti = ts->i;
					while(1) {
						c = getchar(ts);
						if(c < 0)
							goto eob_done;
						if(c == quote) {
							backup(ts, ti);
							buf[nv++] = '>';
							if(nv == BIGBUFSIZE-1) {
								val = buftostr(val, buf, nv);
								nv = 0;
							}
							c = getchar(ts);
							goto valloop_continue;
						}
						if(c == '\n') {
							if(warn)
								trace("warning: apparent unmatched quote\n");
							backup(ts, ti);
							c = '>';
							goto valloop_done;
						}
					}
				}
				else
					goto valloop_done;
			}
			if(quote) {
				if(c == quote) {
					c = getchar(ts);
					if(c < 0)
						goto eob_done;
					goto valloop_done;
				}
				if(c == '\r') {
					c = getchar(ts);
					goto valloop_continue;
				}
				if(c == '\t' || c == '\n')
					c = ' ';
			}
			else {
				if(c < 256 && isspace(c))
					goto valloop_done;
			}
			if(c == '&') {
				c = ampersand(ts);
				if(c == -1)
					goto eob_done;
			}
			buf[nv++] = c;
			if(nv == BIGBUFSIZE-1) {
				val = buftostr(val, buf, nv);
				nv = 0;
			}
			c = getchar(ts);
		}
valloop_done:
		if(afnd) {
			val = buftostr(val, buf, nv);
			al = newattr(attid, val, al);
		}
	}

attrloop_done:
	tok->attr = al;
	(*pai)++;
	return tok->tag;

eob_done:
	if(ts->i == ts->b->dalloclen) {
		if(warn)
			trace("warning: incomplete tag at end of page\n");
		backup(ts, nexti);
		tok->tag = Data;
		tok->text = L"<";
		return Data;
	}
	else
		backup(ts, starti);
	return -1;
}

// We've just read a '<!' at position starti,
// so this may be a comment or other ignored section, or it may
// be just a literal string if there is no close before end of file
// (other browsers do that).
// The accepted practice seems to be (note: contrary to SGML spec!):
// If see <!--, look for --> to close, or if none, > to close.
// If see <!(not --), look for > to close.
// If no close before end of file, leave original characters in as literal data.
//
// If we see ignorable stuff, return Comment.
// Else return nil (caller should back up and try again when more data arrives,
// unless at end of file, in which case caller should just make '<' a data token).
static int
comment(TokenSource* ts)
{
	int	nexti;
	int	havecomment;
	int	commentstart;
	int	c;

	nexti = ts->i;
	havecomment = 0;
	commentstart = 0;
	c = getchar(ts);
	if(c == '-') {
		c = getchar(ts);
		if(c == '-') {
			commentstart = 1;
			if(findstr(ts, L"-->"))
				havecomment = 1;
			else
				backup(ts, nexti);
		}
	}
	if(!havecomment) {
		if(c == '>')
			havecomment = 1;
		else if(c >= 0) {
			if(!commentstart || ts->b->edata == ts->b->dalloclen) {
				if(findstr(ts, L">"))
					havecomment = 1;
			}
		}
	}
	if(havecomment)
		return Comment;
	return -1;
}

// Look for string s in token source.
// If found, return 1, with buffer at next char after s,
// else return 0 (caller should back up).
static int
findstr(TokenSource* ts, Rune* s)
{
	int	c0;
	int	n;
	int	nexti;
	int	i;
	int	c;

	c0 = s[0];
	n = Strlen(s);
	while(1) {
		c = getchar(ts);
		if(c < 0)
			break;
		if(c == c0) {
			if(n == 1)
				return 1;
			nexti = ts->i;
			for(i = 1; i < n; i++) {
				c = getchar(ts);
				if(c < 0)
					goto mainloop_done;
				if(c != s[i])
					break;
			}
			if(i == n)
				return 1;
			backup(ts, nexti);
		}
	}
mainloop_done:
	return 0;
}

// We've just read an '&'; look for an entity reference
// name, and if found, return translated char.
// if there is a complete entity name but it isn't known,
// try prefixes (gets around some buggy HTML out there),
// and if that fails, back up to just past the '&' and return '&'.
// If the entity can't be completed in the current buffer, back up
// to the '&' and return -1.
static int
ampersand(TokenSource* ts)
{
	int	savei;
	int	c;
	int	fnd;
	int	ans;
	int	v;
	int	i;
	int	k;
	Rune	buf[SMALLBUFSIZE];

	savei = ts->i;
	c = getchar(ts);
	fnd = 0;
	ans = -1;
	if(c == '#') {
		c = getchar(ts);
		v = 0;
		while(c >= 0) {
			if(!(c < 256 && isdigit(c)))
				break;
			v = v*10 + c - 48;
			c = getchar(ts);
		}
		if(c >= 0) {
			if(!(c == ';' || c == '\n' || c == '\r'))
				ungetchar(ts, c);
			c = v;
			if(c == 160)
				c = 160;
			if(c >= Winstart && c <= Winend) {
				c = winchars[c - Winstart];
			}
			ans = c;
			fnd = 1;
		}
	}
	else if(c < 256 && isalpha(c)) {
		buf[0] = c;
		k = 1;
		while(1) {
			c = getchar(ts);
			if(c < 0)
				break;
			if(ISNAMCHAR(c)) {
				if(k < SMALLBUFSIZE-1)
					buf[k++] = c;
			}
			else {
				if(!(c == ';' || c == '\n' || c == '\r'))
					ungetchar(ts, c);
				break;
			}
		}
		if(c >= 0) {
			fnd = lookup(chartab, NCHARTAB, buf, k, &ans);
			if(!fnd) {
				// Try prefixes of s
				if(c == ';' || c == '\n' || c == '\r')
					ungetchar(ts, c);
				i = k;
				while(--k > 0) {
					fnd = lookup(chartab, NCHARTAB, buf, k, &ans);
					if(fnd) {
						while(i > k) {
							i--;
							ungetchar(ts, buf[i]);
						}
						break;
					}
				}
			}
		}
	}
	if(!fnd) {
		backup(ts, savei);
		if(c < 0 && ts->i < ts->b->dalloclen) {
			ungetchar(ts, '&');
			return -1;
		}
		else
			ans = '&';
	}
	return ans;
}

// Get next char, obeying ts.chset.
// Returns -1 if no complete character left before current end of data.
static int
getchar(TokenSource* ts)
{
	ByteSource*	bs;
	uchar*	buf;
	int	c;
	int	n;
	int	ok;
	Rune	r;

	bs = ts->b;
	if(ts->i >= bs->edata)
		return -1;
	buf = bs->data;
	c = buf[ts->i];
	switch(ts->chset) {
	case ISO_8859_1:
		if(c >= Winstart && c <= Winend)
			c = winchars[c - Winstart];
		ts->i++;
		break;
	case US_Ascii:
		if(c > 127) {
			if(warn)
				trace("non-ascii char (%x) when US-ASCII specified\n", c);
		}
		ts->i++;
		break;
	case UTF_8:
		ok = fullrune((char*)(buf+ts->i), bs->edata-ts->i);
		n = chartorune(&r, (char*)(buf+ts->i));
		if(ok) {
			if(warn && c == 0x80)
				trace("warning: invalid utf-8 sequence (starts with %x)\n", ts->b->data[ts->i]);
			ts->i += n;
			c = r;
		}
		else {
			// not enough bytes in buf to complete utf-8 char
			if(bs->edata == bs->dalloclen)
				ts->i = bs->edata;	// mark "all used"
			c = -1;
		}
		break;
	case Unicode:
		if(ts->i < bs->edata - 1) {
			//standards say most-significant byte first
			c = (c << 8)|(buf[ts->i + 1]);
			ts->i += 2;
		}
		else {
			if(bs->edata == bs->dalloclen)
				ts->i = bs->edata;	// mark "all used"
			c = -1;
		}
		break;
	}
	if(ts->i > ts->ihigh)
		ts->ihigh = ts->i;
	return c;
}

// Assuming c was the last character returned by getchar, set
// things up so that next getchar will get that same character
// followed by the current 'next character', etc.
static void
ungetchar(TokenSource* ts, int c)
{
	int	n;
	Rune	r;
	char	a[UTFmax];

	n = 1;
	switch(ts->chset) {
	case UTF_8:
		if(c >= 128) {
			r = c;
			n = runetochar(a, &r);
		}
		break;
	case Unicode:
		n = 2;
		break;
	}
	ts->i -= n;
}

// Restore ts so that it is at the state where buf was savebuf
// and the index was savei.
static void
backup(TokenSource* ts, int savei)
{
	if(dbglex)
		trace("lex: backup; i=%d, savei=%d\n", ts->i, savei);
	ts->i = savei;
}


// Look for value associated with attribute attid in token t.
// If there is one, return 1 and put the value in *pans,
// else return 0.
// If xfer is true, transfer ownership of the string to the caller
// (nil it out here); otherwise, caller must duplicate the anser
// if it needs to save it.
int
tokaval(Token* t, int attid, Rune** pans, int xfer)
{
	Attr*	attr;

	attr = t->attr;
	while(attr != nil) {
		if(attr->attid == attid) {
			*pans = attr->value;
			if(xfer)
				attr->value = nil;
			return 1;
		}
		attr = attr->next;
	}
	*pans = nil;
	return 0;
}

int
Tfmt(Fmt *f)
{
	Token*	t;
	int	i;
	int	tag;
	char*	srbra;
	Rune*	aname;
	Rune*	tname;
	Attr*	a;
	char	buf[BIGBUFSIZE];

	t = va_arg(f->args, Token*);
	if(t == nil)
		sprint(buf, "<null>");
	else {
		i = 0;
		if(dbglex > 1)
			i = snprint(buf, sizeof(buf), "[%d]", t->starti);
		tag = t->tag;
		if(tag == Data) {
			i += snprint(buf+i, sizeof(buf)-i-1, "'%S'", t->text);
		}
		else {
			srbra = "";
			if(tag >= RBRA) {
				tag -= RBRA;
				srbra = "/";
			}
			tname = tagnames[tag];
			if(tag == Notfound)
				tname = L"?";
			i += snprint(buf+i, sizeof(buf)-i-1, "<%s%S", srbra, tname);
			for(a = t->attr; a != nil; a = a->next) {
				aname = attrnames[a->attid];
				i += snprint(buf+i, sizeof(buf)-i-1, " %S", aname);
				if(a->value != nil)
					i += snprint(buf+i, sizeof(buf)-i-1, "=%S", a->value);
			}
			i += snprint(buf+i, sizeof(buf)-i-1, ">");
		}
		buf[i] = 0;
	}
	return fmtstrcpy(f, buf);
}

// Attrs own their constituent strings, but build may eventually
// transfer some values to its items and nil them out in the Attr.
Attr*
newattr(int attid, Rune* value, Attr* link)
{
	Attr* ans;

	ans = (Attr*)emalloc(sizeof(Attr));
	ans->attid = attid;
	ans->value = value;
	ans->next = link;
	return ans;
}

Bell Labs OSI certified Powered by Plan 9

(Return to Plan 9 Home Page)

Copyright © 2021 Plan 9 Foundation. All Rights Reserved.
Comments to [email protected].