53 	struct token **tokenlist;
54 	struct token *token;  member
190 const char *show_token(const struct token *token)  in show_token()  argument
194 	if (!token)  in show_token()
196 	switch (token_type(token)) {  in show_token()
204 		return show_ident(token->ident);  in show_token()
207 		return token->number;  in show_token()
210 		return show_special(token->special);  in show_token()
213 		return show_char(token->string->data,  in show_token()
214 			token->string->length - 1, 0, '\'');  in show_token()
216 		return show_char(token->embedded,  in show_token()
217 			token_type(token) - TOKEN_CHAR, 0, '\'');  in show_token()
219 		return show_char(token->string->data,  in show_token()
220 			token->string->length - 1, 'L', '\'');  in show_token()
222 		return show_char(token->embedded,  in show_token()
223 			token_type(token) - TOKEN_WIDE_CHAR, 'L', '\'');  in show_token()
225 		return show_char(token->string->data,  in show_token()
226 			token->string->length - 1, 0, '"');  in show_token()
228 		return show_char(token->string->data,  in show_token()
229 			token->string->length - 1, 'L', '"');  in show_token()
232 		sprintf(buffer, "<beginning of '%s'>", stream_name(token->pos.stream));  in show_token()
236 		sprintf(buffer, "<end of '%s'>", stream_name(token->pos.stream));  in show_token()
248 		sprintf(buffer, "unhandled token type '%d' ", token_type(token));  in show_token()
253 const char *quote_token(const struct token *token)  in quote_token()  argument
257 	switch (token_type(token)) {  in quote_token()
262 		return show_ident(token->ident);  in quote_token()
265 		return token->number;  in quote_token()
268 		return show_special(token->special);  in quote_token()
271 		return quote_char(token->string->data,  in quote_token()
272 			token->string->length - 1, 0, '\'');  in quote_token()
274 		return quote_char(token->embedded,  in quote_token()
275 			token_type(token) - TOKEN_CHAR, 0, '\'');  in quote_token()
277 		return quote_char(token->string->data,  in quote_token()
278 			token->string->length - 1, 'L', '\'');  in quote_token()
280 		return quote_char(token->embedded,  in quote_token()
281 			token_type(token) - TOKEN_WIDE_CHAR, 'L', '\'');  in quote_token()
283 		return quote_char(token->string->data,  in quote_token()
284 			token->string->length - 1, 0, '"');  in quote_token()
286 		return quote_char(token->string->data,  in quote_token()
287 			token->string->length - 1, 'L', '"');  in quote_token()
289 		sprintf(buffer, "unhandled token type '%d' ", token_type(token));  in quote_token()
343 static struct token * alloc_token(stream_t *stream)  in alloc_token()
345 	struct token *token = __alloc_token(0);  in alloc_token()  local
346 	token->pos = stream_pos(stream);  in alloc_token()
347 	return token;  in alloc_token()
458 struct token eof_token_entry;
460 static struct token *mark_eof(stream_t *stream)  in mark_eof()
462 	struct token *end;  in mark_eof()
480 	struct token *token = stream->token;  in add_token()  local
482 	stream->token = NULL;  in add_token()
483 	token->next = NULL;  in add_token()
484 	*stream->tokenlist = token;  in add_token()
485 	stream->tokenlist = &token->next;  in add_token()
490 	stream->newline |= stream->token->pos.newline;  in drop_token()
491 	stream->whitespace |= stream->token->pos.whitespace;  in drop_token()
492 	stream->token = NULL;  in drop_token()
547 	struct token *token;  in get_one_number()  local
577 	token = stream->token;  in get_one_number()
578 	token_type(token) = TOKEN_NUMBER;  in get_one_number()
579 	token->number = xmemdup(buffer, p - buffer);  in get_one_number()
589 	struct token *token = stream->token;  in eat_string()  local
629 		token_type(token) = type + len;  in eat_string()
631 		memcpy(token->embedded, buffer, 4);  in eat_string()
633 		token_type(token) = type;  in eat_string()
638 		token->string = string;  in eat_string()
642 	token = stream->token;  in eat_string()
746 	struct token *token;  in get_one_special()  local
788 	token = stream->token;  in get_one_special()
789 	token_type(token) = TOKEN_SPECIAL;  in get_one_special()
790 	token->special = value;  in get_one_special()
904 struct token *built_in_token(int stream, struct ident *ident)  in built_in_token()
906 	struct token *token;  in built_in_token()  local
908 	token = __alloc_token(0);  in built_in_token()
909 	token->pos.stream = stream;  in built_in_token()
910 	token_type(token) = TOKEN_IDENT;  in built_in_token()
911 	token->ident = ident;  in built_in_token()
912 	return token;  in built_in_token()
917 	struct token *token;  in get_one_identifier()  local
950 	token = stream->token;  in get_one_identifier()
951 	token_type(token) = TOKEN_IDENT;  in get_one_identifier()
952 	token->ident = ident;  in get_one_identifier()
967 static struct token *setup_stream(stream_t *stream, int idx, int fd,  in setup_stream()
970 	struct token *begin;  in setup_stream()
978 	stream->token = NULL;  in setup_stream()
990 static struct token *tokenize_stream(stream_t *stream)  in tokenize_stream()
995 			struct token *token = alloc_token(stream);  in tokenize_stream()  local
996 			stream->token = token;  in tokenize_stream()
1008 struct token * tokenize_buffer(void *buffer, unsigned long size, struct token **endtoken)  in tokenize_buffer()
1011 	struct token *begin;  in tokenize_buffer()
1018 struct token * tokenize(const struct position *pos, const char *name, int fd, struct token *endtoke…  in tokenize()
1020 	struct token *begin, *end;  in tokenize()