tccpp: macro cleanup

- remove TOK_NOSUBST, mark the token itself instead
- get_tok_str(); mask out SYM_FIELD & update uses
- next(): optimize (~5% faster with tcc -E)
- tok_flags: remove some redundancy
- parse_define(): do not remove spaces around '##' and after '#'
    and mark macros with '##' as MACRO_JOIN to avoid unnecessary
    call to macro_twosharps(mstr):
- next_nomacro(): removed, next_nomacro1(): renamed to next_nomacro()
- next_argstream(): cleanup & new function peek_file()
- macro_subst_tok(): handle special macros (__DATE__ etc.)
  like normal macros if they are #defined
- -DPP_DEBUG : more structured output
- pp_error(): better preprocessor expression error message
- tcctok.h: sort basic keywords (somehow)
- testspp/Makefile: generate .expect with 'make testspp.##+'
- tcc.c: tcc -E -o file : put unixy LFs also on windows

libtcc.c

@@ -633,14 +633,17 @@ static void error1(int mode, const char *fmt, va_list ap)
             cstr_printf(&cs, "In file included from %s:%d:\n",
                 (*pf)->filename, (*pf)->line_num - 1);
         cstr_printf(&cs, "%s:%d: ",
-            f->filename, f->line_num - !!(tok_flags & TOK_FLAG_BOL));
+            f->filename, f->line_num - ((tok_flags & TOK_FLAG_BOL) && !macro_ptr));
     } else if (s1->current_filename) {
         cstr_printf(&cs, "%s: ", s1->current_filename);
     } else {
         cstr_printf(&cs, "tcc: ");
     }
     cstr_printf(&cs, mode == ERROR_WARN ? "warning: " : "error: ");
-    cstr_vprintf(&cs, fmt, ap);
+    if (pp_expr > 1)
+        pp_error(&cs); /* special handler for preprocessor expression errors */
+    else
+        cstr_vprintf(&cs, fmt, ap);
     if (!s1->error_func) {
         /* default case: stderr */
         if (s1 && s1->output_type == TCC_OUTPUT_PREPROCESS && s1->ppfp == stdout)
@@ -717,6 +720,7 @@ ST_FUNC void tcc_open_bf(TCCState *s1, const char *filename, int initlen)
     bf->ifdef_stack_ptr = s1->ifdef_stack_ptr;
     bf->fd = -1;
     bf->prev = file;
+    bf->prev_tok_flags = tok_flags;
     file = bf;
     tok_flags = TOK_FLAG_BOL | TOK_FLAG_BOF;
 }
@@ -732,6 +736,7 @@ ST_FUNC void tcc_close(void)
     if (bf->true_filename != bf->filename)
         tcc_free(bf->true_filename);
     file = bf->prev;
+    tok_flags = bf->prev_tok_flags;
     tcc_free(bf);
 }
 

tcc.c

@@ -326,7 +326,7 @@ redo:
             tcc_error_noabort("no input files");
         } else if (s->output_type == TCC_OUTPUT_PREPROCESS) {
             if (s->outfile && 0!=strcmp("-",s->outfile)) {
-                ppfp = fopen(s->outfile, "w");
+                ppfp = fopen(s->outfile, "wb");
                 if (!ppfp)
                     tcc_error_noabort("could not write '%s'", s->outfile);
             }

tcc.h

@@ -583,10 +583,11 @@ typedef struct Sym {
     };
     CType type; /* associated type */
     union {
-        int *vla_array_str; /* vla array code */
         struct Sym *next; /* next related symbol (for fields and anoms) */
-        struct Sym *cleanupstate; /* in defined labels */
+        int *e; /* expanded token stream */
         int asm_label; /* associated asm label */
+        struct Sym *cleanupstate; /* in defined labels */
+        int *vla_array_str; /* vla array code */
     };
     struct Sym *prev; /* prev symbol in stack */
     struct Sym *prev_tok; /* previous symbol for this token */
@@ -645,6 +646,7 @@ typedef struct DLLReference {
 /* field 'Sym.t' for macros */
 #define MACRO_OBJ      0 /* object like macro */
 #define MACRO_FUNC     1 /* function like macro */
+#define MACRO_JOIN     2 /* macro uses ## */
 
 /* field 'Sym.r' for C labels */
 #define LABEL_DEFINED  0 /* label is defined */
@@ -672,6 +674,7 @@ typedef struct BufferedFile {
     int ifndef_macro_saved; /* saved ifndef_macro */
     int *ifdef_stack_ptr; /* ifdef_stack value at the start of the file */
     int include_next_index; /* next search path */
+    int prev_tok_flags; /* saved tok_flags */
     char filename[1024];    /* filename */
     char *true_filename; /* filename not modified by # line directive */
     unsigned char unget[4];
@@ -685,7 +688,7 @@ typedef struct BufferedFile {
 typedef struct TokenString {
     int *str;
     int len;
-    int lastlen;
+    int need_spc;
     int allocated_len;
     int last_line_num;
     int save_line_num;
@@ -1156,7 +1159,6 @@ struct filespec {
 #define TOK_TWODOTS 0xa2 /* C++ token ? */
 #define TOK_TWOSHARPS 0xa3 /* ## preprocessing token */
 #define TOK_PLCHLDR 0xa4 /* placeholder token as defined in C99 */
-#define TOK_NOSUBST 0xa5 /* means following token has already been pp'd */
 #define TOK_PPJOIN  0xa6 /* A '##' in the right position to mean pasting */
 #define TOK_SOTYPE  0xa7 /* alias of '(' for parsing sizeof (type) */
 
@@ -1345,11 +1347,11 @@ ST_DATA CString tokcstr; /* current parsed string, if any */
 /* display benchmark infos */
 ST_DATA int tok_ident;
 ST_DATA TokenSym **table_ident;
+ST_DATA int pp_expr;
 
 #define TOK_FLAG_BOL   0x0001 /* beginning of line before */
 #define TOK_FLAG_BOF   0x0002 /* beginning of file before */
 #define TOK_FLAG_ENDIF 0x0004 /* a endif was found matching starting #ifdef */
-#define TOK_FLAG_EOF   0x0008 /* end of file */
 
 #define PARSE_FLAG_PREPROCESS 0x0001 /* activate preprocessing */
 #define PARSE_FLAG_TOK_NUM    0x0002 /* return numbers instead of TOK_PPNUM */
@@ -1400,6 +1402,8 @@ ST_FUNC void tccpp_delete(TCCState *s);
 ST_FUNC int tcc_preprocess(TCCState *s1);
 ST_FUNC void skip(int c);
 ST_FUNC NORETURN void expect(const char *msg);
+ST_FUNC void pp_error(CString *cs);
+
 
 /* space excluding newline */
 static inline int is_space(int ch) {

tccasm.c

@@ -662,8 +662,7 @@ static void asm_parse_directive(TCCState *s1, int global)
                     tcc_error("we at end of file, .endr not found");
                 tok_str_add_tok(init_str);
             }
-            tok_str_add(init_str, -1);
+            tok_str_add(init_str, TOK_EOF);
-            tok_str_add(init_str, 0);
             begin_macro(init_str, 1);
             while (repeat-- > 0) {
                 tcc_assemble_internal(s1, (parse_flags & PARSE_FLAG_PREPROCESS),

tccdbg.c

@@ -913,7 +913,7 @@ ST_FUNC void tcc_debug_end(TCCState *s1)
                                              : DWARF_ABBREV_STRUCTURE_EMPTY_TYPE);
             dwarf_strp(dwarf_info_section,
                        (t->v & ~SYM_STRUCT) >= SYM_FIRST_ANOM
-                       ? "" : get_tok_str(t->v & ~SYM_STRUCT, NULL));
+                       ? "" : get_tok_str(t->v, NULL));
             dwarf_uleb128(dwarf_info_section, 0);
             dwarf_uleb128(dwarf_info_section, dwarf_line.cur_file);
             dwarf_uleb128(dwarf_info_section, file->line_num);
@@ -1329,7 +1329,7 @@ static void tcc_get_debug_info(TCCState *s1, Sym *s, CString *result)
             cstr_new (&str);
             cstr_printf (&str, "%s:T%d=%c%d",
                          (t->v & ~SYM_STRUCT) >= SYM_FIRST_ANOM
-                         ? "" : get_tok_str(t->v & ~SYM_STRUCT, NULL),
+                         ? "" : get_tok_str(t->v, NULL),
                          debug_type,
                          IS_UNION (t->type.t) ? 'u' : 's',
                          t->c);
@@ -1340,7 +1340,7 @@ static void tcc_get_debug_info(TCCState *s1, Sym *s, CString *result)
 		if (STRUCT_NODEBUG(t))
 		    continue;
                 cstr_printf (&str, "%s:",
-                             get_tok_str(t->v & ~SYM_FIELD, NULL));
+                             get_tok_str(t->v, NULL));
                 tcc_get_debug_info (s1, t, &str);
                 if (t->type.t & VT_BITFIELD) {
                     pos = t->c * 8 + BIT_POS(t->type.t);
@@ -1368,13 +1368,13 @@ static void tcc_get_debug_info(TCCState *s1, Sym *s, CString *result)
             cstr_new (&str);
             cstr_printf (&str, "%s:T%d=e",
                          (t->v & ~SYM_STRUCT) >= SYM_FIRST_ANOM
-                         ? "" : get_tok_str(t->v & ~SYM_STRUCT, NULL),
+                         ? "" : get_tok_str(t->v, NULL),
                          debug_type);
             while (t->next) {
                 t = t->next;
                 cstr_printf (&str, "%s:",
                              (t->v & ~SYM_FIELD) >= SYM_FIRST_ANOM
-                             ? "" : get_tok_str(t->v & ~SYM_FIELD, NULL));
+                             ? "" : get_tok_str(t->v, NULL));
                 cstr_printf (&str, e->type.t & VT_UNSIGNED ? "%u," : "%d,",
                              (int)t->enum_val);
             }
@@ -1461,7 +1461,7 @@ static int tcc_get_dwarf_info(TCCState *s1, Sym *s)
 				  : DWARF_ABBREV_STRUCTURE_EMPTY_TYPE);
 	    dwarf_strp(dwarf_info_section,
                        (t->v & ~SYM_STRUCT) >= SYM_FIRST_ANOM
-                       ? "" : get_tok_str(t->v & ~SYM_STRUCT, NULL));
+                       ? "" : get_tok_str(t->v, NULL));
 	    dwarf_uleb128(dwarf_info_section, t->c);
 	    dwarf_uleb128(dwarf_info_section, dwarf_line.cur_file);
 	    dwarf_uleb128(dwarf_info_section, file->line_num);
@@ -1479,7 +1479,7 @@ static int tcc_get_dwarf_info(TCCState *s1, Sym *s)
 			    e->type.t & VT_BITFIELD ? DWARF_ABBREV_MEMBER_BF
 						    : DWARF_ABBREV_MEMBER);
 		dwarf_strp(dwarf_info_section,
-			   get_tok_str(e->v & ~SYM_FIELD, NULL));
+			   get_tok_str(e->v, NULL));
 		dwarf_uleb128(dwarf_info_section, dwarf_line.cur_file);
 		dwarf_uleb128(dwarf_info_section, file->line_num);
 		pos_type[i++] = dwarf_info_section->data_offset;
@@ -1527,7 +1527,7 @@ static int tcc_get_dwarf_info(TCCState *s1, Sym *s)
 	    dwarf_data1(dwarf_info_section, DWARF_ABBREV_ENUMERATION_TYPE);
 	    dwarf_strp(dwarf_info_section,
                        (t->v & ~SYM_STRUCT) >= SYM_FIRST_ANOM
-                       ? "" : get_tok_str(t->v & ~SYM_STRUCT, NULL));
+                       ? "" : get_tok_str(t->v, NULL));
 	    dwarf_data1(dwarf_info_section,
 		        type & VT_UNSIGNED ? DW_ATE_unsigned : DW_ATE_signed );
 	    dwarf_data1(dwarf_info_section, 4);
@@ -1544,7 +1544,7 @@ static int tcc_get_dwarf_info(TCCState *s1, Sym *s)
 					   : DWARF_ABBREV_ENUMERATOR_SIGNED);
 	        dwarf_strp(dwarf_info_section,
                            (e->v & ~SYM_FIELD) >= SYM_FIRST_ANOM
-                           ? "" : get_tok_str(e->v & ~SYM_FIELD, NULL));
+                           ? "" : get_tok_str(e->v, NULL));
 		if (type & VT_UNSIGNED)
 	            dwarf_uleb128(dwarf_info_section, e->enum_val);
 		else
@@ -1999,7 +1999,7 @@ ST_FUNC void tcc_debug_typedef(TCCState *s1, Sym *sym)
         debug_type = tcc_get_dwarf_info(s1, sym);
 	if (debug_type != -1) {
 	    dwarf_data1(dwarf_info_section, DWARF_ABBREV_TYPEDEF);
-	    dwarf_strp(dwarf_info_section, get_tok_str(sym->v & ~SYM_FIELD, NULL));
+	    dwarf_strp(dwarf_info_section, get_tok_str(sym->v, NULL));
 	    dwarf_uleb128(dwarf_info_section, dwarf_line.cur_file);
 	    dwarf_uleb128(dwarf_info_section, file->line_num);
 	    tcc_debug_check_anon(s1, sym, dwarf_info_section->data_offset);
@@ -2012,7 +2012,7 @@ ST_FUNC void tcc_debug_typedef(TCCState *s1, Sym *sym)
         cstr_new (&str);
         cstr_printf (&str, "%s:t",
                      (sym->v & ~SYM_FIELD) >= SYM_FIRST_ANOM
-                     ? "" : get_tok_str(sym->v & ~SYM_FIELD, NULL));
+                     ? "" : get_tok_str(sym->v, NULL));
         tcc_get_debug_info(s1, sym, &str);
         tcc_debug_stabs(s1, str.data, N_LSYM, 0, NULL, 0, 0);
         cstr_free (&str);

tccgen.c

@@ -7325,10 +7325,8 @@ static void skip_or_save_block(TokenString **str)
 	      break;
 	}
     }
-    if (str) {
+    if (str)
-	tok_str_add(*str, -1);
+	tok_str_add(*str, TOK_EOF);
-	tok_str_add(*str, 0);
-    }
 }
 
 #define EXPR_CONST 1
@@ -8038,8 +8036,7 @@ static void decl_initializer_alloc(CType *type, AttributeDef *ad, int r,
                 tok_str_add_tok(init_str);
                 next();
             }
-            tok_str_add(init_str, -1);
+            tok_str_add(init_str, TOK_EOF);
-            tok_str_add(init_str, 0);
         } else
             skip_or_save_block(&init_str);
         unget_tok(0);

tcctok.h

@@ -1,23 +1,24 @@
 /*********************************************************************/
 /* keywords */
-     DEF(TOK_INT, "int")
-     DEF(TOK_VOID, "void")
-     DEF(TOK_CHAR, "char")
      DEF(TOK_IF, "if")
      DEF(TOK_ELSE, "else")
      DEF(TOK_WHILE, "while")
+     DEF(TOK_FOR, "for")
+     DEF(TOK_DO, "do")
+     DEF(TOK_CONTINUE, "continue")
      DEF(TOK_BREAK, "break")
      DEF(TOK_RETURN, "return")
-     DEF(TOK_FOR, "for")
+     DEF(TOK_GOTO, "goto")
+     DEF(TOK_SWITCH, "switch")
+     DEF(TOK_CASE, "case")
+     DEF(TOK_DEFAULT, "default")
+     DEF(TOK_ASM1, "asm")
+     DEF(TOK_ASM2, "__asm")
+     DEF(TOK_ASM3, "__asm__")
+
      DEF(TOK_EXTERN, "extern")
      DEF(TOK_STATIC, "static")
      DEF(TOK_UNSIGNED, "unsigned")
-     DEF(TOK_GOTO, "goto")
-     DEF(TOK_DO, "do")
-     DEF(TOK_CONTINUE, "continue")
-     DEF(TOK_SWITCH, "switch")
-     DEF(TOK_CASE, "case")
-
      DEF(TOK__Atomic, "_Atomic")
      DEF(TOK_CONST1, "const")
      DEF(TOK_CONST2, "__const") /* gcc keyword */
@@ -25,7 +26,6 @@
      DEF(TOK_VOLATILE1, "volatile")
      DEF(TOK_VOLATILE2, "__volatile") /* gcc keyword */
      DEF(TOK_VOLATILE3, "__volatile__") /* gcc keyword */
-     DEF(TOK_LONG, "long")
      DEF(TOK_REGISTER, "register")
      DEF(TOK_SIGNED1, "signed")
      DEF(TOK_SIGNED2, "__signed") /* gcc keyword */
@@ -43,15 +43,18 @@
      DEF(TOK_GENERIC, "_Generic")
      DEF(TOK_STATIC_ASSERT, "_Static_assert")
 
+     DEF(TOK_VOID, "void")
+     DEF(TOK_CHAR, "char")
+     DEF(TOK_INT, "int")
      DEF(TOK_FLOAT, "float")
      DEF(TOK_DOUBLE, "double")
      DEF(TOK_BOOL, "_Bool")
      DEF(TOK_COMPLEX, "_Complex")
      DEF(TOK_SHORT, "short")
+     DEF(TOK_LONG, "long")
      DEF(TOK_STRUCT, "struct")
      DEF(TOK_UNION, "union")
      DEF(TOK_TYPEDEF, "typedef")
-     DEF(TOK_DEFAULT, "default")
      DEF(TOK_ENUM, "enum")
      DEF(TOK_SIZEOF, "sizeof")
      DEF(TOK_ATTRIBUTE1, "__attribute")
@@ -64,9 +67,6 @@
      DEF(TOK_TYPEOF2, "__typeof")
      DEF(TOK_TYPEOF3, "__typeof__")
      DEF(TOK_LABEL, "__label__")
-     DEF(TOK_ASM1, "asm")
-     DEF(TOK_ASM2, "__asm")
-     DEF(TOK_ASM3, "__asm__")
 
 #ifdef TCC_TARGET_ARM64
      DEF(TOK_UINT128, "__uint128_t")

tests/pp/11.c

@@ -14,7 +14,7 @@ x D4(a, b) y
 x D4(a) y
 x D4() y
 
-#define GNU_COMMA(X,Y...) X,## Y
+#define GNU_COMMA(X,Y...) X,##Y
 
 x GNU_COMMA(A,B,C) y
 x GNU_COMMA(A,B) y
@@ -31,6 +31,6 @@ __NORETURN
 Y(X X() ())
 
 #define DDD(A, B) D_ ## B ## _D_ ## A
-#define CCC(X, ...) DDD(X, ##__VA_ARGS__)
+#define CCC(X, ...) DDD(X,## __VA_ARGS__)
 /* must be D_B_D_A (not D_B _D_A) */
 CCC(A,B)

tests/pp/11.expect

@@ -6,11 +6,11 @@ x a b c d y
 x a b c y
 x a b y
 x a y
-x y
+x  y
 x A,B,C y
 x A,B y
 x A y
-x y
+x  y
 __attribute__((__noreturn__))
 1 2
 D_B_D_A

tests/pp/21.c

@@ -34,3 +34,8 @@ OK
 #else
 NOT OK
 #endif
+
+----- 5 ------
+line __LINE__
+#define __LINE__ # ## #
+line __LINE__

tests/pp/21.expect

@@ -6,3 +6,6 @@ OK
 OK
 ----- 4 ------
 OK
+----- 5 ------
+line 39
+line ##

tests/pp/Makefile

@@ -31,6 +31,10 @@ FILTER = 2>&1 | sed 's,$(SRC)/,,g'
 
 testspp.%: %.test ;
 
+# generate .expect file with tcc, e.g. "make testspp.17+"
+testspp.%+: %.c
+	$(TCC) -E -P $*.[cS] -o $*.expect
+
 # automatically generate .expect files with gcc:
 %.expect: # %.c
 	gcc -E -P $*.[cS] >$*.expect 2>&1

tests/tests2/60_errors_and_warnings.c

@@ -476,4 +476,11 @@ struct A {
     int x;
 };
 
+#elif defined test_pp_error_1
+# if //no expression
+# endif
+#elif defined test_pp_error_2
+# if X(1,2) //undefined function macro
+# endif
+
 #endif

tests/tests2/60_errors_and_warnings.expect

@@ -231,3 +231,9 @@ arg[1] = "Y"
 
 [test_error_incomplete_type]
 60_errors_and_warnings.c:472: error: initialization of incomplete type
+
+[test_pp_error_1]
+60_errors_and_warnings.c:480: error: #if with no expression
+
+[test_pp_error_2]
+60_errors_and_warnings.c:483: error: bad preprocessor expression: #if 0 ( 1 , 2 )