flex에 정의 된 C 토큰 파일이 있습니까?

Question

C 파일을 토큰으로 분할하려고합니다. 컴파일하는 것이 아니라 분석하는 것입니다. 나는 꽤 똑바로 앞으로해야하고, 정의 된 tokens.l (또는 이와 비슷한) 파일을 찾기 위해 온라인에서 C의 모든 문법이 이미 정의되었지만 아무것도 찾을 수 없다고 생각했다. 주변에 떠돌아 다니는 어떤 종류의 문법이 있는지 궁금해하고 있었나요?

Answer 1

예, 적어도 one 주위에 있습니다.

편집 : 처리하지 않는 몇 가지 문제가 있기 때문에

, 아마도 그것은 내가 몇 년 전에 쓴 일부 (손으로 쓴) 렉싱 코드를 찾고 가치가있다. 이것은 기본적으로 번역의 1, 2 및 3 단계 만 처리합니다. DIGRAPH를 정의하면 C++ 다이 그래프를 번역 할 일부 코드가 활성화됩니다. 그러나 메모리가 작동하면 실제로 일어날 것보다 일찍 번역 작업을하고 있지만 어쨌든 원하지 않을 것입니다. OTOH, 이것은 심지어 모든 토큰에 가까운 곳을 인식하지 못하도록합니다 - 주로 주석을 주석, 문자 리터럴, 문자열 리터럴 등으로 구분합니다. OTOH, trigraphs, 라인 스플 라이스 등을 처리합니다.

변환 된 텍스트로 파일을 열면 플랫폼의 줄 끝 문자를 줄 바꿈 문자로 새 줄에 기본 변환으로 남겨 두어야한다고 가정합니다.) 모드. 대부분의 경우, 그게 옳은 일이지만, 원본 파일이이 호스트에 대해 일반적인 것보다 다른 줄 끝 시퀀스를 갖는 크로스 컴파일러와 같은 것을 생성하려는 경우이를 변경해야 할 수도 있습니다.

먼저이 모든 것들에 외부 인터페이스를 정의하는 헤더 : 다음

/* get_src.h */ 
#ifndef GET_SRC_INCLUDED 
#define GET_SRC_INCLUDED 

#include <stdio.h> 

#ifdef __cplusplus 
extern "C" { 
#endif 

/* This is the size of the largest token we'll attempt to deal with. If 
* you want to deal with bigger tokens, change this, and recompile 
* get_src.c. Note that an entire comment is treated as a single token, 
* so long comments could overflow this. In case of an overflow, the 
* entire comment will be read as a single token, but the part larger 
* than this will not be stored. 
*/ 
#define MAX_TOKEN_SIZE 8192 

/* `last_token' will contain the text of the most recently read token (comment, 
* string literal, or character literal). 
*/ 
extern char last_token[]; 

/* This is the maximum number of characters that can be put back into a 
* file opened with parse_fopen or parse_fdopen. 
*/ 
#define MAX_UNGETS 5 

#include <limits.h> 
#include <stdio.h> 

typedef struct { 
    FILE *file; 
    char peeks[MAX_UNGETS]; 
    int last_peek; 
} PFILE; 

/* Some codes we return to indicate having found various items in the 
* source code. ERROR is returned to indicate a newline found in the 
* middle of a character or string literal or if a file ends inside a 
* comment, or if a character literal contains more than two characters. 
* 
* Note that this starts at INT_MIN, the most negative number available 
* in an int. This keeps these symbols from conflicting with any 
* characters read from the file. However, one of these could 
* theoretically conflict with EOF. EOF usually -1, and these are far 
* more negative than that. However, officially EOF can be any value 
* less than 0... 
*/ 
enum { 
    ERROR = INT_MIN, 
    COMMENT, 
    CHAR_LIT, 
    STR_LIT 
}; 

/* Opens a file for parsing and returns a pointer to a structure which 
* can be passed to the other functions in the parser/lexer to identify 
* the file being worked with. 
*/ 
PFILE *parse_fopen(char const *name); 

/* This corresponds closely to fdopen - it takes a FILE * as its 
* only parameter, creates a PFILE structure identifying that file, and 
* returns a pointer to that structure. 
*/ 
PFILE *parse_ffopen(FILE *stream); 

/* Corresponds to fclose. 
*/ 
int parse_fclose(PFILE *stream); 

/* returns characters from `stream' read as C source code. String 
* literals, characters literals and comments are each returned as a 
* single code from those above. All strings of any kind of whitespace 
* are returned as a single space character. 
*/ 
int get_source(PFILE *stream); 

/* Basically, these two work just like the normal versions of the same, 
* with the minor exception that unget_character can unget more than one 
* character. 
*/ 
int get_character(PFILE *stream); 
void unget_character(int ch, PFILE *stream); 

#ifdef __cplusplus 
} 
#endif 

#endif 

그리고 모두의 구현 :

/* get_src.c */ 
#include <stdio.h> 
#include <string.h> 
#include <ctype.h> 
#include <stdlib.h> 

#define GET_SOURCE 
#include "get_src.h" 

static size_t current = 0; 

char last_token[MAX_TOKEN_SIZE]; 

PFILE *parse_fopen(char const *name) { 

    PFILE *temp = malloc(sizeof(PFILE)); 

    if (NULL != temp) { 
     temp->file = fopen(name, "r"); 
     memset(temp->peeks, 0, sizeof(temp->peeks)); 
     temp->last_peek = 0; 
    } 
    return temp; 
} 

PFILE *parse_ffopen(FILE *file) { 

    PFILE *temp = malloc(sizeof(PFILE)); 

    if (NULL != temp) { 
     temp->file = file; 
     memset(temp->peeks, 0, sizeof(temp->peeks)); 
     temp->last_peek = 0; 
    } 
    return temp; 
} 

int parse_fclose(PFILE *stream) { 

    int retval = fclose(stream->file); 

    free(stream); 
    return retval; 
} 

static void addchar(int ch) { 
/* adds the passed character to the end of `last_token' */ 

    if (current < sizeof(last_token) -1) 
     last_token[current++] = (char)ch; 

    if (current == sizeof(last_token)-1) 
     last_token[current] = '\0'; 
} 

static void clear(void) { 
/* clears the previous token and starts building a new one. */ 
    current = 0; 
} 

static int read_char(PFILE *stream) { 
    if (stream->last_peek > 0) 
     return stream->peeks[--stream->last_peek]; 
    return fgetc(stream->file); 
} 

void unget_character(int ch, PFILE * stream) { 
    if (stream->last_peek < sizeof(stream->peeks)) 
     stream->peeks[stream->last_peek++] = ch; 
} 

static int check_trigraph(PFILE *stream) { 
/* Checks for trigraphs and returns the equivalant character if there 
* is one. Expects that the leading '?' of the trigraph has already 
* been read before this is called. 
*/ 

    int ch; 

    if ('?' != (ch=read_char(stream))) { 
     unget_character(ch, stream); 
     return '?'; 
    } 

    ch = read_char(stream); 

    switch(ch) { 
     case '(': return '['; 
     case ')': return ']'; 
     case '/': return '\\'; 
     case '\'': return '^'; 
     case '<': return '{'; 
     case '>': return '}'; 
     case '!': return '|'; 
     case '-': return '~'; 
     case '=': return '#'; 
     default: 
      unget_character('?', stream); 
      unget_character(ch, stream); 
      return '?'; 
    } 
} 

#ifdef DIGRAPH 
static int check_digraph(PFILE *stream, int first) { 
/* Checks for a digraph. The first character of the digraph is 
* transmitted as the second parameter, as there are several possible 
* first characters of a digraph. 
*/ 

    int ch = read_char(stream); 

    switch(first) { 
     case '<': 
      if ('%' == ch) 
       return '{'; 
      if (':' == ch) 
       return '['; 
      break; 
     case ':': 
      if ('>' == ch) 
       return ']'; 
      break; 
     case '%': 
      if ('>' == ch) 
       return '}'; 
      if (':' == ch) 
       return '#'; 
      break; 
    } 

/* If it's not one of the specific combos above, return the characters 
* separately and unchanged by putting the second one back into the 
* stream, and returning the first one as-is. 
*/ 
    unget_character(ch, stream); 
    return first; 
} 
#endif 


static int get_char(PFILE *stream) { 
/* Gets a single character from the stream with any trigraphs or digraphs converted 
* to the single character represented. Note that handling digraphs this early in 
* translation isn't really correct (and shouldn't happen in C at all). 
*/ 
    int ch = read_char(stream); 

    if (ch == '?') 
     return check_trigraph(stream); 

#ifdef DIGRAPH 
    if ((ch == '<' || ch == ':' || ch == '%')) 
     return check_digraph(stream, ch); 
#endif 

    return ch; 
} 

int get_character(PFILE *stream) { 
/* gets a character from `stream'. Any amount of any kind of whitespace 
* is returned as a single space. Escaped new-lines are "eaten" here as well. 
*/ 
    int ch; 

    if (!isspace(ch=get_char(stream)) && ch != '\\') 
     return ch; 

    // handle line-slicing 
    if (ch == '\\') { 
     ch = get_char(stream); 
     if (ch == '\n') 
      ch = get_char(stream); 
     else { 
      unget_character(ch, stream); 
      return ch; 
     } 
    } 

    /* If it's a space, skip over consecutive white-space */ 
    while (isspace(ch) && ('\n' != ch)) 
     ch = get_char(stream); 

    if ('\n' == ch) 
     return ch; 

    /* Then put the non-ws character back */ 
    unget_character(ch, stream); 

    /* and return a single space character... */ 
    return ' '; 
} 

static int read_char_lit(PFILE *stream) { 
/* This is used internally by `get_source' (below) - it expects the 
* opening quote of a character literal to have already been read and 
* returns CHAR_LIT or ERROR if there's a newline before a close 
* quote is found, or if the character literal contains more than two 
* characters after escapes are taken into account. 
*/ 

    int ch; 
    int i; 


    clear(); 
    addchar('\''); 

    for (i=0; i<2 && ('\'' != (ch = read_char(stream))); i++) { 

     addchar(ch); 

     if (ch == '\n') 
      return ERROR; 

     if (ch == '\\') { 
      ch = get_char(stream); 
      addchar(ch); 
     } 
    } 
    addchar('\''); 
    addchar('\0'); 

    if (i > 2) 
     return ERROR; 

    return CHAR_LIT; 
} 

static int read_str_lit(PFILE *stream) { 
/* Used internally by get_source. Expects the opening quote of a string 
* literal to have already been read. Returns STR_LIT, or ERROR if a 
* un-escaped newline is found before the close quote. 
*/ 

    int ch; 

    clear(); 
    addchar('"'); 

    while ('"' != (ch = get_char(stream))) { 

     if ('\n' == ch || EOF == ch) 
      return ERROR; 

     addchar(ch); 

     if(ch == '\\') { 
      ch = read_char(stream); 
      addchar(ch); 
     } 

    } 

    addchar('"'); 
    addchar('\0'); 

    return STR_LIT; 
} 

static int read_comment(PFILE *stream) { 
/* Skips over a comment in stream. Assumes the leading '/' has already 
* been read and skips over the body. If we're reading C++ source, skips 
* C++ single line comments as well as normal C comments. 
*/ 
    int ch; 

    clear(); 

    ch = get_char(stream); 

    /* Handle a single line comment. 
    */ 
    if ('/' == ch) { 
     addchar('/'); 
     addchar('/'); 

     while ('\n' != (ch = get_char(stream))) 
      addchar(ch);  

     addchar('\0'); 
     return COMMENT; 
    } 

    if ('*' != ch) { 
     unget_character(ch, stream); 
     return '/'; 
    } 

    addchar('/'); 

    do { 
     addchar(ch); 
     while ('*' !=(ch = get_char(stream))) 
      if (EOF == ch) 
       return ERROR; 
      else 
       addchar(ch); 
     addchar(ch); 
    } while ('/' != (ch=get_char(stream))); 

    addchar('/'); 
    addchar('\0'); 

    return COMMENT; 
} 

int get_source(PFILE *stream) { 
/* reads and returns a single "item" from the stream. An "item" is a 
* comment, a literal or a single character after trigraph and possible 
* digraph substitution has taken place. 
*/ 

    int ch = get_character(stream); 

    switch(ch) { 
     case '\'': 
      return read_char_lit(stream); 
     case '"': 
      return read_str_lit(stream); 
     case '/': 
      return read_comment(stream); 
     default: 
      return ch; 
    } 
} 

#ifdef TEST 

int main(int argc, char **argv) { 
    PFILE *f; 
    int ch; 

    if (argc != 2) { 
     fprintf(stderr, "Usage: get_src <filename>\n"); 
     return EXIT_FAILURE; 
    } 

    if (NULL==(f= parse_fopen(argv[1]))) { 
     fprintf(stderr, "Unable to open: %s\n", argv[1]); 
     return EXIT_FAILURE; 
    } 

    while (EOF!=(ch=get_source(f))) 
     if (ch < 0) 
      printf("\n%s\n", last_token); 
     else 
      printf("%c", ch); 
    parse_fclose(f); 
    return 0;  
} 

#endif 

잘 모르겠어요 얼마나 쉬운/어려운 그것에 대해를 그것을 Flex 기반의 렉서로 통합 할 수있을 것입니다 - Flex가 캐릭터를 읽는 데 사용하는 것을 정의하기위한 일종의 훅을 가지고 있지만, 사용하려고 시도한 적이 없기 때문에, 그것에 대해 더 많이 말하지 마라. (그리고 궁극적으로, 확실하게 접근하는 어떤 것으로도 말할 수 없다. 그것도 존재한다).