토크나이저(Tokenizer) C 코드

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
/*  Main configuration
   VIM variables:
   vim600: noet sw=4 expandtab fdm=marker nu
   vim<600: noet sw=4 expandtab ts=4 nu
*/
#include <stdio.h>
#define TOK_TOKEN_O 0x01
#define TOK_TOKEN_X 0x10
typedef struct tokened_st{
  int type;
  char *sta;
  char *end;
  struct tokened_st *next;
} tokened;
const char *token="~`!@#$%^&*()-_+=|\\{}[]:\"<>,.?/\n\t ";
int tokenrizer(const char *,const char *,tokened **);
int main(int argc,char **argv){
  int size=0;
  tokened *tokened_head=NULL,*cur=NULL;
  const char *str="int tokenrizer(char *str,char *token,tokened **tokened_head){elice";
  char *buf=NULL;
  printf("TOKENS: >>%s<<\n",token);
  printf("STRINGS: >>%s<<\n",str);
  printf("TOKEN_O is a token else not a token\n");
 
 
הההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההההה
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX