Web_Scraping_Language/lexer.py at main · Deepmalya1/Web_Scraping_Language · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
import ply.lex as lex

# Define the tokens
tokens = (
    'SCRAPE',
    'URL',
    'INTO',
    'JSON_FILE',
    'CSV_FILE',
    'XML_FILE',
    'SELECT',
    'USER_AGENT',
    'DELAY',
    'RETRIES',
    'PROXY',
    'AUTH',
    'HEADER',
    'VALIDATE',
    'FILTER',
    'API'
)

# Regular expressions for tokens
def t_SCRAPE(t):
    r'scrape'
    return t

def t_URL(t):
    r'"[^"]+"'
    return t

def t_INTO(t):
    r'into'
    return t

def t_JSON_FILE(t):
    r'[a-zA-Z0-9_]+\.json'
    return t

def t_CSV_FILE(t):
    r'[a-zA-Z0-9_]+\.csv'
    return t

def t_XML_FILE(t):
    r'[a-zA-Z0-9_]+\.xml'
    return t

def t_SELECT(t):
    r'select'
    return t

def t_USER_AGENT(t):
    r'with_user_agent\s+"[^"]+"'
    return t

def t_DELAY(t):
    r'with_delay\s+\d+s'
    return t

def t_RETRIES(t):
    r'with_retries\s+\d+'
    return t

def t_PROXY(t):
    r'using_proxy\s+"[^"]+"'
    return t

def t_AUTH(t):
    r'using_auth\s+"[^"]+"'
    return t

def t_HEADER(t):
    r'using_headers\s+\{[^}]+\}'
    return t

def t_VALIDATE(t):
    r'validate\s+fields\s+\[[^\]]+\]'
    return t

def t_FILTER(t):
    r'filter\s+by\s+[^\s]+'
    return t

def t_API(t):
    r'using_api'
    return t

# Ignore whitespace
t_ignore = ' \t\n'

def t_error(t):
    print(f"Illegal character '{t.value[0]}'")
    t.lexer.skip(1)

# Build the lexer
lexer = lex.lex()

if __name__ == "__main__":
    # Test the lexer
    test_input = 'scrape "https://example.com" into data.json with_user_agent "Custom UA"'
    lexer.input(test_input)
    for tok in lexer:
        print(tok)