Daetris/parser.py at master · Lactobacillus/Daetris · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
import pickle
import unicodedata
import urllib.request
from bs4 import BeautifulSoup

def save(fileName, target):

	f = open(fileName, 'wb')

	try:

		pickle.dump(target, f)

	except Exception as e:

		exception_info(e)

	finally:

		f.close()

def get_htmls(fileName):

	htmls = list()

	with open(fileName, 'r') as f:

		urls = f.readlines()

	for url in urls:

		response = urllib.request.urlopen(url)
		html = BeautifulSoup(response.read().decode('cp949'))
		htmls.append(html)

	return htmls

def get_time(text):

	time = list()

	for temp in text.split('\n'):

		temp = temp.split(' ')[0]

		if '-' in temp:

			for i in range(int(temp[2]), int(temp[4]) + 1):

				time.append(temp[0] + str(i))

		else:

			time.append(temp[0] + temp[2])

	return time

def get_room(text):

	room = list()

	for temp in text.split('\n'):

		room.append(temp.split(')')[1][1:])

	return list(set(room))

def get_lectures(htmls):

	lectures = list()

	for html in htmls:

		for row in html.find_all('tr'):

			lecture = dict()
			cols = row.find_all('td')

			if len(cols) == 17 and len(cols[7].text) > 0:

				try:

					if '토' in cols[7].text:

						pass

					else:

						lecture['haksu'] = cols[1].text.replace('\n', '')
						lecture['bunban'] = cols[2].text
						lecture['name'] = unicodedata.normalize('NFKC', cols[4].text.replace('\t', '').replace('\n', '').replace('\r', ''))[:-1]
						lecture['professor'] = cols[5].text
						lecture['hakjum'] = cols[6].text[2]
						lecture['time-room'] = cols[7].text
						lecture['time'] = get_time(cols[7].text)
						lecture['room'] = get_room(cols[7].text)

						lectures.append(lecture)

				except:

					pass

	return lectures

if __name__ == '__main__':

	# run in linux
	htmls = get_htmls('urlList.txt')
	lectures = get_lectures(htmls)
	save('2017-2.pickle', lectures)
	print(len(lectures))