From 5dc6e530cd7309c178a982fc905b96ea7d602682 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Thu, 4 Apr 2019 21:47:33 +0900 Subject: [PATCH 1/3] Add parse_MyPage to separate parsing logic and handler --- eviden/core.py | 12 +++++------- eviden/generator.py | 17 +++++++++++++++++ tests/test_generator.py | 22 +++++++++++++++++++--- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/eviden/core.py b/eviden/core.py index e0b2bbc..3a659a2 100644 --- a/eviden/core.py +++ b/eviden/core.py @@ -6,9 +6,9 @@ from .connection import get, get_with_session, post_with_session, authenticate from .generator import ( generate_hidden_params, - generate_project_info, generate_issues, - find_board_id + find_board_id, + parse_MyPage ) BASE_URL = "https://etrack.timedia.co.jp/EasyTracker/" @@ -74,13 +74,11 @@ def login(user_id, password): def list_projects(): URL = BASE_URL + "main/MyPage.aspx" - html = get_with_session(URL) + projects = parse_MyPage(html) - project_info = generate_project_info(html) - - for (group, name) in project_info: - print(f"{name}@{group}") + for project in projects: + print(f"{project.name}@{project.group}") def select_project(name): diff --git a/eviden/generator.py b/eviden/generator.py index af88ec8..4a8628e 100644 --- a/eviden/generator.py +++ b/eviden/generator.py @@ -1,5 +1,6 @@ from bs4 import BeautifulSoup import sys +from typing import Generator, NamedTuple HIDDEN_PARAMS = [ "__VIEWSTATE", @@ -62,3 +63,19 @@ def find_board_id(html, name): return project_name.a.get("href").split("=")[1] sys.exit("その名前のプロジェクトは存在しません") + + +Project = NamedTuple('Project', [('id', str), + ('name', str), + ('group', str)]) + + +def parse_MyPage(html: str) -> Generator[Project, None, None]: + root = BeautifulSoup(html, 'html.parser') + table = root.find(attrs={'id': '_ctl0_ContentPlaceHolder1_gridList'}) + rows = table.find_all('tr')[1:] # skip a table header row + + for row in rows: + group, name, last_updated, description, status = row.find_all('td', recursive=False) + project_id = name.a.get('href').split('=')[-1] + yield Project(project_id, name.text, group.text) diff --git a/tests/test_generator.py b/tests/test_generator.py index 2596c27..178af5c 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -7,6 +7,7 @@ generate_issues, generate_project_info, find_board_id, + parse_MyPage, HIDDEN_PARAMS ) @@ -71,9 +72,10 @@ def randomstr(b): project_info_tr_html = """ {0[0]} - {0[1]} - {0[2]} + {0[1]} {0[3]} + {0[4]} + {0[5]} """ @@ -142,7 +144,7 @@ def test_generate_hidden_params(self): def test_generate_project_info(self): N = random.randint(1, 50) - random_rows = [[randomstr(40) for _ in range(4)] for __ in range(N)] + random_rows = [[randomstr(40) for _ in range(6)] for __ in range(N)] rows = [project_info_tr_html.format(random_rows[i]) for i in range(N)] html = generate_project_info_html.format("".join(rows)) @@ -175,6 +177,20 @@ def test_find_board_id(self): self.assertEqual(data, board_id) + def test_parse_MyPage(self): + N = random.randint(1, 50) + + random_rows = [[randomstr(40) for _ in range(6)] for __ in range(N)] + rows = [project_info_tr_html.format(random_rows[i]) for i in range(N)] + + html = generate_project_info_html.format("".join(rows)) + + projects = parse_MyPage(html) + for i, project in enumerate(projects): + self.assertEqual(project.id, random_rows[i][2]) + self.assertEqual(project.name, random_rows[i][1]) + self.assertEqual(project.group, random_rows[i][0]) + if __name__ == "__main__": unittest.main() From 288c84d1ad06f5e63ca47772bbe13f786417d000 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Thu, 4 Apr 2019 21:47:34 +0900 Subject: [PATCH 2/3] Use parse_MyPage() in select_project() --- eviden/core.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/eviden/core.py b/eviden/core.py index 3a659a2..07e9b26 100644 --- a/eviden/core.py +++ b/eviden/core.py @@ -7,7 +7,6 @@ from .generator import ( generate_hidden_params, generate_issues, - find_board_id, parse_MyPage ) @@ -83,9 +82,15 @@ def list_projects(): def select_project(name): url = BASE_URL + "main/MyPage.aspx" - html = get_with_session(url) - board_id = find_board_id(html, name) + projects = parse_MyPage(html) + + for project in projects: + if project.name == name: + board_id = project.id + break + else: + sys.exit("その名前のプロジェクトは存在しません") status = read_json(STATUS_PATH) status["paramators"]["board_id"] = board_id From 556135dde48629ebee71b2edcc6d66e37598b386 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Thu, 4 Apr 2019 21:47:36 +0900 Subject: [PATCH 3/3] Remove generate_project_info() and find_board_id() (unused) --- eviden/generator.py | 28 ---------------------------- tests/test_generator.py | 28 ---------------------------- 2 files changed, 56 deletions(-) diff --git a/eviden/generator.py b/eviden/generator.py index 4a8628e..1aca9b7 100644 --- a/eviden/generator.py +++ b/eviden/generator.py @@ -1,5 +1,4 @@ from bs4 import BeautifulSoup -import sys from typing import Generator, NamedTuple HIDDEN_PARAMS = [ @@ -25,18 +24,6 @@ def generate_hidden_params(html, request="POST"): return data -def generate_project_info(html): - soup = BeautifulSoup(html, "html.parser") - - TABLE_ID = "_ctl0_ContentPlaceHolder1_gridList" - table = soup.find(attrs={"id": TABLE_ID}) - - rows = table.find_all("tr")[1:] - project_info = [list(map(__to_text, row.find_all("td")[:2])) for row in rows] - - return project_info - - def generate_issues(html): soup = BeautifulSoup(html, "html.parser") @@ -50,21 +37,6 @@ def generate_issues(html): return issues -def find_board_id(html, name): - soup = BeautifulSoup(html, "html.parser") - - TABLE_ID = "_ctl0_ContentPlaceHolder1_gridList" - table = soup.find(attrs={"id": TABLE_ID}) - rows = table.find_all("tr")[1:] - - for row in rows: - project_name = row.find_all("td")[1] - if project_name.text == name: - return project_name.a.get("href").split("=")[1] - - sys.exit("その名前のプロジェクトは存在しません") - - Project = NamedTuple('Project', [('id', str), ('name', str), ('group', str)]) diff --git a/tests/test_generator.py b/tests/test_generator.py index 178af5c..367864a 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -5,8 +5,6 @@ from eviden.generator import ( generate_hidden_params, generate_issues, - generate_project_info, - find_board_id, parse_MyPage, HIDDEN_PARAMS ) @@ -141,19 +139,6 @@ def test_generate_hidden_params(self): self.assertEqual(data, expected) - def test_generate_project_info(self): - N = random.randint(1, 50) - - random_rows = [[randomstr(40) for _ in range(6)] for __ in range(N)] - rows = [project_info_tr_html.format(random_rows[i]) for i in range(N)] - - html = generate_project_info_html.format("".join(rows)) - - data = generate_project_info(html) - expected = [random_rows[i][0:2] for i in range(N)] - - self.assertEqual(data, expected) - def test_generate_issues(self): N = random.randint(1, 50) random_rows = [[randomstr(40) for _ in range(7)] for __ in range(N)] @@ -164,19 +149,6 @@ def test_generate_issues(self): self.assertEqual(data, random_rows) - def test_find_board_id(self): - N = random.randint(1, 50) - pares = [[randomstr(40) for _ in range(3)] for __ in range(N)] - rows = [find_board_id_td_html.format(pares[i]) for i in range(N)] - - html = find_board_id_html.format("".join(rows)) - - M = random.randint(0, N - 1) - _, board_id, name = pares[M] - data = find_board_id(html, name) - - self.assertEqual(data, board_id) - def test_parse_MyPage(self): N = random.randint(1, 50)