From 214bbbf7ec5e844cde445072fdf1c412b61f5ddb Mon Sep 17 00:00:00 2001 From: Ryan Date: Thu, 10 Mar 2022 19:46:29 +1100 Subject: [PATCH 01/48] Add assignees stream --- README.md | 1 + tap_github/repository_streams.py | 33 ++++++++++++++++++++++++++++++++ tap_github/tap.py | 5 +++-- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 65cfd83d..7c249379 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ tap-github --config CONFIG --discover > ./catalog.json ``` ## Contributing +This project uses parent-child streams. Learn more about them [here.](https://gitlab.com/meltano/sdk/-/blob/main/docs/parent_streams.md) ### Initialize your Development Environment diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 95e310c4..49d11f59 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -547,6 +547,39 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]: ).to_dict() +class AssigneesStream(GitHubRestStream): + """Defines 'Assignees' stream which returns possible assignees for issues/prs following GitHub's API convention.""" + + name = "assignees" + path = "/repos/{org}/{repo}/assignees" + primary_keys = ["owner"] + replication_key = "updated_at" + parent_stream_type = RepositoryStream + ignore_parent_replication_key = False + state_partitioning_keys = ["repo", "org"] + + schema = th.PropertiesList( + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("login", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.IntegerType), + th.Property("followers_url", th.DateTimeType), + th.Property("following_url", th.DateTimeType), + th.Property("gists_url", th.DateTimeType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.IntegerType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType) + ).to_dict() + + class IssuesStream(GitHubRestStream): """Defines 'Issues' stream which returns Issues and PRs following GitHub's API convention.""" diff --git a/tap_github/tap.py b/tap_github/tap.py index 8affb5f1..f7d49075 100644 --- a/tap_github/tap.py +++ b/tap_github/tap.py @@ -20,7 +20,7 @@ ReadmeStream, RepositoryStream, StargazersStream, - StatsContributorsStream, + StatsContributorsStream, AssigneesStream, ) from tap_github.user_streams import ( StarredStream, @@ -104,9 +104,10 @@ def discover_streams(self) -> List[Stream]: CommunityProfileStream(tap=self), ContributorsStream(tap=self), EventsStream(tap=self), + AssigneesStream(tap=self), + IssuesStream(tap=self), IssueCommentsStream(tap=self), IssueEventsStream(tap=self), - IssuesStream(tap=self), LanguagesStream(tap=self), PullRequestsStream(tap=self), ReadmeHtmlStream(tap=self), From d2e10df46b665fab327437bfeb75d9c33827c2b9 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Fri, 11 Mar 2022 00:13:39 +1100 Subject: [PATCH 02/48] Fix issues with assignee stream --- tap_github/repository_streams.py | 7 +++---- tap_github/tap.py | 3 ++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 49d11f59..4ca09e53 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -552,16 +552,15 @@ class AssigneesStream(GitHubRestStream): name = "assignees" path = "/repos/{org}/{repo}/assignees" - primary_keys = ["owner"] - replication_key = "updated_at" + primary_keys = ["id"] parent_stream_type = RepositoryStream ignore_parent_replication_key = False state_partitioning_keys = ["repo", "org"] schema = th.PropertiesList( + th.Property("login", th.StringType), th.Property("id", th.IntegerType), th.Property("node_id", th.StringType), - th.Property("login", th.StringType), th.Property("avatar_url", th.StringType), th.Property("gravatar_id", th.StringType), th.Property("url", th.StringType), @@ -576,7 +575,7 @@ class AssigneesStream(GitHubRestStream): th.Property("events_url", th.StringType), th.Property("received_events_url", th.StringType), th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType) + th.Property("site_admin", th.BooleanType), ).to_dict() diff --git a/tap_github/tap.py b/tap_github/tap.py index f7d49075..20e6dfd9 100644 --- a/tap_github/tap.py +++ b/tap_github/tap.py @@ -20,7 +20,8 @@ ReadmeStream, RepositoryStream, StargazersStream, - StatsContributorsStream, AssigneesStream, + StatsContributorsStream, + AssigneesStream, ) from tap_github.user_streams import ( StarredStream, From 08aa7e778f43742ce47905be75bf59971bfc313e Mon Sep 17 00:00:00 2001 From: Ryan Date: Fri, 11 Mar 2022 00:50:12 +1100 Subject: [PATCH 03/48] Add collaborators stream --- tap_github/repository_streams.py | 41 ++++++++++++++++++++++++++++++++ tap_github/tap.py | 2 ++ 2 files changed, 43 insertions(+) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 4ca09e53..8a88d406 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -547,6 +547,47 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]: ).to_dict() +class CollaboratorsStream(GitHubRestStream): + name = "collaborators" + path = "/repos/{org}/{repo}/collaborators" + primary_keys = ["id"] + parent_stream_type = RepositoryStream + ignore_parent_replication_key = False + state_partitioning_keys = ["repo", "org"] + + schema = th.PropertiesList( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.IntegerType), + th.Property("followers_url", th.DateTimeType), + th.Property("following_url", th.DateTimeType), + th.Property("gists_url", th.DateTimeType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.IntegerType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + th.Property( + "permissions", + th.ObjectType( + th.Property("pull", th.BooleanType), + th.Property("triage", th.BooleanType), + th.Property("push", th.BooleanType), + th.Property("maintain", th.BooleanType), + th.Property("admin", th.BooleanType), + ), + ), + th.Property("role_name", th.StringType), + ).to_dict() + + class AssigneesStream(GitHubRestStream): """Defines 'Assignees' stream which returns possible assignees for issues/prs following GitHub's API convention.""" diff --git a/tap_github/tap.py b/tap_github/tap.py index 20e6dfd9..0fde8df3 100644 --- a/tap_github/tap.py +++ b/tap_github/tap.py @@ -22,6 +22,7 @@ StargazersStream, StatsContributorsStream, AssigneesStream, + CollaboratorsStream, ) from tap_github.user_streams import ( StarredStream, @@ -105,6 +106,7 @@ def discover_streams(self) -> List[Stream]: CommunityProfileStream(tap=self), ContributorsStream(tap=self), EventsStream(tap=self), + CollaboratorsStream(tap=self), AssigneesStream(tap=self), IssuesStream(tap=self), IssueCommentsStream(tap=self), From 231596bcbf318a08ccbb18343e27073c389aad5d Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Fri, 11 Mar 2022 01:51:29 +1100 Subject: [PATCH 04/48] Add review comments and reviews stream --- tap_github/repository_streams.py | 128 +++++++++++++++++++++++++++++++ tap_github/tap.py | 4 + 2 files changed, 132 insertions(+) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 8a88d406..895549f6 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -1075,6 +1075,13 @@ def post_process(self, row: dict, context: Optional[dict] = None) -> dict: row["minus_one"] = row.pop("-1", None) return row + def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + return { + "org": context["org"], + "repo": context["repo"], + "pull_number": record["number"], + } + schema = th.PropertiesList( # Parent keys th.Property("repo", th.StringType), @@ -1295,6 +1302,127 @@ def post_process(self, row: dict, context: Optional[dict] = None) -> dict: ).to_dict() +class ReviewsStream(GitHubRestStream): + name = "reviews" + path = "/repos/{org}/{repo}/pulls/{pull_number}/reviews" + primary_keys = ["id"] + parent_stream_type = PullRequestsStream + ignore_parent_replication_key = False + state_partitioning_keys = ["pull_number"] + + schema = th.PropertiesList( + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property( + "user", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), + th.Property("body", th.StringType), + th.Property("state", th.StringType), + th.Property("html_url", th.StringType), + th.Property("pull_request_url", th.StringType), + th.Property( + "_links", + th.ObjectType( + th.Property("html", th.ObjectType(th.Property("href", th.StringType))), + th.Property( + "pull_request", th.ObjectType(th.Property("href", th.StringType)) + ), + ), + ), + th.Property("submitted_at", th.StringType), + th.Property("commit_id", th.StringType), + th.Property("author_association", th.StringType), + ).to_dict() + + +class ReviewCommentsStream(GitHubRestStream): + name = "review_comments" + path = "/repos/{org}/{repo}/pulls/{pull_number}/reviews" + primary_keys = ["id"] + parent_stream_type = PullRequestsStream + ignore_parent_replication_key = False + state_partitioning_keys = ["pull_number"] + + schema = th.PropertiesList( + th.Property("url", th.StringType), + th.Property("pull_request_review_id", th.IntegerType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("diff_hunk", th.StringType), + th.Property("path", th.StringType), + th.Property("position", th.IntegerType), + th.Property("original_position", th.IntegerType), + th.Property("commit_id", th.StringType), + th.Property("original_commit_id", th.StringType), + th.Property("in_reply_to_id", th.IntegerType), + th.Property( + "user", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), + th.Property("body", th.StringType), + th.Property("created_at", th.StringType), + th.Property("updated_at", th.StringType), + th.Property("html_url", th.StringType), + th.Property("pull_request_url", th.StringType), + th.Property("author_association", th.StringType), + th.Property( + "_links", + th.ObjectType( + th.Property("self", th.ObjectType(th.Property("href", th.StringType))), + th.Property("html", th.ObjectType(th.Property("href", th.StringType))), + th.Property( + "pull_request", th.ObjectType(th.Property("href", th.StringType)) + ), + ), + ), + th.Property("start_line", th.IntegerType), + th.Property("original_start_line", th.IntegerType), + th.Property("start_side", th.StringType), + th.Property("line", th.IntegerType), + th.Property("original_line", th.IntegerType), + th.Property("side", th.StringType), + ).to_dict() + + class ContributorsStream(GitHubRestStream): """Defines 'Contributors' stream. Fetching User & Bot contributors.""" diff --git a/tap_github/tap.py b/tap_github/tap.py index 0fde8df3..3ad1cbba 100644 --- a/tap_github/tap.py +++ b/tap_github/tap.py @@ -23,6 +23,8 @@ StatsContributorsStream, AssigneesStream, CollaboratorsStream, + ReviewsStream, + ReviewCommentsStream, ) from tap_github.user_streams import ( StarredStream, @@ -113,6 +115,8 @@ def discover_streams(self) -> List[Stream]: IssueEventsStream(tap=self), LanguagesStream(tap=self), PullRequestsStream(tap=self), + ReviewsStream(tap=self), + ReviewCommentsStream(tap=self), ReadmeHtmlStream(tap=self), ReadmeStream(tap=self), RepositoryStream(tap=self), From 4d967cf1d837320a707781c804d7c6d3f6a29181 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Fri, 11 Mar 2022 01:59:54 +1100 Subject: [PATCH 05/48] Fix review comments stream and use repo parent instead --- tap_github/repository_streams.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 895549f6..7c7c187e 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -1357,11 +1357,11 @@ class ReviewsStream(GitHubRestStream): class ReviewCommentsStream(GitHubRestStream): name = "review_comments" - path = "/repos/{org}/{repo}/pulls/{pull_number}/reviews" + path = "/repos/{org}/{repo}/pulls/comments" primary_keys = ["id"] - parent_stream_type = PullRequestsStream + parent_stream_type = RepositoryStream ignore_parent_replication_key = False - state_partitioning_keys = ["pull_number"] + state_partitioning_keys = ["repo", "org"] schema = th.PropertiesList( th.Property("url", th.StringType), From e3b01bd58c5f9af4958461dec16f460032659566 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Fri, 11 Mar 2022 02:05:17 +1100 Subject: [PATCH 06/48] Fix mypy issue --- tap_github/repository_streams.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 7c7c187e..11ef7b6f 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -1076,10 +1076,16 @@ def post_process(self, row: dict, context: Optional[dict] = None) -> dict: return row def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + if context: + return { + "org": context["org"], + "repo": context["repo"], + "pull_number": record["number"], + } return { - "org": context["org"], - "repo": context["repo"], "pull_number": record["number"], + "org": record["base"]["user"]["login"], + "repo": record["base"]["repo"]["name"], } schema = th.PropertiesList( From f2d7f53d36c631c4e99aadb70ea942f6f8b70516 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Sat, 12 Mar 2022 22:13:14 +1100 Subject: [PATCH 07/48] Fix tests --- tap_github/repository_streams.py | 4 +--- tap_github/tests/test_tap.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 11ef7b6f..8845c42f 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -1077,9 +1077,7 @@ def post_process(self, row: dict, context: Optional[dict] = None) -> dict: def get_child_context(self, record: dict, context: Optional[dict]) -> dict: if context: - return { - "org": context["org"], - "repo": context["repo"], + return context | { "pull_number": record["number"], } return { diff --git a/tap_github/tests/test_tap.py b/tap_github/tests/test_tap.py index 1d29ab62..467331ba 100644 --- a/tap_github/tests/test_tap.py +++ b/tap_github/tests/test_tap.py @@ -10,7 +10,7 @@ from .fixtures import repo_list_config -repo_list_2 = ["octocat/hello-world", "MeltanoLabs/tap-github", "mapswipe/mapswipe"] +repo_list_2 = ["MeltanoLabs/tap-github"] @pytest.mark.repo_list(repo_list_2) From 7f17a656244b85331646c753e73f5c19d54b97c1 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Sat, 12 Mar 2022 22:39:48 +1100 Subject: [PATCH 08/48] add milestone and commit comment streams --- tap_github/repository_streams.py | 100 +++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 8845c42f..72dbb816 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -519,6 +519,57 @@ def post_process(self, row: dict, context: Optional[dict] = None) -> dict: ).to_dict() +class MilestonesStream(GitHubRestStream): + name = "milestones" + path = "/repos/{org}/{repo}/milestones" + primary_keys = ["id"] + replication_key = "updated_at" + parent_stream_type = RepositoryStream + state_partitioning_keys = ["repo", "org"] + ignore_parent_replication_key = False + + schema = th.PropertiesList( + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("labels_url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("number", th.IntegerType), + th.Property("state", th.StringType), + th.Property("title", th.StringType), + th.Property("description", th.StringType), + th.Property( + "creator", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), + th.Property("open_issues", th.IntegerType), + th.Property("closed_issues", th.IntegerType), + th.Property("created_at", th.StringType), + th.Property("updated_at", th.StringType), + th.Property("closed_at", th.StringType), + th.Property("due_on", th.StringType), + ).to_dict() + + class LanguagesStream(GitHubRestStream): name = "languages" path = "/repos/{org}/{repo}/languages" @@ -1028,6 +1079,55 @@ def post_process(self, row: dict, context: Optional[dict] = None) -> dict: ).to_dict() +class CommitCommentsStream(GitHubRestStream): + + name = "commit_comment" + path = "/repos/{org}/{repo}/comments" + primary_keys = ["id"] + replication_key = "updated_at" + parent_stream_type = RepositoryStream + state_partitioning_keys = ["repo", "org"] + ignore_parent_replication_key = False + + schema = th.PropertiesList( + th.Property("html_url", th.StringType), + th.Property("url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("body", th.StringType), + th.Property("path", th.StringType), + th.Property("position", th.IntegerType), + th.Property("line", th.IntegerType), + th.Property("commit_id", th.StringType), + th.Property( + "user", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), + th.Property("created_at", th.StringType), + th.Property("updated_at", th.StringType), + th.Property("author_association", th.StringType), + ).to_dict() + + class PullRequestsStream(GitHubRestStream): """Defines 'PullRequests' stream.""" From 313b74238b920aa9023b2d182958b3867ace51f5 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Sat, 12 Mar 2022 22:57:08 +1100 Subject: [PATCH 09/48] Fix mypy --- tap_github/repository_streams.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 72dbb816..b5dde675 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -1177,7 +1177,9 @@ def post_process(self, row: dict, context: Optional[dict] = None) -> dict: def get_child_context(self, record: dict, context: Optional[dict]) -> dict: if context: - return context | { + return { + "org": context["org"], + "repo": context["repo"], "pull_number": record["number"], } return { From 9ea76f5bc7fa3a9a0a1b39483b59bfb969f00fe4 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Sun, 13 Mar 2022 12:43:18 +1100 Subject: [PATCH 10/48] Fix tests --- tap_github/tests/test_tap.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tap_github/tests/test_tap.py b/tap_github/tests/test_tap.py index 467331ba..505a1c0d 100644 --- a/tap_github/tests/test_tap.py +++ b/tap_github/tests/test_tap.py @@ -17,9 +17,7 @@ def test_validate_repo_list_config(repo_list_config): """Verify that the repositories list is parsed correctly""" repo_list_context = [ - {"org": "octocat", "repo": "hello-world"}, - {"org": "MeltanoLabs", "repo": "tap-github"}, - {"org": "mapswipe", "repo": "mapswipe"}, + {"org": repo.split("/")[0], "repo": repo.split("/")[1]} for repo in repo_list_2 ] tap = TapGitHub(config=repo_list_config) partitions = tap.streams["repositories"].partitions From 2b66a730dfb62aa749dadcfb248be869f42ed59a Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Sun, 13 Mar 2022 14:33:11 +1100 Subject: [PATCH 11/48] commit wip todo streams --- todo_streams.py | 278 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100644 todo_streams.py diff --git a/todo_streams.py b/todo_streams.py new file mode 100644 index 00000000..46be96ba --- /dev/null +++ b/todo_streams.py @@ -0,0 +1,278 @@ +from ff import th +# projects +schema=th.PropertiesList( + th.Property("owner_url", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("columns_url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("name", th.StringType), + th.Property("body", th.StringType), + th.Property("number", th.IntegerType), + th.Property("state", th.StringType), + th.Property("creator", th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType) + ),), + th.Property("created_at", th.StringType), + th.Property("updated_at", th.StringType) + ).to_dict() + +# project cards +schema=th.PropertiesList( + th.Property("url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("note", th.StringType), + th.Property("creator", th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType) + ),), + th.Property("created_at", th.StringType), + th.Property("updated_at", th.StringType), + th.Property("archived", th.BooleanType), + th.Property("column_url", th.StringType), + th.Property("content_url", th.StringType), + th.Property("project_url", th.StringType) + ).to_dict() +# project columns +schema=th.PropertiesList( + th.Property("url", th.StringType), + th.Property("project_url", th.StringType), + th.Property("cards_url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("name", th.StringType), + th.Property("created_at", th.StringType), + th.Property("updated_at", th.StringType) + ).to_dict() +# pr commits +schema=th.PropertiesList( + th.Property("url", th.StringType), + th.Property("sha", th.StringType), + th.Property("node_id", th.StringType), + th.Property("html_url", th.StringType), + th.Property("comments_url", th.StringType), + th.Property("commit", th.ObjectType( + th.Property("url", th.StringType), + th.Property("author", th.ObjectType( + th.Property("name", th.StringType), + th.Property("email", th.StringType), + th.Property("date", th.StringType) + ),), + th.Property("committer", th.ObjectType( + th.Property("name", th.StringType), + th.Property("email", th.StringType), + th.Property("date", th.StringType) + ),), + th.Property("message", th.StringType), + th.Property("tree", th.ObjectType( + th.Property("url", th.StringType), + th.Property("sha", th.StringType) + ),), + th.Property("comment_count", th.IntegerType), + th.Property("verification", th.ObjectType( + th.Property("verified", th.BooleanType), + th.Property("reason", th.StringType), + "signature": null, + "payload": null + ),), + ),), + th.Property("author", th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType) + ),), + th.Property("committer", th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType) + ),), + th.Property("parents",th.ArrayType( + + th.Property("url", th.StringType), + th.Property("sha", th.StringType) + ) + ).to_dict() + + +# releases +schema=th.PropertiesList( + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("assets_url", th.StringType), + th.Property("upload_url", th.StringType), + th.Property("tarball_url", th.StringType), + th.Property("zipball_url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("tag_name", th.StringType), + th.Property("target_commitish", th.StringType), + th.Property("name", th.StringType), + th.Property("body", th.StringType), + th.Property("draft", th.BooleanType), + th.Property("prerelease", th.BooleanType), + th.Property("created_at", th.StringType), + th.Property("published_at", th.StringType), + th.Property("author", th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType) + ),), + th.Property("assets",th.ArrayType( + + th.Property("url", th.StringType), + th.Property("browser_download_url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("name", th.StringType), + th.Property("label", th.StringType), + th.Property("state", th.StringType), + th.Property("content_type", th.StringType), + th.Property("size", th.IntegerType), + th.Property("download_count", th.IntegerType), + th.Property("created_at", th.StringType), + th.Property("updated_at", th.StringType), + th.Property("uploader", th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType) + ),), + ).to_dict() +# teams + schema=th.PropertiesList( + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("name", th.StringType), + th.Property("slug", th.StringType), + th.Property("description", th.StringType), + th.Property("privacy", th.StringType), + th.Property("permission", th.StringType), + th.Property("members_url", th.StringType), + th.Property("repositories_url", th.StringType), + "parent": null + ).to_dict() + +# team members +schema=th.PropertiesList( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType) + ).to_dict() + +# team roles +schema=th.PropertiesList( + th.Property("url", th.StringType), + th.Property("role", th.StringType), + th.Property("state", th.StringType) +).to_dict() \ No newline at end of file From b27871d8f0545f0c577944c7c5dfcf535a1c50f7 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Sun, 13 Mar 2022 14:36:05 +1100 Subject: [PATCH 12/48] fix formatting --- todo_streams.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/todo_streams.py b/todo_streams.py index 46be96ba..94f6fe50 100644 --- a/todo_streams.py +++ b/todo_streams.py @@ -107,8 +107,8 @@ th.Property("verification", th.ObjectType( th.Property("verified", th.BooleanType), th.Property("reason", th.StringType), - "signature": null, - "payload": null + th.Property("signature",th.StringType), + th.Property("payload",th.StringType) ),), ),), th.Property("author", th.ObjectType( @@ -155,7 +155,7 @@ th.Property("url", th.StringType), th.Property("sha", th.StringType) - ) + )) ).to_dict() @@ -230,23 +230,23 @@ th.Property("received_events_url", th.StringType), th.Property("type", th.StringType), th.Property("site_admin", th.BooleanType) - ),), - ).to_dict() + ),)) + )).to_dict() # teams - schema=th.PropertiesList( - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("name", th.StringType), - th.Property("slug", th.StringType), - th.Property("description", th.StringType), - th.Property("privacy", th.StringType), - th.Property("permission", th.StringType), - th.Property("members_url", th.StringType), - th.Property("repositories_url", th.StringType), - "parent": null - ).to_dict() +schema=th.PropertiesList( + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("name", th.StringType), + th.Property("slug", th.StringType), + th.Property("description", th.StringType), + th.Property("privacy", th.StringType), + th.Property("permission", th.StringType), + th.Property("members_url", th.StringType), + th.Property("repositories_url", th.StringType), + th.Property("parent",th.StringType) + ).to_dict() # team members schema=th.PropertiesList( From ff14da1a1073c62d6cbf36891d2eac494bb784f4 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Sun, 13 Mar 2022 14:39:58 +1100 Subject: [PATCH 13/48] [ci skip] format todo file and fix arraytype usage --- todo_streams.py | 441 ++++++++++++++++++++++++++---------------------- 1 file changed, 241 insertions(+), 200 deletions(-) diff --git a/todo_streams.py b/todo_streams.py index 94f6fe50..cd9a98b0 100644 --- a/todo_streams.py +++ b/todo_streams.py @@ -1,6 +1,7 @@ -from ff import th +from singer_sdk import typing as th # JSON Schema typing helpers + # projects -schema=th.PropertiesList( +schema = th.PropertiesList( th.Property("owner_url", th.StringType), th.Property("url", th.StringType), th.Property("html_url", th.StringType), @@ -11,65 +12,71 @@ th.Property("body", th.StringType), th.Property("number", th.IntegerType), th.Property("state", th.StringType), - th.Property("creator", th.ObjectType( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType) - ),), + th.Property( + "creator", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), th.Property("created_at", th.StringType), - th.Property("updated_at", th.StringType) - ).to_dict() + th.Property("updated_at", th.StringType), +).to_dict() # project cards -schema=th.PropertiesList( +schema = th.PropertiesList( th.Property("url", th.StringType), th.Property("id", th.IntegerType), th.Property("node_id", th.StringType), th.Property("note", th.StringType), - th.Property("creator", th.ObjectType( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType) - ),), + th.Property( + "creator", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), th.Property("created_at", th.StringType), th.Property("updated_at", th.StringType), th.Property("archived", th.BooleanType), th.Property("column_url", th.StringType), th.Property("content_url", th.StringType), - th.Property("project_url", th.StringType) - ).to_dict() + th.Property("project_url", th.StringType), +).to_dict() # project columns -schema=th.PropertiesList( +schema = th.PropertiesList( th.Property("url", th.StringType), th.Property("project_url", th.StringType), th.Property("cards_url", th.StringType), @@ -77,90 +84,113 @@ th.Property("node_id", th.StringType), th.Property("name", th.StringType), th.Property("created_at", th.StringType), - th.Property("updated_at", th.StringType) - ).to_dict() + th.Property("updated_at", th.StringType), +).to_dict() # pr commits -schema=th.PropertiesList( +schema = th.PropertiesList( th.Property("url", th.StringType), th.Property("sha", th.StringType), th.Property("node_id", th.StringType), th.Property("html_url", th.StringType), th.Property("comments_url", th.StringType), - th.Property("commit", th.ObjectType( - th.Property("url", th.StringType), - th.Property("author", th.ObjectType( - th.Property("name", th.StringType), - th.Property("email", th.StringType), - th.Property("date", th.StringType) - ),), - th.Property("committer", th.ObjectType( - th.Property("name", th.StringType), - th.Property("email", th.StringType), - th.Property("date", th.StringType) - ),), - th.Property("message", th.StringType), - th.Property("tree", th.ObjectType( - th.Property("url", th.StringType), - th.Property("sha", th.StringType) - ),), - th.Property("comment_count", th.IntegerType), - th.Property("verification", th.ObjectType( - th.Property("verified", th.BooleanType), - th.Property("reason", th.StringType), - th.Property("signature",th.StringType), - th.Property("payload",th.StringType) - ),), - ),), - th.Property("author", th.ObjectType( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType) - ),), - th.Property("committer", th.ObjectType( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType) - ),), - th.Property("parents",th.ArrayType( - - th.Property("url", th.StringType), - th.Property("sha", th.StringType) - )) - ).to_dict() + th.Property( + "commit", + th.ObjectType( + th.Property("url", th.StringType), + th.Property( + "author", + th.ObjectType( + th.Property("name", th.StringType), + th.Property("email", th.StringType), + th.Property("date", th.StringType), + ), + ), + th.Property( + "committer", + th.ObjectType( + th.Property("name", th.StringType), + th.Property("email", th.StringType), + th.Property("date", th.StringType), + ), + ), + th.Property("message", th.StringType), + th.Property( + "tree", + th.ObjectType( + th.Property("url", th.StringType), th.Property("sha", th.StringType) + ), + ), + th.Property("comment_count", th.IntegerType), + th.Property( + "verification", + th.ObjectType( + th.Property("verified", th.BooleanType), + th.Property("reason", th.StringType), + th.Property("signature", th.StringType), + th.Property("payload", th.StringType), + ), + ), + ), + ), + th.Property( + "author", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), + th.Property( + "committer", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), + th.Property( + "parents", + th.ArrayType( + th.ObjectType( + th.Property("url", th.StringType), th.Property("sha", th.StringType) + ) + ), + ), +).to_dict() # releases -schema=th.PropertiesList( +schema = th.PropertiesList( th.Property("url", th.StringType), th.Property("html_url", th.StringType), th.Property("assets_url", th.StringType), @@ -177,79 +207,90 @@ th.Property("prerelease", th.BooleanType), th.Property("created_at", th.StringType), th.Property("published_at", th.StringType), - th.Property("author", th.ObjectType( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType) - ),), - th.Property("assets",th.ArrayType( - - th.Property("url", th.StringType), - th.Property("browser_download_url", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("name", th.StringType), - th.Property("label", th.StringType), - th.Property("state", th.StringType), - th.Property("content_type", th.StringType), - th.Property("size", th.IntegerType), - th.Property("download_count", th.IntegerType), - th.Property("created_at", th.StringType), - th.Property("updated_at", th.StringType), - th.Property("uploader", th.ObjectType( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType) - ),)) - )).to_dict() + th.Property( + "author", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), + th.Property( + "assets", + th.ArrayType( + th.ObjectType( + th.Property("url", th.StringType), + th.Property("browser_download_url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("name", th.StringType), + th.Property("label", th.StringType), + th.Property("state", th.StringType), + th.Property("content_type", th.StringType), + th.Property("size", th.IntegerType), + th.Property("download_count", th.IntegerType), + th.Property("created_at", th.StringType), + th.Property("updated_at", th.StringType), + th.Property( + "uploader", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), + ) + ), + ), +).to_dict() # teams -schema=th.PropertiesList( - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("name", th.StringType), - th.Property("slug", th.StringType), - th.Property("description", th.StringType), - th.Property("privacy", th.StringType), - th.Property("permission", th.StringType), - th.Property("members_url", th.StringType), - th.Property("repositories_url", th.StringType), - th.Property("parent",th.StringType) - ).to_dict() +schema = th.PropertiesList( + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("name", th.StringType), + th.Property("slug", th.StringType), + th.Property("description", th.StringType), + th.Property("privacy", th.StringType), + th.Property("permission", th.StringType), + th.Property("members_url", th.StringType), + th.Property("repositories_url", th.StringType), + th.Property("parent", th.StringType), +).to_dict() # team members -schema=th.PropertiesList( +schema = th.PropertiesList( th.Property("login", th.StringType), th.Property("id", th.IntegerType), th.Property("node_id", th.StringType), @@ -267,12 +308,12 @@ th.Property("events_url", th.StringType), th.Property("received_events_url", th.StringType), th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType) - ).to_dict() + th.Property("site_admin", th.BooleanType), +).to_dict() # team roles -schema=th.PropertiesList( - th.Property("url", th.StringType), - th.Property("role", th.StringType), - th.Property("state", th.StringType) -).to_dict() \ No newline at end of file +schema = th.PropertiesList( + th.Property("url", th.StringType), + th.Property("role", th.StringType), + th.Property("state", th.StringType), +).to_dict() From f146c2a0f0fa30ec5f62cb5135209bc6271e6fdf Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Sun, 13 Mar 2022 14:50:19 +1100 Subject: [PATCH 14/48] [ci skip] more regex magic to convert everything to classes --- todo_streams.py | 615 +++++++++++++++++++++++++----------------------- 1 file changed, 318 insertions(+), 297 deletions(-) diff --git a/todo_streams.py b/todo_streams.py index cd9a98b0..7e9e920f 100644 --- a/todo_streams.py +++ b/todo_streams.py @@ -1,319 +1,340 @@ -from singer_sdk import typing as th # JSON Schema typing helpers +from singer_sdk import typing as th -# projects -schema = th.PropertiesList( - th.Property("owner_url", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("columns_url", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("name", th.StringType), - th.Property("body", th.StringType), - th.Property("number", th.IntegerType), - th.Property("state", th.StringType), - th.Property( - "creator", - th.ObjectType( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType), +from tap_github.client import GitHubRestStream + + +class ProjectsStream(GitHubRestStream): + name = "projects" + schema = th.PropertiesList( + th.Property("owner_url", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("columns_url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("name", th.StringType), + th.Property("body", th.StringType), + th.Property("number", th.IntegerType), + th.Property("state", th.StringType), + th.Property( + "creator", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), ), - ), - th.Property("created_at", th.StringType), - th.Property("updated_at", th.StringType), -).to_dict() + th.Property("created_at", th.StringType), + th.Property("updated_at", th.StringType), + ).to_dict() -# project cards -schema = th.PropertiesList( - th.Property("url", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("note", th.StringType), - th.Property( - "creator", - th.ObjectType( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType), + +class ProjectCardsStream(GitHubRestStream): + name = "project_cards" + schema = th.PropertiesList( + th.Property("url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("note", th.StringType), + th.Property( + "creator", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), ), - ), - th.Property("created_at", th.StringType), - th.Property("updated_at", th.StringType), - th.Property("archived", th.BooleanType), - th.Property("column_url", th.StringType), - th.Property("content_url", th.StringType), - th.Property("project_url", th.StringType), -).to_dict() -# project columns -schema = th.PropertiesList( - th.Property("url", th.StringType), - th.Property("project_url", th.StringType), - th.Property("cards_url", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("name", th.StringType), - th.Property("created_at", th.StringType), - th.Property("updated_at", th.StringType), -).to_dict() -# pr commits -schema = th.PropertiesList( - th.Property("url", th.StringType), - th.Property("sha", th.StringType), - th.Property("node_id", th.StringType), - th.Property("html_url", th.StringType), - th.Property("comments_url", th.StringType), - th.Property( - "commit", - th.ObjectType( - th.Property("url", th.StringType), - th.Property( - "author", - th.ObjectType( - th.Property("name", th.StringType), - th.Property("email", th.StringType), - th.Property("date", th.StringType), + th.Property("created_at", th.StringType), + th.Property("updated_at", th.StringType), + th.Property("archived", th.BooleanType), + th.Property("column_url", th.StringType), + th.Property("content_url", th.StringType), + th.Property("project_url", th.StringType), + ).to_dict() + + +class ProjectColumnsStream(GitHubRestStream): + name = "project_columns" + schema = th.PropertiesList( + th.Property("url", th.StringType), + th.Property("project_url", th.StringType), + th.Property("cards_url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("name", th.StringType), + th.Property("created_at", th.StringType), + th.Property("updated_at", th.StringType), + ).to_dict() + + +class PrCommitsStream(GitHubRestStream): + name = "pr_commits" + schema = th.PropertiesList( + th.Property("url", th.StringType), + th.Property("sha", th.StringType), + th.Property("node_id", th.StringType), + th.Property("html_url", th.StringType), + th.Property("comments_url", th.StringType), + th.Property( + "commit", + th.ObjectType( + th.Property("url", th.StringType), + th.Property( + "author", + th.ObjectType( + th.Property("name", th.StringType), + th.Property("email", th.StringType), + th.Property("date", th.StringType), + ), ), - ), - th.Property( - "committer", - th.ObjectType( - th.Property("name", th.StringType), - th.Property("email", th.StringType), - th.Property("date", th.StringType), + th.Property( + "committer", + th.ObjectType( + th.Property("name", th.StringType), + th.Property("email", th.StringType), + th.Property("date", th.StringType), + ), ), - ), - th.Property("message", th.StringType), - th.Property( - "tree", - th.ObjectType( - th.Property("url", th.StringType), th.Property("sha", th.StringType) + th.Property("message", th.StringType), + th.Property( + "tree", + th.ObjectType( + th.Property("url", th.StringType), + th.Property("sha", th.StringType), + ), ), - ), - th.Property("comment_count", th.IntegerType), - th.Property( - "verification", - th.ObjectType( - th.Property("verified", th.BooleanType), - th.Property("reason", th.StringType), - th.Property("signature", th.StringType), - th.Property("payload", th.StringType), + th.Property("comment_count", th.IntegerType), + th.Property( + "verification", + th.ObjectType( + th.Property("verified", th.BooleanType), + th.Property("reason", th.StringType), + th.Property("signature", th.StringType), + th.Property("payload", th.StringType), + ), ), ), ), - ), - th.Property( - "author", - th.ObjectType( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType), - ), - ), - th.Property( - "committer", - th.ObjectType( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType), + th.Property( + "author", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), ), - ), - th.Property( - "parents", - th.ArrayType( + th.Property( + "committer", th.ObjectType( - th.Property("url", th.StringType), th.Property("sha", th.StringType) - ) + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), + th.Property( + "parents", + th.ArrayType( + th.ObjectType( + th.Property("url", th.StringType), th.Property("sha", th.StringType) + ) + ), ), - ), -).to_dict() + ).to_dict() -# releases -schema = th.PropertiesList( - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("assets_url", th.StringType), - th.Property("upload_url", th.StringType), - th.Property("tarball_url", th.StringType), - th.Property("zipball_url", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("tag_name", th.StringType), - th.Property("target_commitish", th.StringType), - th.Property("name", th.StringType), - th.Property("body", th.StringType), - th.Property("draft", th.BooleanType), - th.Property("prerelease", th.BooleanType), - th.Property("created_at", th.StringType), - th.Property("published_at", th.StringType), - th.Property( - "author", - th.ObjectType( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType), - ), - ), - th.Property( - "assets", - th.ArrayType( +class ReleasesStream(GitHubRestStream): + name = "releases" + schema = th.PropertiesList( + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("assets_url", th.StringType), + th.Property("upload_url", th.StringType), + th.Property("tarball_url", th.StringType), + th.Property("zipball_url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("tag_name", th.StringType), + th.Property("target_commitish", th.StringType), + th.Property("name", th.StringType), + th.Property("body", th.StringType), + th.Property("draft", th.BooleanType), + th.Property("prerelease", th.BooleanType), + th.Property("created_at", th.StringType), + th.Property("published_at", th.StringType), + th.Property( + "author", th.ObjectType( - th.Property("url", th.StringType), - th.Property("browser_download_url", th.StringType), + th.Property("login", th.StringType), th.Property("id", th.IntegerType), th.Property("node_id", th.StringType), - th.Property("name", th.StringType), - th.Property("label", th.StringType), - th.Property("state", th.StringType), - th.Property("content_type", th.StringType), - th.Property("size", th.IntegerType), - th.Property("download_count", th.IntegerType), - th.Property("created_at", th.StringType), - th.Property("updated_at", th.StringType), - th.Property( - "uploader", - th.ObjectType( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), + th.Property( + "assets", + th.ArrayType( + th.ObjectType( + th.Property("url", th.StringType), + th.Property("browser_download_url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("name", th.StringType), + th.Property("label", th.StringType), + th.Property("state", th.StringType), + th.Property("content_type", th.StringType), + th.Property("size", th.IntegerType), + th.Property("download_count", th.IntegerType), + th.Property("created_at", th.StringType), + th.Property("updated_at", th.StringType), + th.Property( + "uploader", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), ), - ), - ) + ) + ), ), - ), -).to_dict() -# teams -schema = th.PropertiesList( - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("name", th.StringType), - th.Property("slug", th.StringType), - th.Property("description", th.StringType), - th.Property("privacy", th.StringType), - th.Property("permission", th.StringType), - th.Property("members_url", th.StringType), - th.Property("repositories_url", th.StringType), - th.Property("parent", th.StringType), -).to_dict() + ).to_dict() + + +class TeamsStream(GitHubRestStream): + name = "teams" + schema = th.PropertiesList( + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("name", th.StringType), + th.Property("slug", th.StringType), + th.Property("description", th.StringType), + th.Property("privacy", th.StringType), + th.Property("permission", th.StringType), + th.Property("members_url", th.StringType), + th.Property("repositories_url", th.StringType), + th.Property("parent", th.StringType), + ).to_dict() + + +class TeamMembersStream(GitHubRestStream): + name = "team_members" + schema = th.PropertiesList( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ).to_dict() -# team members -schema = th.PropertiesList( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType), -).to_dict() -# team roles -schema = th.PropertiesList( - th.Property("url", th.StringType), - th.Property("role", th.StringType), - th.Property("state", th.StringType), -).to_dict() +class TeamRolesStream(GitHubRestStream): + name = "team_roles" + schema = th.PropertiesList( + th.Property("url", th.StringType), + th.Property("role", th.StringType), + th.Property("state", th.StringType), + ).to_dict() From 89fb408b59178a0e17629ddb18dbd2ceab634c5b Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Sun, 13 Mar 2022 19:15:08 +1100 Subject: [PATCH 15/48] Add paths [ci skip] --- todo_streams.py | 65 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/todo_streams.py b/todo_streams.py index 7e9e920f..a161b38c 100644 --- a/todo_streams.py +++ b/todo_streams.py @@ -1,10 +1,21 @@ +from typing import Optional + from singer_sdk import typing as th from tap_github.client import GitHubRestStream +from tap_github.repository_streams import PullRequestsStream, RepositoryStream class ProjectsStream(GitHubRestStream): name = "projects" + path = "/repos/{org}/{repo}/projects" + ignore_parent_replication_key = False + primary_keys = ["id"] + parent_stream_type = RepositoryStream + + def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + return {"project_id": record["id"]} + schema = th.PropertiesList( th.Property("owner_url", th.StringType), th.Property("url", th.StringType), @@ -44,8 +55,34 @@ class ProjectsStream(GitHubRestStream): ).to_dict() +class ProjectColumnsStream(GitHubRestStream): + name = "project_columns" + path = "/projects/{project_id}/columns" + ignore_parent_replication_key = False + primary_keys = ["id"] + parent_stream_type = ProjectsStream + + def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + return {"column_id": record["id"]} + + schema = th.PropertiesList( + th.Property("url", th.StringType), + th.Property("project_url", th.StringType), + th.Property("cards_url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("name", th.StringType), + th.Property("created_at", th.StringType), + th.Property("updated_at", th.StringType), + ).to_dict() + + class ProjectCardsStream(GitHubRestStream): name = "project_cards" + path = "/projects/columns/{column_id}/cards" + ignore_parent_replication_key = False + primary_keys = ["id"] + parent_stream_type = ProjectColumnsStream schema = th.PropertiesList( th.Property("url", th.StringType), th.Property("id", th.IntegerType), @@ -83,22 +120,12 @@ class ProjectCardsStream(GitHubRestStream): ).to_dict() -class ProjectColumnsStream(GitHubRestStream): - name = "project_columns" - schema = th.PropertiesList( - th.Property("url", th.StringType), - th.Property("project_url", th.StringType), - th.Property("cards_url", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("name", th.StringType), - th.Property("created_at", th.StringType), - th.Property("updated_at", th.StringType), - ).to_dict() - - class PrCommitsStream(GitHubRestStream): name = "pr_commits" + path = "/repos/{org}/{repo}/pulls/{pull_number}/commits" + ignore_parent_replication_key = False + primary_keys = ["node_id"] + parent_stream_type = PullRequestsStream schema = th.PropertiesList( th.Property("url", th.StringType), th.Property("sha", th.StringType), @@ -204,6 +231,10 @@ class PrCommitsStream(GitHubRestStream): class ReleasesStream(GitHubRestStream): name = "releases" + path = "/repos/{org}/{repo}/releases" + ignore_parent_replication_key = False + primary_keys = ["id"] + parent_stream_type = RepositoryStream schema = th.PropertiesList( th.Property("url", th.StringType), th.Property("html_url", th.StringType), @@ -289,8 +320,11 @@ class ReleasesStream(GitHubRestStream): ).to_dict() +# TODO not sure what the parent stream is here. class TeamsStream(GitHubRestStream): name = "teams" + primary_keys = ['id'] + path = "/orgs/{org}/teams" schema = th.PropertiesList( th.Property("id", th.IntegerType), th.Property("node_id", th.StringType), @@ -309,6 +343,8 @@ class TeamsStream(GitHubRestStream): class TeamMembersStream(GitHubRestStream): name = "team_members" + primary_keys = ['id'] + path = "/orgs/{org}/teams/{team_slug}/members" schema = th.PropertiesList( th.Property("login", th.StringType), th.Property("id", th.IntegerType), @@ -333,6 +369,7 @@ class TeamMembersStream(GitHubRestStream): class TeamRolesStream(GitHubRestStream): name = "team_roles" + path = "/orgs/{org}/teams/{team_slug}/memberships/{username}" schema = th.PropertiesList( th.Property("url", th.StringType), th.Property("role", th.StringType), From d0352c77020247ade78e4f8ca3b8e2bb67d74b71 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Sun, 13 Mar 2022 19:24:48 +1100 Subject: [PATCH 16/48] Move all streams to main file --- tap_github/repository_streams.py | 322 +++++++++++++++++++++++++++++++ tap_github/tap.py | 14 ++ todo_streams.py | 321 +----------------------------- 3 files changed, 338 insertions(+), 319 deletions(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index b5dde675..bcb9443a 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -570,6 +570,99 @@ class MilestonesStream(GitHubRestStream): ).to_dict() +class ReleasesStream(GitHubRestStream): + name = "releases" + path = "/repos/{org}/{repo}/releases" + ignore_parent_replication_key = False + primary_keys = ["id"] + parent_stream_type = RepositoryStream + state_partitioning_keys = ["repo", "org"] + + schema = th.PropertiesList( + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("assets_url", th.StringType), + th.Property("upload_url", th.StringType), + th.Property("tarball_url", th.StringType), + th.Property("zipball_url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("tag_name", th.StringType), + th.Property("target_commitish", th.StringType), + th.Property("name", th.StringType), + th.Property("body", th.StringType), + th.Property("draft", th.BooleanType), + th.Property("prerelease", th.BooleanType), + th.Property("created_at", th.StringType), + th.Property("published_at", th.StringType), + th.Property( + "author", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), + th.Property( + "assets", + th.ArrayType( + th.ObjectType( + th.Property("url", th.StringType), + th.Property("browser_download_url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("name", th.StringType), + th.Property("label", th.StringType), + th.Property("state", th.StringType), + th.Property("content_type", th.StringType), + th.Property("size", th.IntegerType), + th.Property("download_count", th.IntegerType), + th.Property("created_at", th.StringType), + th.Property("updated_at", th.StringType), + th.Property( + "uploader", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), + ) + ), + ), + ).to_dict() + + class LanguagesStream(GitHubRestStream): name = "languages" path = "/repos/{org}/{repo}/languages" @@ -1408,6 +1501,117 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: ).to_dict() +class PullRequestCommits(GitHubRestStream): + name = "pull_request_commits" + path = "/repos/{org}/{repo}/pulls/{pull_number}/commits" + ignore_parent_replication_key = False + primary_keys = ["node_id"] + parent_stream_type = PullRequestsStream + state_partitioning_keys = ["pull_number"] + + schema = th.PropertiesList( + th.Property("url", th.StringType), + th.Property("sha", th.StringType), + th.Property("node_id", th.StringType), + th.Property("html_url", th.StringType), + th.Property("comments_url", th.StringType), + th.Property( + "commit", + th.ObjectType( + th.Property("url", th.StringType), + th.Property( + "author", + th.ObjectType( + th.Property("name", th.StringType), + th.Property("email", th.StringType), + th.Property("date", th.StringType), + ), + ), + th.Property( + "committer", + th.ObjectType( + th.Property("name", th.StringType), + th.Property("email", th.StringType), + th.Property("date", th.StringType), + ), + ), + th.Property("message", th.StringType), + th.Property( + "tree", + th.ObjectType( + th.Property("url", th.StringType), + th.Property("sha", th.StringType), + ), + ), + th.Property("comment_count", th.IntegerType), + th.Property( + "verification", + th.ObjectType( + th.Property("verified", th.BooleanType), + th.Property("reason", th.StringType), + th.Property("signature", th.StringType), + th.Property("payload", th.StringType), + ), + ), + ), + ), + th.Property( + "author", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), + th.Property( + "committer", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), + th.Property( + "parents", + th.ArrayType( + th.ObjectType( + th.Property("url", th.StringType), th.Property("sha", th.StringType) + ) + ), + ), + ).to_dict() + + class ReviewsStream(GitHubRestStream): name = "reviews" path = "/repos/{org}/{repo}/pulls/{pull_number}/reviews" @@ -1714,3 +1918,121 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]: th.Property("type", th.StringType), th.Property("site_admin", th.BooleanType), ).to_dict() + + +class ProjectsStream(GitHubRestStream): + name = "projects" + path = "/repos/{org}/{repo}/projects" + ignore_parent_replication_key = False + primary_keys = ["id"] + parent_stream_type = RepositoryStream + state_partitioning_keys = ["repo", "org"] + + def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + return {"project_id": record["id"]} + + schema = th.PropertiesList( + th.Property("owner_url", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("columns_url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("name", th.StringType), + th.Property("body", th.StringType), + th.Property("number", th.IntegerType), + th.Property("state", th.StringType), + th.Property( + "creator", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), + th.Property("created_at", th.StringType), + th.Property("updated_at", th.StringType), + ).to_dict() + + +class ProjectColumnsStream(GitHubRestStream): + name = "project_columns" + path = "/projects/{project_id}/columns" + ignore_parent_replication_key = False + primary_keys = ["id"] + parent_stream_type = ProjectsStream + state_partitioning_keys = ["project_id"] + + def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + return {"column_id": record["id"]} + + schema = th.PropertiesList( + th.Property("url", th.StringType), + th.Property("project_url", th.StringType), + th.Property("cards_url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("name", th.StringType), + th.Property("created_at", th.StringType), + th.Property("updated_at", th.StringType), + ).to_dict() + + +class ProjectCardsStream(GitHubRestStream): + name = "project_cards" + path = "/projects/columns/{column_id}/cards" + ignore_parent_replication_key = False + primary_keys = ["id"] + parent_stream_type = ProjectColumnsStream + state_partitioning_keys = ["column_id"] + + schema = th.PropertiesList( + th.Property("url", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("note", th.StringType), + th.Property( + "creator", + th.ObjectType( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ), + ), + th.Property("created_at", th.StringType), + th.Property("updated_at", th.StringType), + th.Property("archived", th.BooleanType), + th.Property("column_url", th.StringType), + th.Property("content_url", th.StringType), + th.Property("project_url", th.StringType), + ).to_dict() diff --git a/tap_github/tap.py b/tap_github/tap.py index 3ad1cbba..87eb45cc 100644 --- a/tap_github/tap.py +++ b/tap_github/tap.py @@ -25,6 +25,13 @@ CollaboratorsStream, ReviewsStream, ReviewCommentsStream, + ProjectsStream, + ProjectColumnsStream, + ProjectCardsStream, + PullRequestCommits, + MilestonesStream, + CommitCommentsStream, + ReleasesStream, ) from tap_github.user_streams import ( StarredStream, @@ -105,9 +112,12 @@ def discover_streams(self) -> List[Stream]: return [ AnonymousContributorsStream(tap=self), CommitsStream(tap=self), + CommitCommentsStream(tap=self), CommunityProfileStream(tap=self), ContributorsStream(tap=self), EventsStream(tap=self), + MilestonesStream(tap=self), + ReleasesStream(tap=self), CollaboratorsStream(tap=self), AssigneesStream(tap=self), IssuesStream(tap=self), @@ -115,6 +125,7 @@ def discover_streams(self) -> List[Stream]: IssueEventsStream(tap=self), LanguagesStream(tap=self), PullRequestsStream(tap=self), + PullRequestCommits(tap=self), ReviewsStream(tap=self), ReviewCommentsStream(tap=self), ReadmeHtmlStream(tap=self), @@ -122,6 +133,9 @@ def discover_streams(self) -> List[Stream]: RepositoryStream(tap=self), StargazersStream(tap=self), StatsContributorsStream(tap=self), + ProjectsStream(tap=self), + ProjectColumnsStream(tap=self), + ProjectCardsStream(tap=self), ] diff --git a/todo_streams.py b/todo_streams.py index a161b38c..cd71bf35 100644 --- a/todo_streams.py +++ b/todo_streams.py @@ -1,329 +1,12 @@ -from typing import Optional - from singer_sdk import typing as th from tap_github.client import GitHubRestStream -from tap_github.repository_streams import PullRequestsStream, RepositoryStream - - -class ProjectsStream(GitHubRestStream): - name = "projects" - path = "/repos/{org}/{repo}/projects" - ignore_parent_replication_key = False - primary_keys = ["id"] - parent_stream_type = RepositoryStream - - def get_child_context(self, record: dict, context: Optional[dict]) -> dict: - return {"project_id": record["id"]} - - schema = th.PropertiesList( - th.Property("owner_url", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("columns_url", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("name", th.StringType), - th.Property("body", th.StringType), - th.Property("number", th.IntegerType), - th.Property("state", th.StringType), - th.Property( - "creator", - th.ObjectType( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType), - ), - ), - th.Property("created_at", th.StringType), - th.Property("updated_at", th.StringType), - ).to_dict() - - -class ProjectColumnsStream(GitHubRestStream): - name = "project_columns" - path = "/projects/{project_id}/columns" - ignore_parent_replication_key = False - primary_keys = ["id"] - parent_stream_type = ProjectsStream - - def get_child_context(self, record: dict, context: Optional[dict]) -> dict: - return {"column_id": record["id"]} - - schema = th.PropertiesList( - th.Property("url", th.StringType), - th.Property("project_url", th.StringType), - th.Property("cards_url", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("name", th.StringType), - th.Property("created_at", th.StringType), - th.Property("updated_at", th.StringType), - ).to_dict() - - -class ProjectCardsStream(GitHubRestStream): - name = "project_cards" - path = "/projects/columns/{column_id}/cards" - ignore_parent_replication_key = False - primary_keys = ["id"] - parent_stream_type = ProjectColumnsStream - schema = th.PropertiesList( - th.Property("url", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("note", th.StringType), - th.Property( - "creator", - th.ObjectType( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType), - ), - ), - th.Property("created_at", th.StringType), - th.Property("updated_at", th.StringType), - th.Property("archived", th.BooleanType), - th.Property("column_url", th.StringType), - th.Property("content_url", th.StringType), - th.Property("project_url", th.StringType), - ).to_dict() - - -class PrCommitsStream(GitHubRestStream): - name = "pr_commits" - path = "/repos/{org}/{repo}/pulls/{pull_number}/commits" - ignore_parent_replication_key = False - primary_keys = ["node_id"] - parent_stream_type = PullRequestsStream - schema = th.PropertiesList( - th.Property("url", th.StringType), - th.Property("sha", th.StringType), - th.Property("node_id", th.StringType), - th.Property("html_url", th.StringType), - th.Property("comments_url", th.StringType), - th.Property( - "commit", - th.ObjectType( - th.Property("url", th.StringType), - th.Property( - "author", - th.ObjectType( - th.Property("name", th.StringType), - th.Property("email", th.StringType), - th.Property("date", th.StringType), - ), - ), - th.Property( - "committer", - th.ObjectType( - th.Property("name", th.StringType), - th.Property("email", th.StringType), - th.Property("date", th.StringType), - ), - ), - th.Property("message", th.StringType), - th.Property( - "tree", - th.ObjectType( - th.Property("url", th.StringType), - th.Property("sha", th.StringType), - ), - ), - th.Property("comment_count", th.IntegerType), - th.Property( - "verification", - th.ObjectType( - th.Property("verified", th.BooleanType), - th.Property("reason", th.StringType), - th.Property("signature", th.StringType), - th.Property("payload", th.StringType), - ), - ), - ), - ), - th.Property( - "author", - th.ObjectType( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType), - ), - ), - th.Property( - "committer", - th.ObjectType( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType), - ), - ), - th.Property( - "parents", - th.ArrayType( - th.ObjectType( - th.Property("url", th.StringType), th.Property("sha", th.StringType) - ) - ), - ), - ).to_dict() - - -class ReleasesStream(GitHubRestStream): - name = "releases" - path = "/repos/{org}/{repo}/releases" - ignore_parent_replication_key = False - primary_keys = ["id"] - parent_stream_type = RepositoryStream - schema = th.PropertiesList( - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("assets_url", th.StringType), - th.Property("upload_url", th.StringType), - th.Property("tarball_url", th.StringType), - th.Property("zipball_url", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("tag_name", th.StringType), - th.Property("target_commitish", th.StringType), - th.Property("name", th.StringType), - th.Property("body", th.StringType), - th.Property("draft", th.BooleanType), - th.Property("prerelease", th.BooleanType), - th.Property("created_at", th.StringType), - th.Property("published_at", th.StringType), - th.Property( - "author", - th.ObjectType( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType), - ), - ), - th.Property( - "assets", - th.ArrayType( - th.ObjectType( - th.Property("url", th.StringType), - th.Property("browser_download_url", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("name", th.StringType), - th.Property("label", th.StringType), - th.Property("state", th.StringType), - th.Property("content_type", th.StringType), - th.Property("size", th.IntegerType), - th.Property("download_count", th.IntegerType), - th.Property("created_at", th.StringType), - th.Property("updated_at", th.StringType), - th.Property( - "uploader", - th.ObjectType( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType), - ), - ), - ) - ), - ), - ).to_dict() # TODO not sure what the parent stream is here. class TeamsStream(GitHubRestStream): name = "teams" - primary_keys = ['id'] + primary_keys = ["id"] path = "/orgs/{org}/teams" schema = th.PropertiesList( th.Property("id", th.IntegerType), @@ -343,7 +26,7 @@ class TeamsStream(GitHubRestStream): class TeamMembersStream(GitHubRestStream): name = "team_members" - primary_keys = ['id'] + primary_keys = ["id"] path = "/orgs/{org}/teams/{team_slug}/members" schema = th.PropertiesList( th.Property("login", th.StringType), From fc70b2a5120b73964af4a471cbe58cb7f07671a7 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Sun, 13 Mar 2022 19:41:41 +1100 Subject: [PATCH 17/48] Add replication keys --- tap_github/repository_streams.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index bcb9443a..2fd29feb 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -577,6 +577,7 @@ class ReleasesStream(GitHubRestStream): primary_keys = ["id"] parent_stream_type = RepositoryStream state_partitioning_keys = ["repo", "org"] + replication_key = "published_at" schema = th.PropertiesList( th.Property("url", th.StringType), @@ -1174,7 +1175,7 @@ def post_process(self, row: dict, context: Optional[dict] = None) -> dict: class CommitCommentsStream(GitHubRestStream): - name = "commit_comment" + name = "commit_comments" path = "/repos/{org}/{repo}/comments" primary_keys = ["id"] replication_key = "updated_at" @@ -1923,7 +1924,9 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]: class ProjectsStream(GitHubRestStream): name = "projects" path = "/repos/{org}/{repo}/projects" + # TODO I think this can be true, but I'm not sure. ignore_parent_replication_key = False + replication_key = "updated_at" primary_keys = ["id"] parent_stream_type = RepositoryStream state_partitioning_keys = ["repo", "org"] @@ -1974,6 +1977,7 @@ class ProjectColumnsStream(GitHubRestStream): name = "project_columns" path = "/projects/{project_id}/columns" ignore_parent_replication_key = False + replication_key = "updated_at" primary_keys = ["id"] parent_stream_type = ProjectsStream state_partitioning_keys = ["project_id"] @@ -1997,6 +2001,7 @@ class ProjectCardsStream(GitHubRestStream): name = "project_cards" path = "/projects/columns/{column_id}/cards" ignore_parent_replication_key = False + replication_key = "updated_at" primary_keys = ["id"] parent_stream_type = ProjectColumnsStream state_partitioning_keys = ["column_id"] From 7730b865e086542f945eb7271c2b2dacd287f4c8 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Mon, 14 Mar 2022 12:46:12 +1100 Subject: [PATCH 18/48] fix tests (change type to datetime) --- tap_github/repository_streams.py | 36 ++++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 2fd29feb..3d95a1e0 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -563,9 +563,9 @@ class MilestonesStream(GitHubRestStream): ), th.Property("open_issues", th.IntegerType), th.Property("closed_issues", th.IntegerType), - th.Property("created_at", th.StringType), - th.Property("updated_at", th.StringType), - th.Property("closed_at", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), + th.Property("closed_at", th.DateTimeType), th.Property("due_on", th.StringType), ).to_dict() @@ -594,8 +594,8 @@ class ReleasesStream(GitHubRestStream): th.Property("body", th.StringType), th.Property("draft", th.BooleanType), th.Property("prerelease", th.BooleanType), - th.Property("created_at", th.StringType), - th.Property("published_at", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("published_at", th.DateTimeType), th.Property( "author", th.ObjectType( @@ -633,8 +633,8 @@ class ReleasesStream(GitHubRestStream): th.Property("content_type", th.StringType), th.Property("size", th.IntegerType), th.Property("download_count", th.IntegerType), - th.Property("created_at", th.StringType), - th.Property("updated_at", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), th.Property( "uploader", th.ObjectType( @@ -1216,8 +1216,8 @@ class CommitCommentsStream(GitHubRestStream): th.Property("site_admin", th.BooleanType), ), ), - th.Property("created_at", th.StringType), - th.Property("updated_at", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), th.Property("author_association", th.StringType), ).to_dict() @@ -1660,7 +1660,7 @@ class ReviewsStream(GitHubRestStream): ), ), ), - th.Property("submitted_at", th.StringType), + th.Property("submitted_at", th.DateTimeType), th.Property("commit_id", th.StringType), th.Property("author_association", th.StringType), ).to_dict() @@ -1710,8 +1710,8 @@ class ReviewCommentsStream(GitHubRestStream): ), ), th.Property("body", th.StringType), - th.Property("created_at", th.StringType), - th.Property("updated_at", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), th.Property("html_url", th.StringType), th.Property("pull_request_url", th.StringType), th.Property("author_association", th.StringType), @@ -1968,8 +1968,8 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: th.Property("site_admin", th.BooleanType), ), ), - th.Property("created_at", th.StringType), - th.Property("updated_at", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), ).to_dict() @@ -1992,8 +1992,8 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: th.Property("id", th.IntegerType), th.Property("node_id", th.StringType), th.Property("name", th.StringType), - th.Property("created_at", th.StringType), - th.Property("updated_at", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), ).to_dict() @@ -2034,8 +2034,8 @@ class ProjectCardsStream(GitHubRestStream): th.Property("site_admin", th.BooleanType), ), ), - th.Property("created_at", th.StringType), - th.Property("updated_at", th.StringType), + th.Property("created_at", th.DateTimeType), + th.Property("updated_at", th.DateTimeType), th.Property("archived", th.BooleanType), th.Property("column_url", th.StringType), th.Property("content_url", th.StringType), From f68c26037cdca5f72309de43188f71cd40470a3b Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Fri, 18 Mar 2022 01:04:59 +1100 Subject: [PATCH 19/48] introduce streams enum --- tap_github/organization_streams.py | 37 ++++++++++++ tap_github/streams.py | 93 ++++++++++++++++++++++++++++++ tap_github/tap.py | 82 +++----------------------- 3 files changed, 138 insertions(+), 74 deletions(-) create mode 100644 tap_github/organization_streams.py create mode 100644 tap_github/streams.py diff --git a/tap_github/organization_streams.py b/tap_github/organization_streams.py new file mode 100644 index 00000000..78b6a873 --- /dev/null +++ b/tap_github/organization_streams.py @@ -0,0 +1,37 @@ +"""User Stream types classes for tap-github.""" + +from typing import Dict, List, Optional + +from singer_sdk import typing as th # JSON Schema typing helpers + +from tap_github.client import GitHubRestStream + + +class OrganizationStream(GitHubRestStream): + name = "organizations" + + @property + def path(self) -> str: # type: ignore + """Return the API endpoint path.""" + return "/orgs/{org}" + + @property + def partitions(self) -> Optional[List[Dict]]: + """Return a list of partitions.""" + return [{"org": org} for org in self.config["organizations"]] + + def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + """Return a child context object from the record and optional provided context. + + By default, will return context if provided and otherwise the record dict. + Developers may override this behavior to send specific information to child + streams for context. + """ + return { + "org": record["login"], + } + + schema = th.PropertiesList( + th.Property("login", th.StringType), + # TODO + ).to_dict() diff --git a/tap_github/streams.py b/tap_github/streams.py new file mode 100644 index 00000000..2047930d --- /dev/null +++ b/tap_github/streams.py @@ -0,0 +1,93 @@ +from enum import Enum +from typing import Mapping, Any, Type + +from singer_sdk.streams.core import Stream + +from tap_github.organization_streams import OrganizationStream +from tap_github.repository_streams import ( + AnonymousContributorsStream, + CommitsStream, + CommunityProfileStream, + ContributorsStream, + EventsStream, + IssueCommentsStream, + IssueEventsStream, + IssuesStream, + LanguagesStream, + PullRequestsStream, + ReadmeHtmlStream, + ReadmeStream, + RepositoryStream, + StargazersStream, + StatsContributorsStream, + AssigneesStream, + CollaboratorsStream, + ReviewsStream, + ReviewCommentsStream, + ProjectsStream, + ProjectColumnsStream, + ProjectCardsStream, + PullRequestCommits, + MilestonesStream, + CommitCommentsStream, + ReleasesStream, +) +from tap_github.user_streams import ( + StarredStream, + UserContributedToStream, + UserStream, +) + + +class Streams(Enum): + valid_queries: set[str] + streams: list[Type[Stream]] + + def __init__(self, valid_queries: set[str], streams: list[Type[Stream]]): + self.valid_queries = valid_queries + self.streams = streams + + REPOSITORY = ( + {"repositories", "organizations", "searches"}, + [ + AnonymousContributorsStream, + CommitsStream, + CommitCommentsStream, + CommunityProfileStream, + ContributorsStream, + EventsStream, + MilestonesStream, + ReleasesStream, + CollaboratorsStream, + AssigneesStream, + IssuesStream, + IssueCommentsStream, + IssueEventsStream, + LanguagesStream, + PullRequestsStream, + PullRequestCommits, + ReviewsStream, + ReviewCommentsStream, + ReadmeHtmlStream, + ReadmeStream, + RepositoryStream, + StargazersStream, + StatsContributorsStream, + ProjectsStream, + ProjectColumnsStream, + ProjectCardsStream, + ], + ) + USERS = ( + {"user_usernames", "user_ids"}, + [ + StarredStream, + UserContributedToStream, + UserStream, + ], + ) + ORGANIZATIONS = ({"organizations"}, [OrganizationStream]) + + @classmethod + def all_valid_queries(cls): + return set.union(*[stream.value for stream in Streams]) diff --git a/tap_github/tap.py b/tap_github/tap.py index 87eb45cc..2499aa39 100644 --- a/tap_github/tap.py +++ b/tap_github/tap.py @@ -5,39 +5,7 @@ from singer_sdk import Stream, Tap from singer_sdk import typing as th # JSON schema typing helpers -from tap_github.repository_streams import ( - AnonymousContributorsStream, - CommitsStream, - CommunityProfileStream, - ContributorsStream, - EventsStream, - IssueCommentsStream, - IssueEventsStream, - IssuesStream, - LanguagesStream, - PullRequestsStream, - ReadmeHtmlStream, - ReadmeStream, - RepositoryStream, - StargazersStream, - StatsContributorsStream, - AssigneesStream, - CollaboratorsStream, - ReviewsStream, - ReviewCommentsStream, - ProjectsStream, - ProjectColumnsStream, - ProjectCardsStream, - PullRequestCommits, - MilestonesStream, - CommitCommentsStream, - ReleasesStream, -) -from tap_github.user_streams import ( - StarredStream, - UserContributedToStream, - UserStream, -) +from tap_github.streams import Streams class TapGitHub(Tap): @@ -92,51 +60,17 @@ class TapGitHub(Tap): def discover_streams(self) -> List[Stream]: """Return a list of discovered streams for each query.""" - VALID_USER_QUERIES = {"user_usernames", "user_ids"} - VALID_REPO_QUERIES = {"repositories", "organizations", "searches"} - VALID_QUERIES = VALID_REPO_QUERIES.union(VALID_USER_QUERIES) - if len(VALID_QUERIES.intersection(self.config)) != 1: + if len(Streams.all_valid_queries().intersection(self.config)) != 1: raise ValueError( "This tap requires one and only one of the following path options: " - f"{VALID_QUERIES}." + f"{Streams.all_valid_queries()}." ) - is_user_query = len(VALID_USER_QUERIES.intersection(self.config)) > 0 - if is_user_query: - return [ - StarredStream(tap=self), - UserContributedToStream(tap=self), - UserStream(tap=self), - ] - else: - return [ - AnonymousContributorsStream(tap=self), - CommitsStream(tap=self), - CommitCommentsStream(tap=self), - CommunityProfileStream(tap=self), - ContributorsStream(tap=self), - EventsStream(tap=self), - MilestonesStream(tap=self), - ReleasesStream(tap=self), - CollaboratorsStream(tap=self), - AssigneesStream(tap=self), - IssuesStream(tap=self), - IssueCommentsStream(tap=self), - IssueEventsStream(tap=self), - LanguagesStream(tap=self), - PullRequestsStream(tap=self), - PullRequestCommits(tap=self), - ReviewsStream(tap=self), - ReviewCommentsStream(tap=self), - ReadmeHtmlStream(tap=self), - ReadmeStream(tap=self), - RepositoryStream(tap=self), - StargazersStream(tap=self), - StatsContributorsStream(tap=self), - ProjectsStream(tap=self), - ProjectColumnsStream(tap=self), - ProjectCardsStream(tap=self), - ] + + for stream_type in Streams: + if len(stream_type.valid_queries.intersection(self.config)) > 0: + return [StreamClass(tap=self) for StreamClass in stream_type.streams] + raise Exception("No valid queries found") # CLI Execution: From 3132b36924a1c905c5522046e98d0d43e5279a45 Mon Sep 17 00:00:00 2001 From: Ryan Date: Fri, 18 Mar 2022 01:11:01 +1100 Subject: [PATCH 20/48] Fix up organization stream --- tap_github/organization_streams.py | 12 +++++++++++- tap_github/streams.py | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tap_github/organization_streams.py b/tap_github/organization_streams.py index 78b6a873..ffa9f711 100644 --- a/tap_github/organization_streams.py +++ b/tap_github/organization_streams.py @@ -33,5 +33,15 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: schema = th.PropertiesList( th.Property("login", th.StringType), - # TODO + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("hooks_url", th.StringType), + th.Property("issues_url", th.StringType), + th.Property("members_url", th.StringType), + th.Property("public_members_url", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("description", th.StringType), ).to_dict() diff --git a/tap_github/streams.py b/tap_github/streams.py index 2047930d..d4d74fcf 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -90,4 +90,4 @@ def __init__(self, valid_queries: set[str], streams: list[Type[Stream]]): @classmethod def all_valid_queries(cls): - return set.union(*[stream.value for stream in Streams]) + return set.union(*[stream.valid_queries for stream in Streams]) From cf7c46786d3d13274eb06ed81154093882214c93 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Fri, 18 Mar 2022 13:03:29 +1100 Subject: [PATCH 21/48] Reverse order of testing versions --- .github/workflows/test_tap.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_tap.yml b/.github/workflows/test_tap.yml index 94ce221d..fe35431b 100644 --- a/.github/workflows/test_tap.yml +++ b/.github/workflows/test_tap.yml @@ -15,7 +15,7 @@ jobs: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} strategy: matrix: - python-version: [3.7, 3.8, 3.9, "3.10"] + python-version: ["3.10", 3.9, 3.8, 3.7] # run the matrix jobs one after the other so they can benefit from caching max-parallel: 1 From 9258fd3f2e3d50b8d23338c586cd408d6b411b41 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Fri, 18 Mar 2022 15:01:05 +1100 Subject: [PATCH 22/48] remove unsupported types from class --- tap_github/streams.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tap_github/streams.py b/tap_github/streams.py index d4d74fcf..1d339fb4 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -40,8 +40,6 @@ class Streams(Enum): - valid_queries: set[str] - streams: list[Type[Stream]] def __init__(self, valid_queries: set[str], streams: list[Type[Stream]]): self.valid_queries = valid_queries From 38e41a0b60325d1849f163938a86ef6b34156346 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Fri, 18 Mar 2022 15:02:53 +1100 Subject: [PATCH 23/48] Fix format --- tap_github/streams.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tap_github/streams.py b/tap_github/streams.py index 1d339fb4..0144233a 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -40,7 +40,6 @@ class Streams(Enum): - def __init__(self, valid_queries: set[str], streams: list[Type[Stream]]): self.valid_queries = valid_queries self.streams = streams From c3d621e5a7f65ea3f3ba9897fb0b0699eb84451f Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Fri, 18 Mar 2022 15:05:36 +1100 Subject: [PATCH 24/48] Try use capital types to pass ci --- tap_github/streams.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tap_github/streams.py b/tap_github/streams.py index 0144233a..b1604044 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Mapping, Any, Type +from typing import Type, Set, List from singer_sdk.streams.core import Stream @@ -40,7 +40,10 @@ class Streams(Enum): - def __init__(self, valid_queries: set[str], streams: list[Type[Stream]]): + valid_queries: Set[str] + streams: List[Type[Stream]] + + def __init__(self, valid_queries: Set[str], streams: List[Type[Stream]]): self.valid_queries = valid_queries self.streams = streams From 549e3991cabdbcfb2a4559d8d0cadf9099b04bae Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Fri, 18 Mar 2022 18:40:50 +1100 Subject: [PATCH 25/48] Fix tap not including org streams on organization given --- tap_github/tap.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tap_github/tap.py b/tap_github/tap.py index 2499aa39..bc27f489 100644 --- a/tap_github/tap.py +++ b/tap_github/tap.py @@ -66,11 +66,16 @@ def discover_streams(self) -> List[Stream]: "This tap requires one and only one of the following path options: " f"{Streams.all_valid_queries()}." ) - + streams = [] for stream_type in Streams: if len(stream_type.valid_queries.intersection(self.config)) > 0: - return [StreamClass(tap=self) for StreamClass in stream_type.streams] - raise Exception("No valid queries found") + streams += [ + StreamClass(tap=self) for StreamClass in stream_type.streams + ] + # if streams is empty, raise exception + if not streams: + raise Exception("No valid streams found.") + return streams # CLI Execution: From 47dc14d8edf311381931970633d8418012f06af1 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Fri, 18 Mar 2022 19:01:12 +1100 Subject: [PATCH 26/48] Add test for org stream --- tap_github/organization_streams.py | 24 ++++++++++++++++- tap_github/tests/fixtures.py | 41 +++++++++++++++++++++--------- tap_github/tests/test_core.py | 18 ++++++++++--- tap_github/tests/test_tap.py | 6 ----- 4 files changed, 67 insertions(+), 22 deletions(-) diff --git a/tap_github/organization_streams.py b/tap_github/organization_streams.py index ffa9f711..bca4f42b 100644 --- a/tap_github/organization_streams.py +++ b/tap_github/organization_streams.py @@ -1,6 +1,6 @@ """User Stream types classes for tap-github.""" -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Iterable, Any from singer_sdk import typing as th # JSON Schema typing helpers @@ -31,6 +31,28 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: "org": record["login"], } + def get_records(self, context: Optional[dict]) -> Iterable[Dict[str, Any]]: + """ + Override the parent method to allow skipping API calls + if the stream is deselected and skip_parent_streams is True in config. + This allows running the tap with fewer API calls and preserving + quota when only syncing a child stream. Without this, + the API call is sent but data is discarded. + """ + if ( + not self.selected + and "skip_parent_streams" in self.config + and self.config["skip_parent_streams"] + and context is not None + ): + # build a minimal mock record so that self._sync_records + # can proceed with child streams + yield { + "org": context["org"], + } + else: + yield from super().get_records(context) + schema = th.PropertiesList( th.Property("login", th.StringType), th.Property("id", th.IntegerType), diff --git a/tap_github/tests/fixtures.py b/tap_github/tests/fixtures.py index 4aecd624..08a7fab4 100644 --- a/tap_github/tests/fixtures.py +++ b/tap_github/tests/fixtures.py @@ -37,38 +37,55 @@ def repo_list_config(request): @pytest.fixture -def usernames_list_config(request): +def username_list_config(request): """ Get a default list of usernames or pass your own by decorating your test with - @pytest.mark.usernames_list(['ericboucher', 'aaronsteers']) + @pytest.mark.username_list(['ericboucher', 'aaronsteers']) """ - marker = request.node.get_closest_marker("usernames_list") + marker = request.node.get_closest_marker("username_list") if marker is None: - usernames_list = ["ericboucher", "aaronsteers"] + username_list = ["ericboucher", "aaronsteers"] else: - usernames_list = marker.args[0] + username_list = marker.args[0] return { "metrics_log_level": "none", "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"), - "user_usernames": usernames_list, + "user_usernames": username_list, } @pytest.fixture -def user_ids_list_config(request): +def user_id_list_config(request): """ Get a default list of usernames or pass your own by decorating your test with - @pytest.mark.user_ids_list(['ericboucher', 'aaronsteers']) + @pytest.mark.user_id_list(['ericboucher', 'aaronsteers']) """ - marker = request.node.get_closest_marker("user_ids_list") + marker = request.node.get_closest_marker("user_id_list") if marker is None: - user_ids_list = [1, 2] + user_id_list = [1, 2] else: - user_ids_list = marker.args[0] + user_id_list = marker.args[0] return { "metrics_log_level": "none", "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"), - "user_ids": user_ids_list, + "user_ids": user_id_list, + } + + +@pytest.fixture +def organization_list_config(request): + """ + Get a default list of usernames or pass your own by decorating your test with + @pytest.mark.user_id_list(['ericboucher', 'aaronsteers']) + """ + marker = request.node.get_closest_marker("organization_list") + + organization_list = ["MeltanoLabs"] if marker is None else marker.args[0] + + return { + "metrics_log_level": "none", + "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"), + "organizations": organization_list, } diff --git a/tap_github/tests/test_core.py b/tap_github/tests/test_core.py index c86c40e8..e34cc087 100644 --- a/tap_github/tests/test_core.py +++ b/tap_github/tests/test_core.py @@ -4,7 +4,12 @@ from tap_github.tap import TapGitHub -from .fixtures import repo_list_config, search_config, usernames_list_config +from .fixtures import ( + repo_list_config, + search_config, + username_list_config, + organization_list_config, +) # Run standard built-in tap tests from the SDK: @@ -22,9 +27,16 @@ def test_standard_tap_tests_for_repo_list_mode(repo_list_config): test() -def test_standard_tap_tests_for_usernames_list_mode(usernames_list_config): +def test_standard_tap_tests_for_username_list_mode(username_list_config): """Run standard tap tests from the SDK.""" - tests = get_standard_tap_tests(TapGitHub, config=usernames_list_config) + tests = get_standard_tap_tests(TapGitHub, config=username_list_config) + for test in tests: + test() + + +def test_standard_tap_tests_for_organization_list_mode(organization_list_config): + """Run standard tap tests from the SDK.""" + tests = get_standard_tap_tests(TapGitHub, config=organization_list_config) for test in tests: test() diff --git a/tap_github/tests/test_tap.py b/tap_github/tests/test_tap.py index 505a1c0d..c3278ee8 100644 --- a/tap_github/tests/test_tap.py +++ b/tap_github/tests/test_tap.py @@ -1,10 +1,4 @@ -import datetime - import pytest -from singer_sdk.helpers._catalog import ( - deselect_all_streams, - set_catalog_stream_selected, -) from tap_github.tap import TapGitHub From 46257e13674f1f15c87f1e0c0a62c3fd069aee91 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Fri, 18 Mar 2022 21:28:49 +1100 Subject: [PATCH 27/48] Add rest of org streams --- .github/workflows/test_tap.yml | 4 +- tap_github/organization_streams.py | 75 ++++++++++++++++++++++++++++++ tap_github/streams.py | 11 ++++- todo_streams.py | 60 ------------------------ 4 files changed, 88 insertions(+), 62 deletions(-) delete mode 100644 todo_streams.py diff --git a/.github/workflows/test_tap.yml b/.github/workflows/test_tap.yml index fe35431b..b587fb97 100644 --- a/.github/workflows/test_tap.yml +++ b/.github/workflows/test_tap.yml @@ -28,7 +28,9 @@ jobs: path: '**/api_calls_tests_cache.sqlite' # github cache expires after 1wk, and we expire the content after 24h # this key should not need to change unless we need to clear the cache - key: api-cache-v2 + key: api-cache-v3 + restore-keys: | + api-cache-v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: diff --git a/tap_github/organization_streams.py b/tap_github/organization_streams.py index bca4f42b..a3ead317 100644 --- a/tap_github/organization_streams.py +++ b/tap_github/organization_streams.py @@ -67,3 +67,78 @@ def get_records(self, context: Optional[dict]) -> Iterable[Dict[str, Any]]: th.Property("avatar_url", th.StringType), th.Property("description", th.StringType), ).to_dict() + + +class TeamsStream(GitHubRestStream): + name = "teams" + primary_keys = ["id"] + path = "/orgs/{org}/teams" + ignore_parent_replication_key = True + parent_stream_type = OrganizationStream + state_partitioning_keys = ["org"] + + def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + return {"team_slug": record["slug"]} + + schema = th.PropertiesList( + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("name", th.StringType), + th.Property("slug", th.StringType), + th.Property("description", th.StringType), + th.Property("privacy", th.StringType), + th.Property("permission", th.StringType), + th.Property("members_url", th.StringType), + th.Property("repositories_url", th.StringType), + th.Property("parent", th.StringType), + ).to_dict() + + +class TeamMembersStream(GitHubRestStream): + name = "team_members" + primary_keys = ["id"] + path = "/orgs/{org}/teams/{team_slug}/members" + ignore_parent_replication_key = True + parent_stream_type = TeamsStream + state_partitioning_keys = ["org", "team_slug"] + + def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + return {"username": record["login"]} + + schema = th.PropertiesList( + th.Property("login", th.StringType), + th.Property("id", th.IntegerType), + th.Property("node_id", th.StringType), + th.Property("avatar_url", th.StringType), + th.Property("gravatar_id", th.StringType), + th.Property("url", th.StringType), + th.Property("html_url", th.StringType), + th.Property("followers_url", th.StringType), + th.Property("following_url", th.StringType), + th.Property("gists_url", th.StringType), + th.Property("starred_url", th.StringType), + th.Property("subscriptions_url", th.StringType), + th.Property("organizations_url", th.StringType), + th.Property("repos_url", th.StringType), + th.Property("events_url", th.StringType), + th.Property("received_events_url", th.StringType), + th.Property("type", th.StringType), + th.Property("site_admin", th.BooleanType), + ).to_dict() + + +class TeamRolesStream(GitHubRestStream): + name = "team_roles" + path = "/orgs/{org}/teams/{team_slug}/memberships/{username}" + ignore_parent_replication_key = True + primary_keys = ["url"] + parent_stream_type = TeamMembersStream + state_partitioning_keys = ["org", "team_slug", "username"] + + schema = th.PropertiesList( + th.Property("url", th.StringType), + th.Property("role", th.StringType), + th.Property("state", th.StringType), + ).to_dict() diff --git a/tap_github/streams.py b/tap_github/streams.py index b1604044..5e6526e6 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -37,6 +37,12 @@ UserContributedToStream, UserStream, ) +from tap_github.organization_streams import ( + OrganizationStream, + TeamsStream, + TeamMembersStream, + TeamRolesStream, +) class Streams(Enum): @@ -86,7 +92,10 @@ def __init__(self, valid_queries: Set[str], streams: List[Type[Stream]]): UserStream, ], ) - ORGANIZATIONS = ({"organizations"}, [OrganizationStream]) + ORGANIZATIONS = ( + {"organizations"}, + [OrganizationStream, TeamsStream, TeamMembersStream, TeamRolesStream], + ) @classmethod def all_valid_queries(cls): diff --git a/todo_streams.py b/todo_streams.py deleted file mode 100644 index cd71bf35..00000000 --- a/todo_streams.py +++ /dev/null @@ -1,60 +0,0 @@ -from singer_sdk import typing as th - -from tap_github.client import GitHubRestStream - - -# TODO not sure what the parent stream is here. -class TeamsStream(GitHubRestStream): - name = "teams" - primary_keys = ["id"] - path = "/orgs/{org}/teams" - schema = th.PropertiesList( - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("name", th.StringType), - th.Property("slug", th.StringType), - th.Property("description", th.StringType), - th.Property("privacy", th.StringType), - th.Property("permission", th.StringType), - th.Property("members_url", th.StringType), - th.Property("repositories_url", th.StringType), - th.Property("parent", th.StringType), - ).to_dict() - - -class TeamMembersStream(GitHubRestStream): - name = "team_members" - primary_keys = ["id"] - path = "/orgs/{org}/teams/{team_slug}/members" - schema = th.PropertiesList( - th.Property("login", th.StringType), - th.Property("id", th.IntegerType), - th.Property("node_id", th.StringType), - th.Property("avatar_url", th.StringType), - th.Property("gravatar_id", th.StringType), - th.Property("url", th.StringType), - th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), - th.Property("type", th.StringType), - th.Property("site_admin", th.BooleanType), - ).to_dict() - - -class TeamRolesStream(GitHubRestStream): - name = "team_roles" - path = "/orgs/{org}/teams/{team_slug}/memberships/{username}" - schema = th.PropertiesList( - th.Property("url", th.StringType), - th.Property("role", th.StringType), - th.Property("state", th.StringType), - ).to_dict() From 8f702cb91d71f033dbb91d6401325d7e247d64c8 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Fri, 18 Mar 2022 21:35:02 +1100 Subject: [PATCH 28/48] [ci skip] Temp changes for testing --- tap_github/tap.py | 2 +- tap_github/tests/fixtures.py | 2 +- tap_github/tests/test_tap.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tap_github/tap.py b/tap_github/tap.py index bc27f489..07105d07 100644 --- a/tap_github/tap.py +++ b/tap_github/tap.py @@ -74,7 +74,7 @@ def discover_streams(self) -> List[Stream]: ] # if streams is empty, raise exception if not streams: - raise Exception("No valid streams found.") + raise ValueError("No valid streams found.") return streams diff --git a/tap_github/tests/fixtures.py b/tap_github/tests/fixtures.py index 08a7fab4..609ad43a 100644 --- a/tap_github/tests/fixtures.py +++ b/tap_github/tests/fixtures.py @@ -82,7 +82,7 @@ def organization_list_config(request): """ marker = request.node.get_closest_marker("organization_list") - organization_list = ["MeltanoLabs"] if marker is None else marker.args[0] + organization_list = ["oviohub"] if marker is None else marker.args[0] return { "metrics_log_level": "none", diff --git a/tap_github/tests/test_tap.py b/tap_github/tests/test_tap.py index c3278ee8..5f9ad571 100644 --- a/tap_github/tests/test_tap.py +++ b/tap_github/tests/test_tap.py @@ -4,7 +4,7 @@ from .fixtures import repo_list_config -repo_list_2 = ["MeltanoLabs/tap-github"] +repo_list_2 = ["oviohub/ovio-xplore-app"] @pytest.mark.repo_list(repo_list_2) From 1f5aad36026992b2c25ba4d5dd5c42feededeece Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Sat, 19 Mar 2022 01:21:10 +1100 Subject: [PATCH 29/48] Fix parent context being missing --- tap_github/organization_streams.py | 8 ++++++-- tap_github/tests/fixtures.py | 2 +- tap_github/tests/test_tap.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tap_github/organization_streams.py b/tap_github/organization_streams.py index a3ead317..dfc652a4 100644 --- a/tap_github/organization_streams.py +++ b/tap_github/organization_streams.py @@ -78,7 +78,7 @@ class TeamsStream(GitHubRestStream): state_partitioning_keys = ["org"] def get_child_context(self, record: dict, context: Optional[dict]) -> dict: - return {"team_slug": record["slug"]} + return {"org": context["org"], "team_slug": record["slug"]} schema = th.PropertiesList( th.Property("id", th.IntegerType), @@ -105,7 +105,11 @@ class TeamMembersStream(GitHubRestStream): state_partitioning_keys = ["org", "team_slug"] def get_child_context(self, record: dict, context: Optional[dict]) -> dict: - return {"username": record["login"]} + return { + "org": context["org"], + "team_slug": context["team_slug"], + "username": record["login"], + } schema = th.PropertiesList( th.Property("login", th.StringType), diff --git a/tap_github/tests/fixtures.py b/tap_github/tests/fixtures.py index 609ad43a..08a7fab4 100644 --- a/tap_github/tests/fixtures.py +++ b/tap_github/tests/fixtures.py @@ -82,7 +82,7 @@ def organization_list_config(request): """ marker = request.node.get_closest_marker("organization_list") - organization_list = ["oviohub"] if marker is None else marker.args[0] + organization_list = ["MeltanoLabs"] if marker is None else marker.args[0] return { "metrics_log_level": "none", diff --git a/tap_github/tests/test_tap.py b/tap_github/tests/test_tap.py index 5f9ad571..c3278ee8 100644 --- a/tap_github/tests/test_tap.py +++ b/tap_github/tests/test_tap.py @@ -4,7 +4,7 @@ from .fixtures import repo_list_config -repo_list_2 = ["oviohub/ovio-xplore-app"] +repo_list_2 = ["MeltanoLabs/tap-github"] @pytest.mark.repo_list(repo_list_2) From 25c05c5aeb867e61d50176e4cbb52a5f6341ae94 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Sat, 19 Mar 2022 01:48:00 +1100 Subject: [PATCH 30/48] Set ignore parent replication to true for project --- tap_github/repository_streams.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 3d95a1e0..34eb121f 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -1924,8 +1924,7 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]: class ProjectsStream(GitHubRestStream): name = "projects" path = "/repos/{org}/{repo}/projects" - # TODO I think this can be true, but I'm not sure. - ignore_parent_replication_key = False + ignore_parent_replication_key = True replication_key = "updated_at" primary_keys = ["id"] parent_stream_type = RepositoryStream From abf840798bf3099adee42335147ec1deaa217735 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Sat, 19 Mar 2022 02:01:04 +1100 Subject: [PATCH 31/48] fix mypy issue --- tap_github/organization_streams.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tap_github/organization_streams.py b/tap_github/organization_streams.py index dfc652a4..dbfeb452 100644 --- a/tap_github/organization_streams.py +++ b/tap_github/organization_streams.py @@ -78,7 +78,13 @@ class TeamsStream(GitHubRestStream): state_partitioning_keys = ["org"] def get_child_context(self, record: dict, context: Optional[dict]) -> dict: - return {"org": context["org"], "team_slug": record["slug"]} + new_context = {"team_slug": record["slug"]} + if context: + return { + **context, + **new_context, + } + return new_context schema = th.PropertiesList( th.Property("id", th.IntegerType), @@ -105,11 +111,13 @@ class TeamMembersStream(GitHubRestStream): state_partitioning_keys = ["org", "team_slug"] def get_child_context(self, record: dict, context: Optional[dict]) -> dict: - return { - "org": context["org"], - "team_slug": context["team_slug"], - "username": record["login"], - } + new_context = {"username": record["login"]} + if context: + return { + **context, + **new_context, + } + return new_context schema = th.PropertiesList( th.Property("login", th.StringType), From 53cca95f8916ab6726eb94bc1f3c758aebb74bc5 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Sat, 19 Mar 2022 15:27:54 +1100 Subject: [PATCH 32/48] fix mistyped params --- tap_github/repository_streams.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 34eb121f..33a620bd 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -707,13 +707,13 @@ class CollaboratorsStream(GitHubRestStream): th.Property("avatar_url", th.StringType), th.Property("gravatar_id", th.StringType), th.Property("url", th.StringType), - th.Property("html_url", th.IntegerType), + th.Property("html_url", th.StringType), th.Property("followers_url", th.DateTimeType), th.Property("following_url", th.DateTimeType), th.Property("gists_url", th.DateTimeType), th.Property("starred_url", th.StringType), th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.IntegerType), + th.Property("organizations_url", th.StringType), th.Property("repos_url", th.StringType), th.Property("events_url", th.StringType), th.Property("received_events_url", th.StringType), @@ -750,13 +750,13 @@ class AssigneesStream(GitHubRestStream): th.Property("avatar_url", th.StringType), th.Property("gravatar_id", th.StringType), th.Property("url", th.StringType), - th.Property("html_url", th.IntegerType), + th.Property("html_url", th.StringType), th.Property("followers_url", th.DateTimeType), th.Property("following_url", th.DateTimeType), th.Property("gists_url", th.DateTimeType), th.Property("starred_url", th.StringType), th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.IntegerType), + th.Property("organizations_url", th.StringType), th.Property("repos_url", th.StringType), th.Property("events_url", th.StringType), th.Property("received_events_url", th.StringType), From 17192aa74fbe8898c2356a9c8a9951584b6a8155 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Sat, 19 Mar 2022 15:37:08 +1100 Subject: [PATCH 33/48] Add parent keys --- tap_github/organization_streams.py | 12 +++++++ tap_github/repository_streams.py | 53 ++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/tap_github/organization_streams.py b/tap_github/organization_streams.py index dbfeb452..bf096f30 100644 --- a/tap_github/organization_streams.py +++ b/tap_github/organization_streams.py @@ -87,6 +87,9 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: return new_context schema = th.PropertiesList( + # Parent Keys + th.Property("org", th.StringType), + # Rest th.Property("id", th.IntegerType), th.Property("node_id", th.StringType), th.Property("url", th.StringType), @@ -120,6 +123,10 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: return new_context schema = th.PropertiesList( + # Parent keys + th.Property("org", th.StringType), + th.Property("team_slug", th.StringType), + # Rest th.Property("login", th.StringType), th.Property("id", th.IntegerType), th.Property("node_id", th.StringType), @@ -150,6 +157,11 @@ class TeamRolesStream(GitHubRestStream): state_partitioning_keys = ["org", "team_slug", "username"] schema = th.PropertiesList( + # Parent keys + th.Property("org", th.StringType), + th.Property("team_slug", th.StringType), + th.Property("username", th.StringType), + # Rest th.Property("url", th.StringType), th.Property("role", th.StringType), th.Property("state", th.StringType), diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 33a620bd..ffec8fce 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -529,6 +529,10 @@ class MilestonesStream(GitHubRestStream): ignore_parent_replication_key = False schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + # Rest th.Property("url", th.StringType), th.Property("html_url", th.StringType), th.Property("labels_url", th.StringType), @@ -580,6 +584,10 @@ class ReleasesStream(GitHubRestStream): replication_key = "published_at" schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + # Rest th.Property("url", th.StringType), th.Property("html_url", th.StringType), th.Property("assets_url", th.StringType), @@ -701,6 +709,10 @@ class CollaboratorsStream(GitHubRestStream): state_partitioning_keys = ["repo", "org"] schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + # Rest th.Property("login", th.StringType), th.Property("id", th.IntegerType), th.Property("node_id", th.StringType), @@ -744,6 +756,10 @@ class AssigneesStream(GitHubRestStream): state_partitioning_keys = ["repo", "org"] schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + # Rest th.Property("login", th.StringType), th.Property("id", th.IntegerType), th.Property("node_id", th.StringType), @@ -991,6 +1007,10 @@ def post_process(self, row: dict, context: Optional[dict] = None) -> dict: return row schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + # Rest th.Property("id", th.IntegerType), th.Property("node_id", th.StringType), th.Property("issue_number", th.IntegerType), @@ -1184,6 +1204,10 @@ class CommitCommentsStream(GitHubRestStream): ignore_parent_replication_key = False schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + # Rest th.Property("html_url", th.StringType), th.Property("url", th.StringType), th.Property("id", th.IntegerType), @@ -1511,6 +1535,11 @@ class PullRequestCommits(GitHubRestStream): state_partitioning_keys = ["pull_number"] schema = th.PropertiesList( + # Parent keys + th.Property("org", th.StringType), + th.Property("repo", th.StringType), + th.Property("pull_number", th.IntegerType), + # Rest th.Property("url", th.StringType), th.Property("sha", th.StringType), th.Property("node_id", th.StringType), @@ -1622,6 +1651,11 @@ class ReviewsStream(GitHubRestStream): state_partitioning_keys = ["pull_number"] schema = th.PropertiesList( + # Parent keys + th.Property("pull_number", th.IntegerType), + th.Property("org", th.IntegerType), + th.Property("repo", th.IntegerType), + # Rest th.Property("id", th.IntegerType), th.Property("node_id", th.StringType), th.Property( @@ -1675,6 +1709,10 @@ class ReviewCommentsStream(GitHubRestStream): state_partitioning_keys = ["repo", "org"] schema = th.PropertiesList( + # Parent keys + th.Property("org", th.IntegerType), + th.Property("repo", th.IntegerType), + # Rest th.Property("url", th.StringType), th.Property("pull_request_review_id", th.IntegerType), th.Property("id", th.IntegerType), @@ -1934,6 +1972,10 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: return {"project_id": record["id"]} schema = th.PropertiesList( + # Parent keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + # Rest th.Property("owner_url", th.StringType), th.Property("url", th.StringType), th.Property("html_url", th.StringType), @@ -1985,6 +2027,11 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: return {"column_id": record["id"]} schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.IntegerType), + th.Property("org", th.IntegerType), + th.Property("project_id", th.IntegerType), + # Rest th.Property("url", th.StringType), th.Property("project_url", th.StringType), th.Property("cards_url", th.StringType), @@ -2006,6 +2053,12 @@ class ProjectCardsStream(GitHubRestStream): state_partitioning_keys = ["column_id"] schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.IntegerType), + th.Property("org", th.IntegerType), + th.Property("project_id", th.IntegerType), + th.Property("column_id", th.IntegerType), + # Properties th.Property("url", th.StringType), th.Property("id", th.IntegerType), th.Property("node_id", th.StringType), From b423cd2c67b188993088fa2619327f0197fdf2e1 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Sat, 19 Mar 2022 15:44:08 +1100 Subject: [PATCH 34/48] Fix mistyped params --- tap_github/repository_streams.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index ffec8fce..9e4b93eb 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -1653,8 +1653,8 @@ class ReviewsStream(GitHubRestStream): schema = th.PropertiesList( # Parent keys th.Property("pull_number", th.IntegerType), - th.Property("org", th.IntegerType), - th.Property("repo", th.IntegerType), + th.Property("org", th.StringType), + th.Property("repo", th.StringType), # Rest th.Property("id", th.IntegerType), th.Property("node_id", th.StringType), @@ -1710,8 +1710,8 @@ class ReviewCommentsStream(GitHubRestStream): schema = th.PropertiesList( # Parent keys - th.Property("org", th.IntegerType), - th.Property("repo", th.IntegerType), + th.Property("org", th.StringType), + th.Property("repo", th.StringType), # Rest th.Property("url", th.StringType), th.Property("pull_request_review_id", th.IntegerType), @@ -2028,8 +2028,8 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: schema = th.PropertiesList( # Parent Keys - th.Property("repo", th.IntegerType), - th.Property("org", th.IntegerType), + th.Property("repo", th.StringType), + th.Property("org", th.StringType), th.Property("project_id", th.IntegerType), # Rest th.Property("url", th.StringType), @@ -2054,8 +2054,8 @@ class ProjectCardsStream(GitHubRestStream): schema = th.PropertiesList( # Parent Keys - th.Property("repo", th.IntegerType), - th.Property("org", th.IntegerType), + th.Property("repo", th.StringType), + th.Property("org", th.StringType), th.Property("project_id", th.IntegerType), th.Property("column_id", th.IntegerType), # Properties From b35d675d63adbd05258941a6cef3cfd218f94187 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Sat, 19 Mar 2022 16:02:50 +1100 Subject: [PATCH 35/48] Fix mistyped ids in events --- tap_github/repository_streams.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 9e4b93eb..11f3e6b3 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -389,7 +389,7 @@ def post_process(self, row: dict, context: Optional[dict] = None) -> dict: return row schema = th.PropertiesList( - th.Property("id", th.IntegerType), + th.Property("id", th.StringType), th.Property("type", th.StringType), th.Property("repo", th.StringType), th.Property("org", th.StringType), @@ -404,14 +404,14 @@ def post_process(self, row: dict, context: Optional[dict] = None) -> dict: th.Property( "target_repo", th.ObjectType( - th.Property("id", th.StringType), + th.Property("id", th.IntegerType), th.Property("name", th.StringType), ), ), th.Property( "target_org", th.ObjectType( - th.Property("id", th.StringType), + th.Property("id", th.IntegerType), th.Property("login", th.StringType), ), ), From 66f050c8dfa5ebde3b03cb00f1ce6dcf93e8a8f7 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Mon, 21 Mar 2022 19:33:14 +1100 Subject: [PATCH 36/48] [ci skip] Remove pointless comment --- tap_github/tap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_github/tap.py b/tap_github/tap.py index 07105d07..ed8ab64a 100644 --- a/tap_github/tap.py +++ b/tap_github/tap.py @@ -72,7 +72,7 @@ def discover_streams(self) -> List[Stream]: streams += [ StreamClass(tap=self) for StreamClass in stream_type.streams ] - # if streams is empty, raise exception + if not streams: raise ValueError("No valid streams found.") return streams From ac6b55412c03c7fda34eacc61ec10dec88352b3e Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Tue, 22 Mar 2022 23:44:28 +1100 Subject: [PATCH 37/48] Change ignore parent key to true --- tap_github/repository_streams.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 11f3e6b3..d3cb8ccc 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -526,7 +526,7 @@ class MilestonesStream(GitHubRestStream): replication_key = "updated_at" parent_stream_type = RepositoryStream state_partitioning_keys = ["repo", "org"] - ignore_parent_replication_key = False + ignore_parent_replication_key = True schema = th.PropertiesList( # Parent Keys @@ -577,7 +577,7 @@ class MilestonesStream(GitHubRestStream): class ReleasesStream(GitHubRestStream): name = "releases" path = "/repos/{org}/{repo}/releases" - ignore_parent_replication_key = False + ignore_parent_replication_key = True primary_keys = ["id"] parent_stream_type = RepositoryStream state_partitioning_keys = ["repo", "org"] @@ -705,7 +705,7 @@ class CollaboratorsStream(GitHubRestStream): path = "/repos/{org}/{repo}/collaborators" primary_keys = ["id"] parent_stream_type = RepositoryStream - ignore_parent_replication_key = False + ignore_parent_replication_key = True state_partitioning_keys = ["repo", "org"] schema = th.PropertiesList( @@ -752,7 +752,7 @@ class AssigneesStream(GitHubRestStream): path = "/repos/{org}/{repo}/assignees" primary_keys = ["id"] parent_stream_type = RepositoryStream - ignore_parent_replication_key = False + ignore_parent_replication_key = True state_partitioning_keys = ["repo", "org"] schema = th.PropertiesList( @@ -1194,14 +1194,13 @@ def post_process(self, row: dict, context: Optional[dict] = None) -> dict: class CommitCommentsStream(GitHubRestStream): - name = "commit_comments" path = "/repos/{org}/{repo}/comments" primary_keys = ["id"] replication_key = "updated_at" parent_stream_type = RepositoryStream state_partitioning_keys = ["repo", "org"] - ignore_parent_replication_key = False + ignore_parent_replication_key = True schema = th.PropertiesList( # Parent keys @@ -1529,7 +1528,7 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: class PullRequestCommits(GitHubRestStream): name = "pull_request_commits" path = "/repos/{org}/{repo}/pulls/{pull_number}/commits" - ignore_parent_replication_key = False + ignore_parent_replication_key = True primary_keys = ["node_id"] parent_stream_type = PullRequestsStream state_partitioning_keys = ["pull_number"] @@ -1647,7 +1646,7 @@ class ReviewsStream(GitHubRestStream): path = "/repos/{org}/{repo}/pulls/{pull_number}/reviews" primary_keys = ["id"] parent_stream_type = PullRequestsStream - ignore_parent_replication_key = False + ignore_parent_replication_key = True state_partitioning_keys = ["pull_number"] schema = th.PropertiesList( @@ -1705,7 +1704,7 @@ class ReviewCommentsStream(GitHubRestStream): path = "/repos/{org}/{repo}/pulls/comments" primary_keys = ["id"] parent_stream_type = RepositoryStream - ignore_parent_replication_key = False + ignore_parent_replication_key = True state_partitioning_keys = ["repo", "org"] schema = th.PropertiesList( @@ -2017,7 +2016,7 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: class ProjectColumnsStream(GitHubRestStream): name = "project_columns" path = "/projects/{project_id}/columns" - ignore_parent_replication_key = False + ignore_parent_replication_key = True replication_key = "updated_at" primary_keys = ["id"] parent_stream_type = ProjectsStream @@ -2046,7 +2045,7 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: class ProjectCardsStream(GitHubRestStream): name = "project_cards" path = "/projects/columns/{column_id}/cards" - ignore_parent_replication_key = False + ignore_parent_replication_key = True replication_key = "updated_at" primary_keys = ["id"] parent_stream_type = ProjectColumnsStream From 6b6d93435537c026ab1780ed9203f4ab145d2b28 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Fri, 25 Mar 2022 12:34:33 +1100 Subject: [PATCH 38/48] update ignore_parent_replication and remove unneeded import --- tap_github/repository_streams.py | 4 ++-- tap_github/streams.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index d3cb8ccc..ed0f7104 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -1528,7 +1528,7 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: class PullRequestCommits(GitHubRestStream): name = "pull_request_commits" path = "/repos/{org}/{repo}/pulls/{pull_number}/commits" - ignore_parent_replication_key = True + ignore_parent_replication_key = False primary_keys = ["node_id"] parent_stream_type = PullRequestsStream state_partitioning_keys = ["pull_number"] @@ -1646,7 +1646,7 @@ class ReviewsStream(GitHubRestStream): path = "/repos/{org}/{repo}/pulls/{pull_number}/reviews" primary_keys = ["id"] parent_stream_type = PullRequestsStream - ignore_parent_replication_key = True + ignore_parent_replication_key = False state_partitioning_keys = ["pull_number"] schema = th.PropertiesList( diff --git a/tap_github/streams.py b/tap_github/streams.py index 5e6526e6..931a3131 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -3,7 +3,6 @@ from singer_sdk.streams.core import Stream -from tap_github.organization_streams import OrganizationStream from tap_github.repository_streams import ( AnonymousContributorsStream, CommitsStream, From ef162de3f9a102f98d5fed8e49392c229578e47f Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Fri, 25 Mar 2022 16:44:04 +1100 Subject: [PATCH 39/48] Simple comment [ci skip] --- tap_github/streams.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tap_github/streams.py b/tap_github/streams.py index 931a3131..65788947 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -45,6 +45,10 @@ class Streams(Enum): + """ + Represents all streams our tap supports, and which queries (by username, by organization, etc.) you can use. + """ + valid_queries: Set[str] streams: List[Type[Stream]] From 47e42d9081b013b4bab25142127e6831665b66c2 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Fri, 25 Mar 2022 17:56:54 +1100 Subject: [PATCH 40/48] Work on comments [ci skip] --- tap_github/organization_streams.py | 33 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/tap_github/organization_streams.py b/tap_github/organization_streams.py index bf096f30..3a4bc31b 100644 --- a/tap_github/organization_streams.py +++ b/tap_github/organization_streams.py @@ -8,25 +8,17 @@ class OrganizationStream(GitHubRestStream): + """Defines a GitHub Organization Stream. + API Reference: https://docs.github.com/en/rest/reference/orgs#get-an-organization + """ name = "organizations" - - @property - def path(self) -> str: # type: ignore - """Return the API endpoint path.""" - return "/orgs/{org}" + path = "/orgs/{org}" @property def partitions(self) -> Optional[List[Dict]]: - """Return a list of partitions.""" return [{"org": org} for org in self.config["organizations"]] def get_child_context(self, record: dict, context: Optional[dict]) -> dict: - """Return a child context object from the record and optional provided context. - - By default, will return context if provided and otherwise the record dict. - Developers may override this behavior to send specific information to child - streams for context. - """ return { "org": record["login"], } @@ -40,10 +32,10 @@ def get_records(self, context: Optional[dict]) -> Iterable[Dict[str, Any]]: the API call is sent but data is discarded. """ if ( - not self.selected - and "skip_parent_streams" in self.config - and self.config["skip_parent_streams"] - and context is not None + not self.selected + and "skip_parent_streams" in self.config + and self.config["skip_parent_streams"] + and context is not None ): # build a minimal mock record so that self._sync_records # can proceed with child streams @@ -70,6 +62,9 @@ def get_records(self, context: Optional[dict]) -> Iterable[Dict[str, Any]]: class TeamsStream(GitHubRestStream): + """ + API Reference: https://docs.github.com/en/rest/reference/teams#list-teams + """ name = "teams" primary_keys = ["id"] path = "/orgs/{org}/teams" @@ -106,6 +101,9 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: class TeamMembersStream(GitHubRestStream): + """ + API Reference: https://docs.github.com/en/rest/reference/teams#list-team-members + """ name = "team_members" primary_keys = ["id"] path = "/orgs/{org}/teams/{team_slug}/members" @@ -149,6 +147,9 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: class TeamRolesStream(GitHubRestStream): + """ + API Reference: https://docs.github.com/en/rest/reference/teams#get-team-membership-for-a-user + """ name = "team_roles" path = "/orgs/{org}/teams/{team_slug}/memberships/{username}" ignore_parent_replication_key = True From 91bbcd3b9dd807a9a3941303f5c859c9d0ee3cb5 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Fri, 25 Mar 2022 21:48:22 +1100 Subject: [PATCH 41/48] Work on comments [ci skip] --- tap_github/client.py | 4 +- tap_github/organization_streams.py | 33 +++---- tap_github/repository_streams.py | 136 +++++------------------------ tap_github/user_streams.py | 10 +-- 4 files changed, 41 insertions(+), 142 deletions(-) diff --git a/tap_github/client.py b/tap_github/client.py index e1943ba3..463e3e17 100644 --- a/tap_github/client.py +++ b/tap_github/client.py @@ -121,7 +121,7 @@ def get_next_page_token( return (previous_token or 1) + 1 def get_url_params( - self, context: Optional[dict], next_page_token: Optional[Any] + self, context: Optional[Dict], next_page_token: Optional[Any] ) -> Dict[str, Any]: """Return a dictionary of values to be used in URL parameterization.""" params: dict = {"per_page": self.MAX_PER_PAGE} @@ -257,7 +257,7 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]: yield from extract_jsonpath(self.query_jsonpath, input=resp_json) def get_url_params( - self, context: Optional[dict], next_page_token: Optional[Any] + self, context: Optional[Dict], next_page_token: Optional[Any] ) -> Dict[str, Any]: """Return a dictionary of values to be used in URL parameterization.""" params = context or dict() diff --git a/tap_github/organization_streams.py b/tap_github/organization_streams.py index 3a4bc31b..37ceee65 100644 --- a/tap_github/organization_streams.py +++ b/tap_github/organization_streams.py @@ -11,6 +11,7 @@ class OrganizationStream(GitHubRestStream): """Defines a GitHub Organization Stream. API Reference: https://docs.github.com/en/rest/reference/orgs#get-an-organization """ + name = "organizations" path = "/orgs/{org}" @@ -18,12 +19,12 @@ class OrganizationStream(GitHubRestStream): def partitions(self) -> Optional[List[Dict]]: return [{"org": org} for org in self.config["organizations"]] - def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + def get_child_context(self, record: Dict, context: Optional[Dict]) -> dict: return { "org": record["login"], } - def get_records(self, context: Optional[dict]) -> Iterable[Dict[str, Any]]: + def get_records(self, context: Optional[Dict]) -> Iterable[Dict[str, Any]]: """ Override the parent method to allow skipping API calls if the stream is deselected and skip_parent_streams is True in config. @@ -32,10 +33,10 @@ def get_records(self, context: Optional[dict]) -> Iterable[Dict[str, Any]]: the API call is sent but data is discarded. """ if ( - not self.selected - and "skip_parent_streams" in self.config - and self.config["skip_parent_streams"] - and context is not None + not self.selected + and "skip_parent_streams" in self.config + and self.config["skip_parent_streams"] + and context is not None ): # build a minimal mock record so that self._sync_records # can proceed with child streams @@ -65,6 +66,7 @@ class TeamsStream(GitHubRestStream): """ API Reference: https://docs.github.com/en/rest/reference/teams#list-teams """ + name = "teams" primary_keys = ["id"] path = "/orgs/{org}/teams" @@ -72,7 +74,7 @@ class TeamsStream(GitHubRestStream): parent_stream_type = OrganizationStream state_partitioning_keys = ["org"] - def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + def get_child_context(self, record: Dict, context: Optional[Dict]) -> dict: new_context = {"team_slug": record["slug"]} if context: return { @@ -104,14 +106,15 @@ class TeamMembersStream(GitHubRestStream): """ API Reference: https://docs.github.com/en/rest/reference/teams#list-team-members """ + name = "team_members" primary_keys = ["id"] path = "/orgs/{org}/teams/{team_slug}/members" ignore_parent_replication_key = True parent_stream_type = TeamsStream - state_partitioning_keys = ["org", "team_slug"] + state_partitioning_keys = ["team_slug", "org"] - def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + def get_child_context(self, record: Dict, context: Optional[Dict]) -> dict: new_context = {"username": record["login"]} if context: return { @@ -132,15 +135,6 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: th.Property("gravatar_id", th.StringType), th.Property("url", th.StringType), th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), th.Property("type", th.StringType), th.Property("site_admin", th.BooleanType), ).to_dict() @@ -150,12 +144,13 @@ class TeamRolesStream(GitHubRestStream): """ API Reference: https://docs.github.com/en/rest/reference/teams#get-team-membership-for-a-user """ + name = "team_roles" path = "/orgs/{org}/teams/{team_slug}/memberships/{username}" ignore_parent_replication_key = True primary_keys = ["url"] parent_stream_type = TeamMembersStream - state_partitioning_keys = ["org", "team_slug", "username"] + state_partitioning_keys = ["username", "team_slug", "org"] schema = th.PropertiesList( # Parent keys diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index ed0f7104..64622f29 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -17,7 +17,7 @@ class RepositoryStream(GitHubRestStream): name = "repositories" def get_url_params( - self, context: Optional[dict], next_page_token: Optional[Any] + self, context: Optional[Dict], next_page_token: Optional[Any] ) -> Dict[str, Any]: """Return a dictionary of values to be used in URL parameterization.""" assert context is not None, f"Context cannot be empty for '{self.name}' stream." @@ -62,7 +62,7 @@ def partitions(self) -> Optional[List[Dict]]: return [{"org": org} for org in self.config["organizations"]] return None - def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + def get_child_context(self, record: Dict, context: Optional[Dict]) -> dict: """Return a child context object from the record and optional provided context. By default, will return context if provided and otherwise the record dict. @@ -74,7 +74,7 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: "repo": record["name"], } - def get_records(self, context: Optional[dict]) -> Iterable[Dict[str, Any]]: + def get_records(self, context: Optional[Dict]) -> Iterable[Dict[str, Any]]: """ Override the parent method to allow skipping API calls if the stream is deselected and skip_parent_streams is True in config. @@ -369,7 +369,7 @@ class EventsStream(GitHubRestStream): # GitHub is missing the "since" parameter on this endpoint. missing_since_parameter = True - def get_records(self, context: Optional[dict] = None) -> Iterable[Dict[str, Any]]: + def get_records(self, context: Optional[Dict] = None) -> Iterable[Dict[str, Any]]: """Return a generator of row-type dictionary objects. Each row emitted should be a dictionary of property names to their values. """ @@ -379,7 +379,7 @@ def get_records(self, context: Optional[dict] = None) -> Iterable[Dict[str, Any] return super().get_records(context) - def post_process(self, row: dict, context: Optional[dict] = None) -> dict: + def post_process(self, row: dict, context: Optional[Dict] = None) -> dict: # TODO - We should think about the best approach to handle this. An alternative would be to # do a 'dumb' tap that just keeps the same schemas as GitHub without renaming these # objects to "target_". They are worth keeping, however, as they can be different from @@ -552,15 +552,6 @@ class MilestonesStream(GitHubRestStream): th.Property("gravatar_id", th.StringType), th.Property("url", th.StringType), th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), th.Property("type", th.StringType), th.Property("site_admin", th.BooleanType), ), @@ -614,15 +605,6 @@ class ReleasesStream(GitHubRestStream): th.Property("gravatar_id", th.StringType), th.Property("url", th.StringType), th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), th.Property("type", th.StringType), th.Property("site_admin", th.BooleanType), ), @@ -793,7 +775,7 @@ class IssuesStream(GitHubRestStream): state_partitioning_keys = ["repo", "org"] def get_url_params( - self, context: Optional[dict], next_page_token: Optional[Any] + self, context: Optional[Dict], next_page_token: Optional[Any] ) -> Dict[str, Any]: """Return a dictionary of values to be used in URL parameterization.""" assert context is not None, f"Context cannot be empty for '{self.name}' stream." @@ -822,7 +804,7 @@ def http_headers(self) -> dict: headers["Accept"] = "application/vnd.github.squirrel-girl-preview" return headers - def post_process(self, row: dict, context: Optional[dict] = None) -> dict: + def post_process(self, row: dict, context: Optional[Dict] = None) -> dict: row["type"] = "pull_request" if "pull_request" in row else "issue" if row["body"] is not None: # some issue bodies include control characters such as \x00 @@ -985,7 +967,7 @@ class IssueCommentsStream(GitHubRestStream): # we have gaps in our data tolerated_http_errors = [502] - def get_records(self, context: Optional[dict] = None) -> Iterable[Dict[str, Any]]: + def get_records(self, context: Optional[Dict] = None) -> Iterable[Dict[str, Any]]: """Return a generator of row-type dictionary objects. Each row emitted should be a dictionary of property names to their values. @@ -996,7 +978,7 @@ def get_records(self, context: Optional[dict] = None) -> Iterable[Dict[str, Any] return super().get_records(context) - def post_process(self, row: dict, context: Optional[dict] = None) -> dict: + def post_process(self, row: dict, context: Optional[Dict] = None) -> dict: row["issue_number"] = int(row["issue_url"].split("/")[-1]) if row["body"] is not None: # some comment bodies include control characters such as \x00 @@ -1054,7 +1036,7 @@ class IssueEventsStream(GitHubRestStream): # GitHub is missing the "since" parameter on this endpoint. missing_since_parameter = True - def get_records(self, context: Optional[dict] = None) -> Iterable[Dict[str, Any]]: + def get_records(self, context: Optional[Dict] = None) -> Iterable[Dict[str, Any]]: """Return a generator of row-type dictionary objects. Each row emitted should be a dictionary of property names to their values. @@ -1065,7 +1047,7 @@ def get_records(self, context: Optional[dict] = None) -> Iterable[Dict[str, Any] return super().get_records(context) - def post_process(self, row: dict, context: Optional[dict] = None) -> dict: + def post_process(self, row: dict, context: Optional[Dict] = None) -> dict: row["issue_number"] = int(row["issue"].pop("number")) row["issue_url"] = row["issue"].pop("url") return row @@ -1111,7 +1093,7 @@ class CommitsStream(GitHubRestStream): state_partitioning_keys = ["repo", "org"] ignore_parent_replication_key = True - def post_process(self, row: dict, context: Optional[dict] = None) -> dict: + def post_process(self, row: dict, context: Optional[Dict] = None) -> dict: """ Add a timestamp top-level field to be used as state replication key. It's not clear from github's API docs which time (author or committer) @@ -1226,15 +1208,6 @@ class CommitCommentsStream(GitHubRestStream): th.Property("gravatar_id", th.StringType), th.Property("url", th.StringType), th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), th.Property("type", th.StringType), th.Property("site_admin", th.BooleanType), ), @@ -1259,7 +1232,7 @@ class PullRequestsStream(GitHubRestStream): missing_since_parameter = True def get_url_params( - self, context: Optional[dict], next_page_token: Optional[Any] + self, context: Optional[Dict], next_page_token: Optional[Any] ) -> Dict[str, Any]: """Return a dictionary of values to be used in URL parameterization.""" assert context is not None, f"Context cannot be empty for '{self.name}' stream." @@ -1279,7 +1252,7 @@ def http_headers(self) -> dict: headers["Accept"] = "application/vnd.github.squirrel-girl-preview" return headers - def post_process(self, row: dict, context: Optional[dict] = None) -> dict: + def post_process(self, row: dict, context: Optional[Dict] = None) -> dict: if row["body"] is not None: # some pr bodies include control characters such as \x00 # that some targets (such as postgresql) choke on. This ensures @@ -1292,7 +1265,7 @@ def post_process(self, row: dict, context: Optional[dict] = None) -> dict: row["minus_one"] = row.pop("-1", None) return row - def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + def get_child_context(self, record: Dict, context: Optional[Dict]) -> dict: if context: return { "org": context["org"], @@ -1531,7 +1504,7 @@ class PullRequestCommits(GitHubRestStream): ignore_parent_replication_key = False primary_keys = ["node_id"] parent_stream_type = PullRequestsStream - state_partitioning_keys = ["pull_number"] + state_partitioning_keys = ["pull_number", "repo", "org"] schema = th.PropertiesList( # Parent keys @@ -1594,15 +1567,6 @@ class PullRequestCommits(GitHubRestStream): th.Property("gravatar_id", th.StringType), th.Property("url", th.StringType), th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), th.Property("type", th.StringType), th.Property("site_admin", th.BooleanType), ), @@ -1617,15 +1581,6 @@ class PullRequestCommits(GitHubRestStream): th.Property("gravatar_id", th.StringType), th.Property("url", th.StringType), th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), th.Property("type", th.StringType), th.Property("site_admin", th.BooleanType), ), @@ -1647,7 +1602,7 @@ class ReviewsStream(GitHubRestStream): primary_keys = ["id"] parent_stream_type = PullRequestsStream ignore_parent_replication_key = False - state_partitioning_keys = ["pull_number"] + state_partitioning_keys = ["pull_number", "repo", "org"] schema = th.PropertiesList( # Parent keys @@ -1667,15 +1622,6 @@ class ReviewsStream(GitHubRestStream): th.Property("gravatar_id", th.StringType), th.Property("url", th.StringType), th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), th.Property("type", th.StringType), th.Property("site_admin", th.BooleanType), ), @@ -1733,15 +1679,6 @@ class ReviewCommentsStream(GitHubRestStream): th.Property("gravatar_id", th.StringType), th.Property("url", th.StringType), th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), th.Property("type", th.StringType), th.Property("site_admin", th.BooleanType), ), @@ -1793,15 +1730,6 @@ class ContributorsStream(GitHubRestStream): th.Property("gravatar_id", th.StringType), th.Property("url", th.StringType), th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), th.Property("type", th.StringType), th.Property("site_admin", th.BooleanType), th.Property("contributions", th.IntegerType), @@ -1819,7 +1747,7 @@ class AnonymousContributorsStream(GitHubRestStream): state_partitioning_keys = ["repo", "org"] def get_url_params( - self, context: Optional[dict], next_page_token: Optional[Any] + self, context: Optional[Dict], next_page_token: Optional[Any] ) -> Dict[str, Any]: """Return a dictionary of values to be used in URL parameterization.""" assert context is not None, f"Context cannot be empty for '{self.name}' stream." @@ -1868,7 +1796,7 @@ def http_headers(self) -> dict: headers["Accept"] = "application/vnd.github.v3.star+json" return headers - def post_process(self, row: dict, context: Optional[dict] = None) -> dict: + def post_process(self, row: dict, context: Optional[Dict] = None) -> dict: """ Add a user_id top-level field to be used as state replication key. """ @@ -1967,7 +1895,7 @@ class ProjectsStream(GitHubRestStream): parent_stream_type = RepositoryStream state_partitioning_keys = ["repo", "org"] - def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + def get_child_context(self, record: Dict, context: Optional[Dict]) -> dict: return {"project_id": record["id"]} schema = th.PropertiesList( @@ -1995,15 +1923,6 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: th.Property("gravatar_id", th.StringType), th.Property("url", th.StringType), th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), th.Property("type", th.StringType), th.Property("site_admin", th.BooleanType), ), @@ -2020,9 +1939,9 @@ class ProjectColumnsStream(GitHubRestStream): replication_key = "updated_at" primary_keys = ["id"] parent_stream_type = ProjectsStream - state_partitioning_keys = ["project_id"] + state_partitioning_keys = ["project_id", "repo", "org"] - def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + def get_child_context(self, record: Dict, context: Optional[Dict]) -> dict: return {"column_id": record["id"]} schema = th.PropertiesList( @@ -2049,7 +1968,7 @@ class ProjectCardsStream(GitHubRestStream): replication_key = "updated_at" primary_keys = ["id"] parent_stream_type = ProjectColumnsStream - state_partitioning_keys = ["column_id"] + state_partitioning_keys = ["column_id", "project_id", "repo", "org"] schema = th.PropertiesList( # Parent Keys @@ -2072,15 +1991,6 @@ class ProjectCardsStream(GitHubRestStream): th.Property("gravatar_id", th.StringType), th.Property("url", th.StringType), th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), th.Property("type", th.StringType), th.Property("site_admin", th.BooleanType), ), diff --git a/tap_github/user_streams.py b/tap_github/user_streams.py index 47a09b82..18ad837d 100644 --- a/tap_github/user_streams.py +++ b/tap_github/user_streams.py @@ -29,13 +29,7 @@ def partitions(self) -> Optional[List[Dict]]: return [{"id": id} for id in self.config["user_ids"]] return None - def get_child_context(self, record: dict, context: Optional[dict]) -> dict: - """Return a child context object from the record and optional provided context. - - By default, will return context if provided and otherwise the record dict. - Developers may override this behavior to send specific information to child - streams for context. - """ + def get_child_context(self, record: Dict, context: Optional[Dict]) -> dict: return { "username": record["login"], } @@ -101,7 +95,7 @@ def http_headers(self) -> dict: headers["Accept"] = "application/vnd.github.v3.star+json" return headers - def post_process(self, row: dict, context: Optional[dict] = None) -> dict: + def post_process(self, row: dict, context: Optional[Dict] = None) -> dict: """ Add a repo_id top-level field to be used as state replication key. """ From e6298e033f9a609655c4f73868b7c14dfea4ee65 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Fri, 25 Mar 2022 21:53:46 +1100 Subject: [PATCH 42/48] Fix mistyped stuff (good catch Laurent) and more comment addressing --- tap_github/repository_streams.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 64622f29..c8ec2070 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -635,15 +635,6 @@ class ReleasesStream(GitHubRestStream): th.Property("gravatar_id", th.StringType), th.Property("url", th.StringType), th.Property("html_url", th.StringType), - th.Property("followers_url", th.StringType), - th.Property("following_url", th.StringType), - th.Property("gists_url", th.StringType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), th.Property("type", th.StringType), th.Property("site_admin", th.BooleanType), ), @@ -702,15 +693,6 @@ class CollaboratorsStream(GitHubRestStream): th.Property("gravatar_id", th.StringType), th.Property("url", th.StringType), th.Property("html_url", th.StringType), - th.Property("followers_url", th.DateTimeType), - th.Property("following_url", th.DateTimeType), - th.Property("gists_url", th.DateTimeType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), th.Property("type", th.StringType), th.Property("site_admin", th.BooleanType), th.Property( @@ -749,15 +731,6 @@ class AssigneesStream(GitHubRestStream): th.Property("gravatar_id", th.StringType), th.Property("url", th.StringType), th.Property("html_url", th.StringType), - th.Property("followers_url", th.DateTimeType), - th.Property("following_url", th.DateTimeType), - th.Property("gists_url", th.DateTimeType), - th.Property("starred_url", th.StringType), - th.Property("subscriptions_url", th.StringType), - th.Property("organizations_url", th.StringType), - th.Property("repos_url", th.StringType), - th.Property("events_url", th.StringType), - th.Property("received_events_url", th.StringType), th.Property("type", th.StringType), th.Property("site_admin", th.BooleanType), ).to_dict() From 643c99b030a73ced4b129c4918e5b0c535bef0b2 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Mon, 28 Mar 2022 00:38:41 +1100 Subject: [PATCH 43/48] Update fixture comment [ci skip] --- tap_github/tests/fixtures.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tap_github/tests/fixtures.py b/tap_github/tests/fixtures.py index 08a7fab4..1fb7b8b5 100644 --- a/tap_github/tests/fixtures.py +++ b/tap_github/tests/fixtures.py @@ -77,8 +77,8 @@ def user_id_list_config(request): @pytest.fixture def organization_list_config(request): """ - Get a default list of usernames or pass your own by decorating your test with - @pytest.mark.user_id_list(['ericboucher', 'aaronsteers']) + Get a default list of organizations or pass your own by decorating your test with + @pytest.mark.organization_list(['MeltanoLabs', 'oviohub']) """ marker = request.node.get_closest_marker("organization_list") From 3ff49ec74792a26c979ac515d03974fdb8653212 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Mon, 28 Mar 2022 00:43:04 +1100 Subject: [PATCH 44/48] Add bunch of meltano lab sample projects --- tap_github/tests/test_tap.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tap_github/tests/test_tap.py b/tap_github/tests/test_tap.py index c3278ee8..a9efc82b 100644 --- a/tap_github/tests/test_tap.py +++ b/tap_github/tests/test_tap.py @@ -4,7 +4,11 @@ from .fixtures import repo_list_config -repo_list_2 = ["MeltanoLabs/tap-github"] +repo_list_2 = [ + "MeltanoLabs/tap-github", + "MeltanoLabs/tap-gitlab", + "MeltanoLabs/target-athena", +] @pytest.mark.repo_list(repo_list_2) From 0b3f52fd4a44d5f999bbb054731a6f398abfd81b Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Mon, 28 Mar 2022 01:00:01 +1100 Subject: [PATCH 45/48] update state partitioning keys --- tap_github/repository_streams.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index c8ec2070..e2390200 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -1477,7 +1477,7 @@ class PullRequestCommits(GitHubRestStream): ignore_parent_replication_key = False primary_keys = ["node_id"] parent_stream_type = PullRequestsStream - state_partitioning_keys = ["pull_number", "repo", "org"] + state_partitioning_keys = ["repo", "org"] schema = th.PropertiesList( # Parent keys @@ -1575,7 +1575,7 @@ class ReviewsStream(GitHubRestStream): primary_keys = ["id"] parent_stream_type = PullRequestsStream ignore_parent_replication_key = False - state_partitioning_keys = ["pull_number", "repo", "org"] + state_partitioning_keys = ["repo", "org"] schema = th.PropertiesList( # Parent keys @@ -1941,7 +1941,7 @@ class ProjectCardsStream(GitHubRestStream): replication_key = "updated_at" primary_keys = ["id"] parent_stream_type = ProjectColumnsStream - state_partitioning_keys = ["column_id", "project_id", "repo", "org"] + state_partitioning_keys = ["project_id", "repo", "org"] schema = th.PropertiesList( # Parent Keys From 0ec1d16041a70f40bb8fa57ffa78405599146d19 Mon Sep 17 00:00:00 2001 From: Ryan Samarakoon Date: Thu, 31 Mar 2022 14:03:45 +1100 Subject: [PATCH 46/48] Fix merge --- tap_github/streams.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tap_github/streams.py b/tap_github/streams.py index 65788947..2ae5df57 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -30,6 +30,9 @@ MilestonesStream, CommitCommentsStream, ReleasesStream, + WorkflowsStream, + WorkflowRunJobsStream, + WorkflowRunsStream, ) from tap_github.user_streams import ( StarredStream, @@ -85,6 +88,9 @@ def __init__(self, valid_queries: Set[str], streams: List[Type[Stream]]): ProjectsStream, ProjectColumnsStream, ProjectCardsStream, + WorkflowsStream, + WorkflowRunJobsStream, + WorkflowRunsStream, ], ) USERS = ( From 36136ed7deadf7a7acdf8a4773d222980f8630d7 Mon Sep 17 00:00:00 2001 From: Eric Boucher Date: Tue, 5 Apr 2022 17:44:51 -0400 Subject: [PATCH 47/48] Add ORG_LEVEL_TOKEN to be used only for specific streams --- .github/workflows/test_tap.yml | 1 + tap_github/tests/test_core.py | 16 +++++++++++++--- tap_github/tests/test_tap.py | 32 ++++++++++++++++++++++++++++++-- 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_tap.yml b/.github/workflows/test_tap.yml index b587fb97..a5fa2481 100644 --- a/.github/workflows/test_tap.yml +++ b/.github/workflows/test_tap.yml @@ -13,6 +13,7 @@ jobs: runs-on: ubuntu-latest env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + ORG_LEVEL_TOKEN: ${{secrets.ORG_LEVEL_TOKEN}} strategy: matrix: python-version: ["3.10", 3.9, 3.8, 3.7] diff --git a/tap_github/tests/test_core.py b/tap_github/tests/test_core.py index e34cc087..edbc527a 100644 --- a/tap_github/tests/test_core.py +++ b/tap_github/tests/test_core.py @@ -1,4 +1,8 @@ """Tests standard tap features using the built-in SDK tests library.""" +import os +import logging + +from unittest import mock from singer_sdk.testing import get_standard_tap_tests @@ -34,11 +38,17 @@ def test_standard_tap_tests_for_username_list_mode(username_list_config): test() +# This token needs to have read:org access for the organization listed in fixtures.py +# Default is "MeltanoLabs" +ORG_LEVEL_TOKEN = os.environ.get("ORG_LEVEL_TOKEN") + + +@mock.patch.dict(os.environ, {"GITHUB_TOKEN": ORG_LEVEL_TOKEN or ""}) def test_standard_tap_tests_for_organization_list_mode(organization_list_config): """Run standard tap tests from the SDK.""" + if not ORG_LEVEL_TOKEN: + logging.warning('No "ORG_LEVEL_TOKEN" found. Skipping organization tap tests.') + return tests = get_standard_tap_tests(TapGitHub, config=organization_list_config) for test in tests: test() - - -# TODO: Create additional tests as appropriate for your tap. diff --git a/tap_github/tests/test_tap.py b/tap_github/tests/test_tap.py index a9efc82b..9dedb9e6 100644 --- a/tap_github/tests/test_tap.py +++ b/tap_github/tests/test_tap.py @@ -1,5 +1,9 @@ +import os +import logging import pytest +from unittest.mock import patch + from tap_github.tap import TapGitHub from .fixtures import repo_list_config @@ -22,6 +26,27 @@ def test_validate_repo_list_config(repo_list_config): assert partitions == repo_list_context +def alternative_sync_chidren(self, child_context: dict) -> None: + for child_stream in self.child_streams: + # Use org:write access level credentials for collaborators stream + if child_stream.name in ["collaborators"]: + ORG_LEVEL_TOKEN = os.environ.get("ORG_LEVEL_TOKEN") + if not ORG_LEVEL_TOKEN: + logging.warning( + 'No "ORG_LEVEL_TOKEN" found. Skipping collaborators stream sync.' + ) + continue + SAVED_GTHUB_TOKEN = os.environ.get("GITHUB_TOKEN") + os.environ["GITHUB_TOKEN"] = ORG_LEVEL_TOKEN + child_stream.sync(context=child_context) + os.environ["GITHUB_TOKEN"] = SAVED_GTHUB_TOKEN or "" + continue + + # default behavior: + if child_stream.selected or child_stream.has_selected_descendents: + child_stream.sync(context=child_context) + + @pytest.mark.repo_list(repo_list_2) def test_get_a_repository_in_repo_list_mode(capsys, repo_list_config): """ @@ -38,8 +63,11 @@ def test_get_a_repository_in_repo_list_mode(capsys, repo_list_config): # ) # discard previous output to stdout (potentially from other tests) capsys.readouterr() - tap2 = TapGitHub(config=repo_list_config, catalog=catalog) - tap2.sync_all() + with patch( + "singer_sdk.streams.core.Stream._sync_children", alternative_sync_chidren + ): + tap2 = TapGitHub(config=repo_list_config, catalog=catalog) + tap2.sync_all() captured = capsys.readouterr() # Verify we got the right number of records (one per repo in the list) assert captured.out.count('{"type": "RECORD", "stream": "repositories"') == len( From 202845b73248a45179cb5016dd28226287c93fe7 Mon Sep 17 00:00:00 2001 From: Eric Boucher Date: Wed, 6 Apr 2022 10:18:36 -0400 Subject: [PATCH 48/48] Add docstring to alternative_sync_chidren --- tap_github/tests/test_tap.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tap_github/tests/test_tap.py b/tap_github/tests/test_tap.py index 9dedb9e6..6ad8e852 100644 --- a/tap_github/tests/test_tap.py +++ b/tap_github/tests/test_tap.py @@ -27,6 +27,10 @@ def test_validate_repo_list_config(repo_list_config): def alternative_sync_chidren(self, child_context: dict) -> None: + """ + Override for Stream._sync_children. + Enabling us to use an ORG_LEVEL_TOKEN for the collaborators sream. + """ for child_stream in self.child_streams: # Use org:write access level credentials for collaborators stream if child_stream.name in ["collaborators"]: