From 11a8db06f854c29fcf110cce8a334edc1a92dbc4 Mon Sep 17 00:00:00 2001 From: Ryan Date: Tue, 9 Nov 2021 19:59:07 +1100 Subject: [PATCH 1/7] Add pull request files stream --- tap_github/streams.py | 35 +++++++++++++++++++++++++++++++++++ tap_github/tap.py | 2 ++ 2 files changed, 37 insertions(+) diff --git a/tap_github/streams.py b/tap_github/streams.py index 0f351cc0..dc384955 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -675,6 +675,15 @@ def get_url_params( params["state"] = "all" return params + def get_child_context(self, record: dict, context: Optional[dict]) -> dict: + """ + Give the PR number to children streams + """ + + return super().get_child_context(record, context) | { + "pull_number": record["number"], + } + @property def http_headers(self) -> dict: """Return the http headers needed. @@ -906,6 +915,32 @@ def http_headers(self) -> dict: ).to_dict() +class PullRequestFilesStream(GitHubStream): + """Defines 'PullRequestFiles' stream.""" + + name = "pull_request_files" + path = "/repos/{org}/{repo}/pulls/{pull_number}/files" + primary_keys = ["sha"] + parent_stream_type = PullRequestsStream + ignore_parent_replication_key = True + state_partitioning_keys = ["repo", "org"] + + schema = th.PropertiesList( + # Key + th.Property("sha", th.StringType), + # Rest + th.Property("filename", th.StringType), + th.Property("status", th.StringType), + th.Property("additions", th.IntegerType), + th.Property("deletions", th.IntegerType), + th.Property("changes", th.IntegerType), + th.Property("blob_url", th.StringType), + th.Property("raw_url", th.StringType), + th.Property("contents_url", th.StringType), + th.Property("patch", th.StringType), + ).to_dict() + + class StargazersStream(GitHubStream): """Defines 'Stargazers' stream. Warning: this stream does NOT track star deletions.""" diff --git a/tap_github/tap.py b/tap_github/tap.py index d1fc1ebe..4f01cae7 100644 --- a/tap_github/tap.py +++ b/tap_github/tap.py @@ -12,6 +12,7 @@ IssueEventsStream, IssuesStream, PullRequestsStream, + PullRequestFilesStream, ReadmeStream, RepositoryStream, StargazersStream, @@ -51,6 +52,7 @@ def discover_streams(self) -> List[Stream]: IssueEventsStream(tap=self), IssuesStream(tap=self), PullRequestsStream(tap=self), + PullRequestFilesStream(tap=self), ReadmeStream(tap=self), RepositoryStream(tap=self), StargazersStream(tap=self), From b2b387b7388f511a827f0f95a6e4c7953c590d1b Mon Sep 17 00:00:00 2001 From: Ryan Date: Tue, 9 Nov 2021 20:13:22 +1100 Subject: [PATCH 2/7] Make filename key after study into prop meanings --- tap_github/streams.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tap_github/streams.py b/tap_github/streams.py index dc384955..2dba33cf 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -920,16 +920,16 @@ class PullRequestFilesStream(GitHubStream): name = "pull_request_files" path = "/repos/{org}/{repo}/pulls/{pull_number}/files" - primary_keys = ["sha"] + primary_keys = ["filename"] parent_stream_type = PullRequestsStream - ignore_parent_replication_key = True + ignore_parent_replication_key = False state_partitioning_keys = ["repo", "org"] schema = th.PropertiesList( # Key - th.Property("sha", th.StringType), - # Rest th.Property("filename", th.StringType), + # Rest + th.Property("sha", th.StringType), th.Property("status", th.StringType), th.Property("additions", th.IntegerType), th.Property("deletions", th.IntegerType), From 91e4be056ae64d4366e29610390fe726d216f58c Mon Sep 17 00:00:00 2001 From: Ryan Date: Tue, 9 Nov 2021 20:25:41 +1100 Subject: [PATCH 3/7] Remove modern (3.9) syntax for 3.5+ syntax --- tap_github/streams.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tap_github/streams.py b/tap_github/streams.py index 2dba33cf..a6d4259f 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -1,6 +1,7 @@ """Stream type classes for tap-github.""" from typing import Any, Dict, Iterable, List, Optional + from singer_sdk import typing as th # JSON Schema typing helpers from tap_github.client import GitHubStream @@ -680,8 +681,11 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: Give the PR number to children streams """ - return super().get_child_context(record, context) | { - "pull_number": record["number"], + return { + **super().get_child_context(record, context), + **{ + "pull_number": record["number"], + }, } @property From 6552901f91860a7fc92f70554071e700a20b5b4d Mon Sep 17 00:00:00 2001 From: Ryan Date: Wed, 10 Nov 2021 16:15:47 +1100 Subject: [PATCH 4/7] add pr number to state_partitioning_keys --- tap_github/streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_github/streams.py b/tap_github/streams.py index a6d4259f..f176b264 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -927,7 +927,7 @@ class PullRequestFilesStream(GitHubStream): primary_keys = ["filename"] parent_stream_type = PullRequestsStream ignore_parent_replication_key = False - state_partitioning_keys = ["repo", "org"] + state_partitioning_keys = ["repo", "org", "pull_number"] schema = th.PropertiesList( # Key From a3d6bc575247d59353737a1c450695b5c8f16c9e Mon Sep 17 00:00:00 2001 From: Ryan Date: Wed, 10 Nov 2021 17:53:18 +1100 Subject: [PATCH 5/7] Add parent key --- tap_github/streams.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tap_github/streams.py b/tap_github/streams.py index f176b264..f2ae89ae 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -929,9 +929,15 @@ class PullRequestFilesStream(GitHubStream): ignore_parent_replication_key = False state_partitioning_keys = ["repo", "org", "pull_number"] + def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[dict]: + # Add the pull_number from context + row["pull_number"] = context["pull_number"] + return row + schema = th.PropertiesList( - # Key + # Key (including parent) th.Property("filename", th.StringType), + th.Property("pull_number", th.IntegerType), # Rest th.Property("sha", th.StringType), th.Property("status", th.StringType), From 809c5ef253ff8928d57857b1dda679c6d9607f53 Mon Sep 17 00:00:00 2001 From: Ryan Date: Wed, 10 Nov 2021 18:03:02 +1100 Subject: [PATCH 6/7] Remove redundant post_processing --- tap_github/streams.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tap_github/streams.py b/tap_github/streams.py index f2ae89ae..b5450eac 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -929,11 +929,6 @@ class PullRequestFilesStream(GitHubStream): ignore_parent_replication_key = False state_partitioning_keys = ["repo", "org", "pull_number"] - def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[dict]: - # Add the pull_number from context - row["pull_number"] = context["pull_number"] - return row - schema = th.PropertiesList( # Key (including parent) th.Property("filename", th.StringType), From 303f36ee6dab7d2d0062f0d07bf5c81a32b1e8f2 Mon Sep 17 00:00:00 2001 From: Ryan Date: Thu, 11 Nov 2021 15:09:50 +1100 Subject: [PATCH 7/7] Make primary key use more vars --- tap_github/streams.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tap_github/streams.py b/tap_github/streams.py index b5450eac..7ae56741 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -924,15 +924,17 @@ class PullRequestFilesStream(GitHubStream): name = "pull_request_files" path = "/repos/{org}/{repo}/pulls/{pull_number}/files" - primary_keys = ["filename"] + state_partitioning_keys = ["repo", "org", "pull_number"] + primary_keys = state_partitioning_keys + ["filename"] parent_stream_type = PullRequestsStream ignore_parent_replication_key = False - state_partitioning_keys = ["repo", "org", "pull_number"] schema = th.PropertiesList( - # Key (including parent) - th.Property("filename", th.StringType), + # Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), th.Property("pull_number", th.IntegerType), + th.Property("filename", th.StringType), # Rest th.Property("sha", th.StringType), th.Property("status", th.StringType),