From 0a4f0c7a81b6b7dfa55b59f4d06f7685af08122b Mon Sep 17 00:00:00 2001 From: vikasrathee-cs Date: Wed, 24 Jan 2024 19:03:43 +0530 Subject: [PATCH] Add change for File Identifier, macro for Oauth 2 and access token fields and other bugs. Add change for File Identifier, macro for Oauth 2 and access token fields. --- docs/GoogleDrive-batchsink.md | 5 + docs/GoogleDrive-batchsource.md | 15 +++ docs/GoogleSheets-batchsink.md | 5 + docs/GoogleSheets-batchsource.md | 21 ++- .../google/common/GoogleAuthBaseConfig.java | 125 ++++++++++++++---- .../google/common/GoogleDriveClient.java | 28 ++-- .../common/GoogleDriveFilteringClient.java | 4 + .../plugin/google/common/OAuthMethod.java | 24 ++++ .../google/common/ValidationResult.java | 10 +- .../drive/source/GoogleDriveSourceConfig.java | 10 ++ .../source/GoogleSheetsRecordReader.java | 32 +++-- .../source/GoogleSheetsSourceConfig.java | 66 ++++++--- .../common/GoogleAuthBaseConfigTest.java | 69 ++++++++++ .../sink/GoogleSheetsSinkClientTest.java | 2 + widgets/GoogleDrive-batchsink.json | 56 ++++++++ widgets/GoogleDrive-batchsource.json | 56 ++++++++ widgets/GoogleSheets-batchsink.json | 60 ++++++++- widgets/GoogleSheets-batchsource.json | 67 +++++++++- 18 files changed, 577 insertions(+), 78 deletions(-) create mode 100644 src/main/java/io/cdap/plugin/google/common/OAuthMethod.java diff --git a/docs/GoogleDrive-batchsink.md b/docs/GoogleDrive-batchsink.md index 6ff834a7..3c2f336e 100644 --- a/docs/GoogleDrive-batchsink.md +++ b/docs/GoogleDrive-batchsink.md @@ -43,6 +43,11 @@ Make sure that: OAuth2 client credentials can be generated on Google Cloud [Credentials Page](https://console.cloud.google.com/apis/credentials) +**OAuth Method:** The method used to get OAuth access tokens. The oauth access token can be directly provided, +or a client id, client secret, and refresh token can be provided. + +**Access Token:** Short lived access token for connect. + **Client ID:** OAuth2 client id used to identify the application. **Client Secret:** OAuth2 client secret used to access the authorization server. diff --git a/docs/GoogleDrive-batchsource.md b/docs/GoogleDrive-batchsource.md index c8dbd921..63eea2d6 100644 --- a/docs/GoogleDrive-batchsource.md +++ b/docs/GoogleDrive-batchsource.md @@ -17,6 +17,16 @@ https://drive.google.com/drive/folders/1dyUEebJaFnWa3Z4n0BFMVAXQ7mfUH11g?resourc ``` Then the Directory Identifier would be `1dyUEebJaFnWa3Z4n0BFMVAXQ7mfUH11g`. +**File Identifier:** Identifier of the file. + +This comes after `file/d/ or document/d/ or spreadsheets/d/` in the URL. For example, if the URL is +``` +https://docs.google.com/file/d/17W3vOhBwe0i24OdVNsbz8rAMClzUitKeAbumTqWFrkows +``` +Then the File Identifier would be `17W3vOhBwe0i24OdVNsbz8rAMClzUitKeAbumTqWFrkows`. +Either Directory Identifier or File Identifier should have a value. Filters will not work +while providing File Identifier. + **File Metadata Properties:** Properties that represent metadata of files. They will be a part of output structured record. Descriptions for properties can be view at [Drive API file reference](https://developers.google.com/drive/api/v3/reference/files). @@ -58,6 +68,11 @@ Make sure that: OAuth2 client credentials can be generated on Google Cloud [Credentials Page](https://console.cloud.google.com/apis/credentials) +**OAuth Method:** The method used to get OAuth access tokens. The oauth access token can be directly provided, +or a client id, client secret, and refresh token can be provided. + +**Access Token:** Short lived access token for connect. + **Client ID:** OAuth2 client id used to identify the application. **Client Secret:** OAuth2 client secret used to access the authorization server. diff --git a/docs/GoogleSheets-batchsink.md b/docs/GoogleSheets-batchsink.md index d632e050..f82b75a4 100644 --- a/docs/GoogleSheets-batchsink.md +++ b/docs/GoogleSheets-batchsink.md @@ -46,6 +46,11 @@ Make sure that: OAuth2 client credentials can be generated on Google Cloud [Credentials Page](https://console.cloud.google.com/apis/credentials) +**OAuth Method:** The method used to get OAuth access tokens. The oauth access token can be directly provided, +or a client id, client secret, and refresh token can be provided. + +**Access Token:** Short lived access token for connect. + **Client ID:** OAuth2 client id used to identify the application. **Client Secret:** OAuth2 client secret used to access the authorization server. diff --git a/docs/GoogleSheets-batchsource.md b/docs/GoogleSheets-batchsource.md index ccf285fb..0e529424 100644 --- a/docs/GoogleSheets-batchsource.md +++ b/docs/GoogleSheets-batchsource.md @@ -17,6 +17,16 @@ https://drive.google.com/drive/folders/1dyUEebJaFnWa3Z4n0BFMVAXQ7mfUH11g?resourc ``` Then the Directory Identifier would be `1dyUEebJaFnWa3Z4n0BFMVAXQ7mfUH11g`. +**File Identifier:** Identifier of the spreadsheet file. + +This comes after `spreadsheets/d/` in the URL. For example, if the URL is +``` +https://docs.google.com/spreadsheets/d/17W3vOhBwe0i24OdVNsbz8rAMClzUitKeAbumTqWFrkows +``` +Then the File Identifier would be `17W3vOhBwe0i24OdVNsbz8rAMClzUitKeAbumTqWFrkows`. +Either Directory Identifier or File Identifier should have a value. Filters will not work +while providing File Identifier. + ### Filtering **Filter:** Filter that can be applied to the files in the selected directory. @@ -55,6 +65,11 @@ Make sure that: OAuth2 client credentials can be generated on Google Cloud [Credentials Page](https://console.cloud.google.com/apis/credentials) +**OAuth Method:** The method used to get OAuth access tokens. The oauth access token can be directly provided, +or a client id, client secret, and refresh token can be provided. + +**Access Token:** Short lived access token for connect. + **Client ID:** OAuth2 client id used to identify the application. **Client Secret:** OAuth2 client secret used to access the authorization server. @@ -124,10 +139,10 @@ _Treat first row as column names_ - the plugin uses first row for schema definin **Column Names Row Number:** Number of the row to be treated as a header. Only shown when the 'Column Names Selection' field is set to 'Custom row as column names' header. -**Number of Columns to Read:** Last column plugin will read as data. It will be ignored if the Column -Names Row contains less number of columns. +**Number of Columns to Read:** Last column plugin will read as data. It will be ignored if the Column +Names Row contains less number of columns. Set it to 0 to read all the columns in the sheet. -**Number of Rows to Read:** Last row plugin will read as data. +**Number of Rows to Read:** Last row plugin will read as data. Set it to 0 to read all the rows in the sheet. **Read Buffer Size:** Number of rows the source reads with a single API request. Default value is 100. diff --git a/src/main/java/io/cdap/plugin/google/common/GoogleAuthBaseConfig.java b/src/main/java/io/cdap/plugin/google/common/GoogleAuthBaseConfig.java index 0672f10b..b4c860cb 100644 --- a/src/main/java/io/cdap/plugin/google/common/GoogleAuthBaseConfig.java +++ b/src/main/java/io/cdap/plugin/google/common/GoogleAuthBaseConfig.java @@ -47,8 +47,14 @@ public abstract class GoogleAuthBaseConfig extends PluginConfig { public static final String CLIENT_SECRET_LABEL = "Client secret"; public static final String REFRESH_TOKEN = "refreshToken"; public static final String REFRESH_TOKEN_LABEL = "Refresh token"; + + public static final String ACCESS_TOKEN = "accessToken"; + public static final String ACCESS_TOKEN_LABEL = "Access token"; + + public static final String OAUTH_METHOD = "oauthMethod"; public static final String ACCOUNT_FILE_PATH = "accountFilePath"; public static final String DIRECTORY_IDENTIFIER = "directoryIdentifier"; + public static final String FILE_IDENTIFIER = "fileIdentifier"; public static final String NAME_SERVICE_ACCOUNT_TYPE = "serviceAccountType"; public static final String NAME_SERVICE_ACCOUNT_JSON = "serviceAccountJSON"; public static final String SERVICE_ACCOUNT_FILE_PATH = "filePath"; @@ -76,21 +82,38 @@ public abstract class GoogleAuthBaseConfig extends PluginConfig { @Macro protected String serviceAccountType; + @Macro @Nullable + @Name(OAUTH_METHOD) + @Description("The method used to get OAuth access tokens. " + + "The oauth access token can be directly provided, " + + "or a client id, client secret, and refresh token can be provided.") + private String oauthMethod; + + @Nullable + @Macro @Name(CLIENT_ID) @Description("OAuth2 client id.") private String clientId; @Nullable + @Macro @Name(CLIENT_SECRET) @Description("OAuth2 client secret.") private String clientSecret; @Nullable + @Macro @Name(REFRESH_TOKEN) @Description("OAuth2 refresh token.") private String refreshToken; + @Nullable + @Macro + @Name(ACCESS_TOKEN) + @Description("Short lived access token for connect.") + private String accessToken; + @Nullable @Macro @Name(ACCOUNT_FILE_PATH) @@ -108,12 +131,22 @@ public abstract class GoogleAuthBaseConfig extends PluginConfig { @Macro protected String serviceAccountJson; + @Nullable + @Macro @Name(DIRECTORY_IDENTIFIER) @Description("Identifier of the folder. This comes after “folders/” in the URL. For example, if the URL was " + "“https://drive.google.com/drive/folders/1dyUEebJaFnWa3Z4n0BFMVAXQ7mfUH11g?resourcekey=0-XVijrJSp3E3gkdJp20MpCQ”, " + "then the Directory Identifier would be “1dyUEebJaFnWa3Z4n0BFMVAXQ7mfUH11g”.") private String directoryIdentifier; + @Nullable + @Macro + @Name(FILE_IDENTIFIER) + @Description("Identifier of the file. This comes after “file/d/ or spreadsheets/d/ or document/d/” in the URL. " + + "For example, if the URL was “https://drive.google.com/file/d/16npTpL3ozkAzB5kLQ-oQD3IlTZhnnh2w1/view”, " + + "then the File Identifier would be “16npTpL3ozkAzB5kLQ-oQD3IlTZhnnh2w1”.") + private String fileIdentifier; + /** * Returns the ValidationResult. * @@ -122,7 +155,7 @@ public abstract class GoogleAuthBaseConfig extends PluginConfig { */ public ValidationResult validate(FailureCollector collector) { IdUtils.validateReferenceName(referenceName, collector); - + checkIfDirectoryOrFileIdentifierExists(collector); ValidationResult validationResult = new ValidationResult(); if (validateAuthType(collector)) { AuthType authType = getAuthType(); @@ -143,13 +176,10 @@ public ValidationResult validate(FailureCollector collector) { try { GoogleDriveClient client = new GoogleDriveClient(this); - // validate auth - validateCredentials(collector, client); - - // validate directory - validateDirectoryIdentifier(collector, client); - - validationResult.setDirectoryAccessible(true); + // check directory or file access + if (isDirectoryOrFileAccessible(collector, client)) { + validationResult.setDirectoryOrFileAccessible(true); + } } catch (Exception e) { collector.addFailure( String.format("Exception during authentication/directory properties check: %s.", e.getMessage()), @@ -177,9 +207,13 @@ private boolean validateAuthType(FailureCollector collector) { } private boolean validateOAuth2Properties(FailureCollector collector) { - return checkPropertyIsSet(collector, clientId, CLIENT_ID, CLIENT_ID_LABEL) - & checkPropertyIsSet(collector, clientSecret, CLIENT_SECRET, CLIENT_SECRET_LABEL) - & checkPropertyIsSet(collector, refreshToken, REFRESH_TOKEN, REFRESH_TOKEN_LABEL); + if (OAuthMethod.REFRESH_TOKEN.equals(getOAuthMethod())) { + return checkPropertyIsSet(collector, clientId, CLIENT_ID, CLIENT_ID_LABEL) + & checkPropertyIsSet(collector, clientSecret, CLIENT_SECRET, CLIENT_SECRET_LABEL) + & checkPropertyIsSet(collector, refreshToken, REFRESH_TOKEN, REFRESH_TOKEN_LABEL); + } else { + return checkPropertyIsSet(collector, accessToken, ACCESS_TOKEN, ACCESS_TOKEN_LABEL); + } } private boolean validateServiceAccount(FailureCollector collector) { @@ -206,27 +240,41 @@ private boolean validateServiceAccount(FailureCollector collector) { return collector.getValidationFailures().size() == 0; } - private void validateCredentials(FailureCollector collector, GoogleDriveClient driveClient) throws IOException { - try { - driveClient.checkRootFolder(); - } catch (GoogleJsonResponseException e) { - collector.addFailure(e.getDetails().getMessage(), "Provide valid credentials.") - .withConfigProperty(NAME_SERVICE_ACCOUNT_TYPE) - .withStacktrace(e.getStackTrace()); - } - } - - private void validateDirectoryIdentifier(FailureCollector collector, GoogleDriveClient driveClient) + private boolean isDirectoryOrFileAccessible(FailureCollector collector, GoogleDriveClient driveClient) throws IOException { - if (!containsMacro(DIRECTORY_IDENTIFIER)) { + + if (directoryIdentifier != null && !containsMacro(DIRECTORY_IDENTIFIER)) { try { driveClient.isFolderAccessible(directoryIdentifier); + return true; } catch (GoogleJsonResponseException e) { collector.addFailure(e.getDetails().getMessage(), "Provide an existing folder identifier.") .withConfigProperty(DIRECTORY_IDENTIFIER) .withStacktrace(e.getStackTrace()); } } + + if (fileIdentifier != null && !containsMacro(FILE_IDENTIFIER)) { + try { + driveClient.isFileAccessible(fileIdentifier); + return true; + } catch (GoogleJsonResponseException e) { + collector.addFailure(e.getDetails().getMessage(), "Provide an existing file identifier.") + .withConfigProperty(FILE_IDENTIFIER) + .withStacktrace(e.getStackTrace()); + } + } + return false; + } + + protected void checkIfDirectoryOrFileIdentifierExists(FailureCollector collector) { + if (directoryIdentifier == null && !containsMacro(DIRECTORY_IDENTIFIER) && + fileIdentifier == null && !containsMacro(FILE_IDENTIFIER)) { + collector.addFailure("Both Directory Identifier and File Identifier can not be null.", + "Provide either Directory Identifier or File Identifier.") + .withConfigProperty(DIRECTORY_IDENTIFIER) + .withConfigProperty(FILE_IDENTIFIER); + } } protected boolean checkPropertyIsSet(FailureCollector collector, String propertyValue, String propertyName, @@ -250,6 +298,10 @@ public String getDirectoryIdentifier() { return directoryIdentifier; } + public String getFileIdentifier() { + return fileIdentifier; + } + public AuthType getAuthType() { return AuthType.fromValue(authType); } @@ -277,6 +329,13 @@ public void setAccountFilePath(String accountFilePath) { public void setDirectoryIdentifier(String directoryIdentifier) { this.directoryIdentifier = directoryIdentifier; } + public void setFileIdentifier(String fileIdentifier) { + this.fileIdentifier = fileIdentifier; + } + + public void setOauthMethod(String oauthMethod) { + this.oauthMethod = oauthMethod; + } public void setClientId(String clientId) { this.clientId = clientId; @@ -290,6 +349,10 @@ public void setRefreshToken(String refreshToken) { this.refreshToken = refreshToken; } + public void setAccessToken(String accessToken) { + this.accessToken = accessToken; + } + @Nullable public String getClientId() { return clientId; @@ -305,6 +368,11 @@ public String getRefreshToken() { return refreshToken; } + @Nullable + public String getAccessToken() { + return accessToken; + } + @Nullable public String getServiceAccountFilePath() { if (containsMacro(ACCOUNT_FILE_PATH) || Strings.isNullOrEmpty(accountFilePath) @@ -344,4 +412,15 @@ public Boolean isServiceAccountFilePath() { String serviceAccountType = getServiceAccountType(); return Strings.isNullOrEmpty(serviceAccountType) ? null : serviceAccountType.equals(SERVICE_ACCOUNT_FILE_PATH); } + + public OAuthMethod getOAuthMethod() { + if (oauthMethod == null) { + return OAuthMethod.REFRESH_TOKEN; + } + try { + return OAuthMethod.valueOf(oauthMethod.toUpperCase()); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid oauth method " + oauthMethod); + } + } } diff --git a/src/main/java/io/cdap/plugin/google/common/GoogleDriveClient.java b/src/main/java/io/cdap/plugin/google/common/GoogleDriveClient.java index 50bb2043..d9235be8 100644 --- a/src/main/java/io/cdap/plugin/google/common/GoogleDriveClient.java +++ b/src/main/java/io/cdap/plugin/google/common/GoogleDriveClient.java @@ -43,7 +43,6 @@ */ public class GoogleDriveClient { private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance(); - private static final String ROOT_FOLDER_ID = "root"; protected final Drive service; protected final C config; protected NetHttpTransport httpTransport; @@ -88,13 +87,18 @@ protected Drive getDriveClient() throws IOException { } protected Credential getOAuth2Credential() { - GoogleCredential credential = new GoogleCredential.Builder() - .setTransport(httpTransport) - .setJsonFactory(JSON_FACTORY) - .setClientSecrets(config.getClientId(), - config.getClientSecret()) - .build(); - credential.createScoped(getRequiredScopes()).setRefreshToken(config.getRefreshToken()); + GoogleCredential credential; + GoogleCredential.Builder builder = new GoogleCredential.Builder() + .setTransport(httpTransport) + .setJsonFactory(JSON_FACTORY); + if (OAuthMethod.ACCESS_TOKEN.equals(config.getOAuthMethod())) { + credential = builder.build(); + credential.createScoped(getRequiredScopes()).setAccessToken(config.getAccessToken()); + } else { + credential = builder.setClientSecrets(config.getClientId(), + config.getClientSecret()).build(); + credential.createScoped(getRequiredScopes()).setRefreshToken(config.getRefreshToken()); + } return credential; } @@ -121,11 +125,11 @@ protected List getRequiredScopes() { return Collections.singletonList(DriveScopes.DRIVE_READONLY); } - public void checkRootFolder() throws IOException { - service.files().get(ROOT_FOLDER_ID).setSupportsAllDrives(true).execute(); - } - public void isFolderAccessible(String folderId) throws IOException { service.files().get(folderId).setSupportsAllDrives(true).execute(); } + + public void isFileAccessible(String fileId) throws IOException { + service.files().get(fileId).setSupportsAllDrives(true).execute(); + } } diff --git a/src/main/java/io/cdap/plugin/google/common/GoogleDriveFilteringClient.java b/src/main/java/io/cdap/plugin/google/common/GoogleDriveFilteringClient.java index 8a92ec55..ca4c3ce2 100644 --- a/src/main/java/io/cdap/plugin/google/common/GoogleDriveFilteringClient.java +++ b/src/main/java/io/cdap/plugin/google/common/GoogleDriveFilteringClient.java @@ -71,6 +71,10 @@ public List getFilesSummary(List exportedTypes, int filesNum String nextToken = ""; int retrievedFiles = 0; int actualFilesNumber = filesNumber; + if (config.getFileIdentifier() != null) { + files.add(service.files().get(config.getFileIdentifier()).setSupportsAllDrives(true).execute()); + return files; + } Drive.Files.List request = service.files().list() .setSupportsAllDrives(true) .setIncludeItemsFromAllDrives(true) diff --git a/src/main/java/io/cdap/plugin/google/common/OAuthMethod.java b/src/main/java/io/cdap/plugin/google/common/OAuthMethod.java new file mode 100644 index 00000000..f7a44121 --- /dev/null +++ b/src/main/java/io/cdap/plugin/google/common/OAuthMethod.java @@ -0,0 +1,24 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.google.common; + +/** + * Methods of getting an OAuth access token. + */ +public enum OAuthMethod { + ACCESS_TOKEN, + REFRESH_TOKEN; +} diff --git a/src/main/java/io/cdap/plugin/google/common/ValidationResult.java b/src/main/java/io/cdap/plugin/google/common/ValidationResult.java index 9b65421f..e6194a2f 100644 --- a/src/main/java/io/cdap/plugin/google/common/ValidationResult.java +++ b/src/main/java/io/cdap/plugin/google/common/ValidationResult.java @@ -20,13 +20,13 @@ * Wrapper to save validation results. Is used to transfer validation results. */ public class ValidationResult { - private boolean directoryAccessible = false; + private boolean directoryOrFileAccessible = false; - public boolean isDirectoryAccessible() { - return directoryAccessible; + public boolean isDirectoryOrFileAccessible() { + return directoryOrFileAccessible; } - public void setDirectoryAccessible(boolean directoryAccessible) { - this.directoryAccessible = directoryAccessible; + public void setDirectoryOrFileAccessible(boolean directoryOrFileAccessible) { + this.directoryOrFileAccessible = directoryOrFileAccessible; } } diff --git a/src/main/java/io/cdap/plugin/google/drive/source/GoogleDriveSourceConfig.java b/src/main/java/io/cdap/plugin/google/drive/source/GoogleDriveSourceConfig.java index cb4e2b6c..13d7dbbf 100644 --- a/src/main/java/io/cdap/plugin/google/drive/source/GoogleDriveSourceConfig.java +++ b/src/main/java/io/cdap/plugin/google/drive/source/GoogleDriveSourceConfig.java @@ -324,6 +324,10 @@ public static GoogleDriveSourceConfig of(JsonObject properties) throws IOExcepti googleDriveSourceConfig.setDirectoryIdentifier( properties.get(GoogleDriveSourceConfig.DIRECTORY_IDENTIFIER).getAsString()); } + if (properties.has(GoogleDriveSourceConfig.FILE_IDENTIFIER)) { + googleDriveSourceConfig.setFileIdentifier( + properties.get(GoogleDriveSourceConfig.FILE_IDENTIFIER).getAsString()); + } if (properties.has(GoogleDriveSourceConfig.FILTER)) { googleDriveSourceConfig.setFilter(properties.get(GoogleDriveSourceConfig.FILTER).getAsString()); } @@ -364,6 +368,12 @@ public static GoogleDriveSourceConfig of(JsonObject properties) throws IOExcepti if (properties.has(GoogleDriveSourceConfig.REFRESH_TOKEN)) { googleDriveSourceConfig.setRefreshToken(properties.get(GoogleDriveSourceConfig.REFRESH_TOKEN).getAsString()); } + if (properties.has(GoogleDriveSourceConfig.ACCESS_TOKEN)) { + googleDriveSourceConfig.setAccessToken(properties.get(GoogleDriveSourceConfig.ACCESS_TOKEN).getAsString()); + } + if (properties.has(GoogleDriveSourceConfig.OAUTH_METHOD)) { + googleDriveSourceConfig.setOauthMethod(properties.get(GoogleDriveSourceConfig.OAUTH_METHOD).getAsString()); + } return googleDriveSourceConfig; } } diff --git a/src/main/java/io/cdap/plugin/google/sheets/source/GoogleSheetsRecordReader.java b/src/main/java/io/cdap/plugin/google/sheets/source/GoogleSheetsRecordReader.java index 00bcf205..993845ca 100644 --- a/src/main/java/io/cdap/plugin/google/sheets/source/GoogleSheetsRecordReader.java +++ b/src/main/java/io/cdap/plugin/google/sheets/source/GoogleSheetsRecordReader.java @@ -17,6 +17,7 @@ package io.cdap.plugin.google.sheets.source; import com.github.rholder.retry.RetryException; +import com.google.api.services.sheets.v4.model.Sheet; import com.google.gson.reflect.TypeToken; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; @@ -89,48 +90,55 @@ public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptCont } private void populateBufferedTasks() { - int firstDataRow = config.getActualFirstDataRow(); - int lastDataRow = config.getActualLastDataRow(); - List sheetTitles; + List sheetTitles; try { sheetTitles = getSheetTitles(); } catch (ExecutionException | RetryException e) { throw new RuntimeException("Exception during sheet titles retrieving.", e); } sheetTitles.forEach(t -> { + int firstDataRow = config.getActualFirstDataRow(); + // Each sheet can have different number of records so last row can be different sheet wise + // and in case last data row is set to 0, It will fetch all records from the sheet + int lastDataRow = config.getActualLastDataRow(t.getProperties().getGridProperties().getRowCount()); int rowsNumber = lastDataRow - firstDataRow + 1; overallRowsNumber += rowsNumber; int counter = 0; + String title = t.getProperties().getTitle(); while (rowsNumber > bufferSize) { - rowTaskQueue.add(new GroupedRowTask(t, counter * bufferSize + firstDataRow, bufferSize)); + rowTaskQueue.add( + new GroupedRowTask(title, counter * bufferSize + firstDataRow, bufferSize)); counter++; rowsNumber -= bufferSize; } - rowTaskQueue.add(new GroupedRowTask(t, counter * bufferSize + firstDataRow, + rowTaskQueue.add(new GroupedRowTask(title, counter * bufferSize + firstDataRow, rowsNumber)); }); currentRowIndex = -1; currentGroupedRowTask = null; } - private List getSheetTitles() throws ExecutionException, RetryException { - List sheetTitles = new ArrayList<>(); + private List getSheetTitles() throws ExecutionException, RetryException { + List sheetList = new ArrayList<>(); switch (config.getSheetsToPull()) { case ALL: - sheetTitles = googleSheetsSourceClient.getSheetsTitles(fileId); + sheetList = googleSheetsSourceClient.getSheets(fileId); break; case NUMBERS: List sheetIndexes = config.getSheetsIdentifiers().stream() .map(s -> Integer.parseInt(s)).collect(Collectors.toList()); - sheetTitles = googleSheetsSourceClient.getSheets(fileId).stream() + sheetList = googleSheetsSourceClient.getSheets(fileId).stream() .filter(s -> sheetIndexes.contains(s.getProperties().getIndex())) - .map(s -> s.getProperties().getTitle()).collect(Collectors.toList()); + .collect(Collectors.toList()); break; case TITLES: - sheetTitles = config.getSheetsIdentifiers(); + List sheetTitles = config.getSheetsIdentifiers(); + sheetList = googleSheetsSourceClient.getSheets(fileId).stream() + .filter(s -> sheetTitles.contains(s.getProperties().getTitle())) + .collect(Collectors.toList()); break; } - return sheetTitles; + return sheetList; } @Override diff --git a/src/main/java/io/cdap/plugin/google/sheets/source/GoogleSheetsSourceConfig.java b/src/main/java/io/cdap/plugin/google/sheets/source/GoogleSheetsSourceConfig.java index af906d15..36ac502b 100644 --- a/src/main/java/io/cdap/plugin/google/sheets/source/GoogleSheetsSourceConfig.java +++ b/src/main/java/io/cdap/plugin/google/sheets/source/GoogleSheetsSourceConfig.java @@ -117,7 +117,7 @@ public class GoogleSheetsSourceConfig extends GoogleFilteringSourceConfig { @Name(NAME_SCHEMA) @Description("The schema of the table to read.") @Macro - private transient Schema schema = null; + private String schema; @Name(FORMATTING) @Description("Output format for numeric sheet cells. " + @@ -186,12 +186,12 @@ public class GoogleSheetsSourceConfig extends GoogleFilteringSourceConfig { @Name(LAST_DATA_COLUMN) @Description("Last column plugin will read as data. It will be ignored if the Column Names " + - "Row contain less number of columns.") + "Row contain less number of columns. Set it to 0 to read all the columns in the sheet.") @Macro private String lastDataColumn; @Name(LAST_DATA_ROW) - @Description("Last row plugin will read as data.") + @Description("Last row plugin will read as data. Set it to 0 to read all the rows in the sheet.") @Macro private String lastDataRow; @@ -267,22 +267,26 @@ public GoogleSheetsSourceConfig(String referenceName) { * @param collector throws validation exception */ public Schema getSchema(FailureCollector collector) { - if (schema == null && shouldGetSchema()) { + Schema existingSchema = this.getSchema(); + if (existingSchema == null && shouldGetSchema()) { if (dataSchemaInfo.isEmpty()) { collector.addFailure("There are no headers to process.", "Perhaps no validation step was executed before schema generation.") .withConfigProperty(SCHEMA); } - schema = SchemaBuilder.buildSchema(this, new ArrayList<>(dataSchemaInfo.values())); + existingSchema = SchemaBuilder.buildSchema(this, new ArrayList<>(dataSchemaInfo.values())); } - return schema; + return existingSchema; } private boolean shouldGetSchema() { return !containsMacro(SHEETS_TO_PULL) && !containsMacro(SHEETS_IDENTIFIERS) && !containsMacro(COLUMN_NAMES_SELECTION) && !containsMacro(CUSTOM_COLUMN_NAMES_ROW) && !containsMacro(LAST_DATA_COLUMN) && !containsMacro(NAME_SERVICE_ACCOUNT_TYPE) && - !containsMacro(ACCOUNT_FILE_PATH) && !containsMacro(NAME_SERVICE_ACCOUNT_JSON); + !containsMacro(ACCOUNT_FILE_PATH) && !containsMacro(NAME_SERVICE_ACCOUNT_JSON) && + !containsMacro(CLIENT_ID) && !containsMacro(CLIENT_SECRET) && + !containsMacro(REFRESH_TOKEN) && !containsMacro(FILE_IDENTIFIER) && + !containsMacro(DIRECTORY_IDENTIFIER) && !containsMacro(ACCESS_TOKEN); } /** @@ -300,7 +304,8 @@ public ValidationResult validate(FailureCollector collector) { validateLastDataColumnIndexAndLastRowIndex(collector); validateSpreadsheetAndSheetFieldNames(collector); - if (collector.getValidationFailures().isEmpty() && validationResult.isDirectoryAccessible()) { + if (collector.getValidationFailures().isEmpty() && + (validationResult.isDirectoryOrFileAccessible())) { GoogleDriveFilteringClient driveClient; GoogleSheetsSourceClient sheetsSourceClient; try { @@ -315,7 +320,7 @@ public ValidationResult validate(FailureCollector collector) { spreadsheetsFiles = driveClient .getFilesSummary(Collections.singletonList(ExportedType.SPREADSHEETS), 1); } catch (ExecutionException | RetryException e) { - collector.addFailure("Invalid search query, see https://developers.google.com/drive/api/v3/ref-search-terms", + collector.addFailure(String.format("Failed while getting file schema due to reason : %s", e.getMessage()), null).withStacktrace(e.getStackTrace()); return validationResult; } @@ -357,14 +362,14 @@ private void validateColumnNamesRow(FailureCollector collector) { private void validateLastDataColumnIndexAndLastRowIndex(FailureCollector collector) { if (!containsMacro(LAST_DATA_COLUMN)) { - if (getLastDataColumn() <= 0) { - collector.addFailure("Last Data Column Index should be greater than 0", + if (getLastDataColumn() < 0) { + collector.addFailure("Last Data Column Index should be equal or greater than 0", null).withConfigProperty(LAST_DATA_COLUMN); } } if (!containsMacro(LAST_DATA_ROW)) { - if (getLastDataRow() <= 0) { - collector.addFailure("Last Data Row Index should be greater than 0", + if (getLastDataRow() < 0) { + collector.addFailure("Last Data Row Index should be equal or greater than 0", null).withConfigProperty(LAST_DATA_ROW); } } @@ -579,6 +584,9 @@ private LinkedHashMap processColumns(List columnHeaders = new LinkedHashMap<>(); List headerTitles = new ArrayList<>(); + if (lastDataColumn == 0) { + lastDataColumn = columnsRow.size(); + } for (int i = 0; i < Math.min(columnsRow.size(), lastDataColumn); i++) { CellData columnHeaderCell = columnsRow.get(i); int index = i; @@ -691,8 +699,8 @@ public int getActualFirstDataRow() { * Returns the int. * @return The int */ - public int getActualLastDataRow() { - int lastDataRow = getLastDataRow(); + public int getActualLastDataRow(int recordsInSheet) { + int lastDataRow = getLastDataRow() == 0 ? recordsInSheet : getLastDataRow(); if (isExtractMetadata() && getFirstFooterRow() > 0) { lastDataRow = Math.min(lastDataRow, getFirstFooterRow() - 1); } @@ -909,11 +917,11 @@ public int getLastFooterRow() { } public Integer getLastDataColumn() { - return Integer.parseInt(lastDataColumn); + return lastDataColumn != null ? Integer.parseInt(lastDataColumn) : 0; } public Integer getLastDataRow() { - return Integer.parseInt(lastDataRow); + return lastDataRow != null ? Integer.parseInt(lastDataRow) : 0; } public String getMetadataCells() { @@ -1010,7 +1018,7 @@ public void setSheetsIdentifiers(String sheetsIdentifiers) { } public void setSchema(String schema) throws IOException { - this.schema = Schema.parseJson(schema); + this.schema = schema; } public void setFormatting(String formatting) { @@ -1257,7 +1265,27 @@ public static GoogleSheetsSourceConfig of(JsonObject properties) throws IOExcept googleSheetsSourceConfig.setDirectoryIdentifier( properties.get(GoogleSheetsSourceConfig.DIRECTORY_IDENTIFIER).getAsString()); } - + if (properties.has(GoogleSheetsSourceConfig.FILE_IDENTIFIER)) { + googleSheetsSourceConfig.setFileIdentifier( + properties.get(GoogleSheetsSourceConfig.FILE_IDENTIFIER).getAsString()); + } + if (properties.has(GoogleSheetsSourceConfig.OAUTH_METHOD)) { + googleSheetsSourceConfig.setOauthMethod( + properties.get(GoogleSheetsSourceConfig.OAUTH_METHOD).getAsString()); + } + if (properties.has(GoogleSheetsSourceConfig.ACCESS_TOKEN)) { + googleSheetsSourceConfig.setAccessToken( + properties.get(GoogleSheetsSourceConfig.ACCESS_TOKEN).getAsString()); + } return googleSheetsSourceConfig; } + @Nullable + public Schema getSchema() { + try { + return !this.containsMacro("schema") && + !Strings.isNullOrEmpty(this.schema) ? Schema.parseJson(this.schema) : null; + } catch (Exception var2) { + throw new IllegalArgumentException("Invalid schema: " + var2.getMessage(), var2); + } + } } diff --git a/src/test/java/io/cdap/plugin/google/common/GoogleAuthBaseConfigTest.java b/src/test/java/io/cdap/plugin/google/common/GoogleAuthBaseConfigTest.java index 4b452f2b..09faf5fe 100644 --- a/src/test/java/io/cdap/plugin/google/common/GoogleAuthBaseConfigTest.java +++ b/src/test/java/io/cdap/plugin/google/common/GoogleAuthBaseConfigTest.java @@ -57,6 +57,7 @@ public void testValidationErrorFilePath() { config.setModificationDateRange("today"); config.getBodyFormat("string"); config.setStartDate("today"); + config.setFileIdentifier("fileId"); FailureCollector collector = new DefaultFailureCollector("stageConfig", Collections.EMPTY_MAP); config.validate(collector); Assert.assertEquals(1, collector.getValidationFailures().size()); @@ -76,6 +77,7 @@ public void testValidationErrorJSON() { config.setModificationDateRange("today"); config.getBodyFormat("string"); config.setStartDate("today"); + config.setFileIdentifier("fileId"); FailureCollector collector = new DefaultFailureCollector("stageConfig", Collections.EMPTY_MAP); config.validate(collector); Assert.assertEquals(1, collector.getValidationFailures().size()); @@ -84,4 +86,71 @@ public void testValidationErrorJSON() { Assert.assertEquals("serviceAccountJSON", collector.getValidationFailures().get(0).getCauses().get(0).getAttribute("stageConfig")); } + + @Test + public void testWithBothFileAndDirectoryAsNull() { + GoogleDriveSourceConfig config = new GoogleDriveSourceConfig("ref"); + config.setReferenceName("validationErrorFilePath"); + config.setAuthType("oAuth2"); + config.setModificationDateRange("today"); + config.getBodyFormat("string"); + config.setStartDate("today"); + config.setAccessToken("access"); + config.setOauthMethod(OAuthMethod.ACCESS_TOKEN.name()); + FailureCollector collector = new DefaultFailureCollector("stageConfig", Collections.EMPTY_MAP); + config.validate(collector); + Assert.assertEquals(1, collector.getValidationFailures().size()); + Assert.assertEquals("Both Directory Identifier and File Identifier can not be null.", + collector.getValidationFailures().get(0).getMessage()); + Assert.assertEquals("directoryIdentifier", + collector.getValidationFailures().get(0).getCauses().get(0).getAttribute("stageConfig")); + Assert.assertEquals("fileIdentifier", + collector.getValidationFailures().get(0).getCauses().get(1).getAttribute("stageConfig")); + } + + @Test + public void testValidationOauthWithoutAccessToken() { + GoogleDriveSourceConfig config = new GoogleDriveSourceConfig("ref"); + config.setReferenceName("validationErrorFilePath"); + config.setAuthType("oAuth2"); + config.setOauthMethod(OAuthMethod.ACCESS_TOKEN.name()); + config.setModificationDateRange("today"); + config.getBodyFormat("string"); + config.setStartDate("today"); + config.setFileIdentifier("file"); + FailureCollector collector = new DefaultFailureCollector("stageConfig", Collections.EMPTY_MAP); + config.validate(collector); + Assert.assertEquals(1, collector.getValidationFailures().size()); + Assert.assertEquals("'Access token' property is empty or macro is not available.", + collector.getValidationFailures().get(0).getMessage()); + Assert.assertEquals("accessToken", + collector.getValidationFailures().get(0).getCauses().get(0).getAttribute("stageConfig")); + } + + @Test + public void testValidationOauthWithoutRefreshToken() { + GoogleDriveSourceConfig config = new GoogleDriveSourceConfig(null); + config.setReferenceName("validationErrorFilePath"); + config.setAuthType("oAuth2"); + config.setOauthMethod(OAuthMethod.REFRESH_TOKEN.name()); + config.setModificationDateRange("today"); + config.getBodyFormat("string"); + config.setStartDate("today"); + config.setFileIdentifier("file"); + FailureCollector collector = new DefaultFailureCollector("stageConfig", Collections.EMPTY_MAP); + config.validate(collector); + Assert.assertEquals(3, collector.getValidationFailures().size()); + Assert.assertEquals("'Client ID' property is empty or macro is not available.", + collector.getValidationFailures().get(0).getMessage()); + Assert.assertEquals("'Client secret' property is empty or macro is not available.", + collector.getValidationFailures().get(1).getMessage()); + Assert.assertEquals("'Refresh token' property is empty or macro is not available.", + collector.getValidationFailures().get(2).getMessage()); + Assert.assertEquals("clientId", + collector.getValidationFailures().get(0).getCauses().get(0).getAttribute("stageConfig")); + Assert.assertEquals("clientSecret", + collector.getValidationFailures().get(1).getCauses().get(0).getAttribute("stageConfig")); + Assert.assertEquals("refreshToken", + collector.getValidationFailures().get(2).getCauses().get(0).getAttribute("stageConfig")); + } } diff --git a/src/test/java/io/cdap/plugin/google/sheets/sink/GoogleSheetsSinkClientTest.java b/src/test/java/io/cdap/plugin/google/sheets/sink/GoogleSheetsSinkClientTest.java index 870d08b3..67d327d6 100644 --- a/src/test/java/io/cdap/plugin/google/sheets/sink/GoogleSheetsSinkClientTest.java +++ b/src/test/java/io/cdap/plugin/google/sheets/sink/GoogleSheetsSinkClientTest.java @@ -21,6 +21,7 @@ import com.google.api.services.sheets.v4.model.MergeCellsRequest; import com.google.api.services.sheets.v4.model.RowData; import io.cdap.plugin.google.common.AuthType; +import io.cdap.plugin.google.common.OAuthMethod; import io.cdap.plugin.google.sheets.sink.utils.ComplexHeader; import io.cdap.plugin.google.sheets.sink.utils.FlatteredRowsRecord; import io.cdap.plugin.google.sheets.sink.utils.FlatteredRowsRequest; @@ -46,6 +47,7 @@ public class GoogleSheetsSinkClientTest { public static void setupClient() throws IOException { sinkConfig = EasyMock.createMock(GoogleSheetsSinkConfig.class); EasyMock.expect(sinkConfig.getAuthType()).andReturn(AuthType.OAUTH2).anyTimes(); + EasyMock.expect(sinkConfig.getOAuthMethod()).andReturn(OAuthMethod.REFRESH_TOKEN).anyTimes(); EasyMock.expect(sinkConfig.getRefreshToken()).andReturn("dsfdsfdsfdsf").anyTimes(); EasyMock.expect(sinkConfig.getClientSecret()).andReturn("dsfdsfdsfdsf").anyTimes(); EasyMock.expect(sinkConfig.getClientId()).andReturn("dsdsrfegvrb").anyTimes(); diff --git a/widgets/GoogleDrive-batchsink.json b/widgets/GoogleDrive-batchsink.json index e4f0f075..2311105d 100644 --- a/widgets/GoogleDrive-batchsink.json +++ b/widgets/GoogleDrive-batchsink.json @@ -31,6 +31,11 @@ "widget-type": "textbox", "label": "Directory Identifier", "name": "directoryIdentifier" + }, + { + "widget-type": "hidden", + "label": "File Identifier", + "name": "fileIdentifier" } ] }, @@ -56,6 +61,33 @@ ] } }, + { + "name": "oauthMethod", + "label": "OAuth Method", + "widget-type": "radio-group", + "widget-attributes": { + "layout": "inline", + "default": "REFRESH_TOKEN", + "options": [ + { + "id": "REFRESH_TOKEN", + "label": "Refresh Token" + }, + { + "id": "ACCESS_TOKEN", + "label": "Access Token" + } + ] + } + }, + { + "name": "accessToken", + "label": "Access Token", + "widget-type": "textbox", + "widget-attributes": { + "placeholder": "${oauthAccessToken(provider,credential)}" + } + }, { "widget-type": "textbox", "label": "Client ID", @@ -115,6 +147,30 @@ "operator": "equal to", "value": "oAuth2" }, + "show": [ + { + "name": "oauthMethod", + "type": "property" + } + ] + }, + { + "name": "Authenticate with OAuth2 Access Token", + "condition": { + "expression": "oauthMethod == 'ACCESS_TOKEN' && authType == 'oAuth2'" + }, + "show": [ + { + "name": "accessToken", + "type": "property" + } + ] + }, + { + "name": "Authenticate with OAuth2 Refresh Token", + "condition": { + "expression": "oauthMethod == 'REFRESH_TOKEN' && authType == 'oAuth2'" + }, "show": [ { "name": "clientId", diff --git a/widgets/GoogleDrive-batchsource.json b/widgets/GoogleDrive-batchsource.json index 2917353b..a8f5e6c6 100644 --- a/widgets/GoogleDrive-batchsource.json +++ b/widgets/GoogleDrive-batchsource.json @@ -17,6 +17,11 @@ "label": "Directory Identifier", "name": "directoryIdentifier" }, + { + "widget-type": "textbox", + "label": "File Identifier", + "name": "fileIdentifier" + }, { "name": "fileMetadataProperties", "widget-type": "multi-select", @@ -311,6 +316,33 @@ ] } }, + { + "name": "oauthMethod", + "label": "OAuth Method", + "widget-type": "radio-group", + "widget-attributes": { + "layout": "inline", + "default": "REFRESH_TOKEN", + "options": [ + { + "id": "REFRESH_TOKEN", + "label": "Refresh Token" + }, + { + "id": "ACCESS_TOKEN", + "label": "Access Token" + } + ] + } + }, + { + "name": "accessToken", + "label": "Access Token", + "widget-type": "textbox", + "widget-attributes": { + "placeholder": "${oauthAccessToken(provider,credential)}" + } + }, { "widget-type": "textbox", "label": "Client ID", @@ -488,6 +520,30 @@ "operator": "equal to", "value": "oAuth2" }, + "show": [ + { + "name": "oauthMethod", + "type": "property" + } + ] + }, + { + "name": "Authenticate with OAuth2 Access Token", + "condition": { + "expression": "oauthMethod == 'ACCESS_TOKEN' && authType == 'oAuth2'" + }, + "show": [ + { + "name": "accessToken", + "type": "property" + } + ] + }, + { + "name": "Authenticate with OAuth2 Refresh Token", + "condition": { + "expression": "oauthMethod == 'REFRESH_TOKEN' && authType == 'oAuth2'" + }, "show": [ { "name": "clientId", diff --git a/widgets/GoogleSheets-batchsink.json b/widgets/GoogleSheets-batchsink.json index 2b1a112e..7a01cab0 100644 --- a/widgets/GoogleSheets-batchsink.json +++ b/widgets/GoogleSheets-batchsink.json @@ -17,6 +17,11 @@ "label": "Directory Identifier", "name": "directoryIdentifier" }, + { + "widget-type": "hidden", + "label": "File Identifier", + "name": "fileIdentifier" + }, { "widget-type": "textbox", "label": "Spreadsheet Name Field", @@ -115,6 +120,33 @@ "label": "Service Account JSON", "name": "serviceAccountJSON" }, + { + "name": "oauthMethod", + "label": "OAuth Method", + "widget-type": "radio-group", + "widget-attributes": { + "layout": "inline", + "default": "REFRESH_TOKEN", + "options": [ + { + "id": "REFRESH_TOKEN", + "label": "Refresh Token" + }, + { + "id": "ACCESS_TOKEN", + "label": "Access Token" + } + ] + } + }, + { + "name": "accessToken", + "label": "Access Token", + "widget-type": "textbox", + "widget-attributes": { + "placeholder": "${oauthAccessToken(provider,credential)}" + } + }, { "widget-type": "textbox", "label": "Client ID", @@ -234,7 +266,33 @@ { "name": "Authenticate with OAuth2", "condition": { - "expression": "authType == 'oAuth2'" + "property": "authType", + "operator": "equal to", + "value": "oAuth2" + }, + "show": [ + { + "name": "oauthMethod", + "type": "property" + } + ] + }, + { + "name": "Authenticate with OAuth2 Access Token", + "condition": { + "expression": "oauthMethod == 'ACCESS_TOKEN' && authType == 'oAuth2'" + }, + "show": [ + { + "name": "accessToken", + "type": "property" + } + ] + }, + { + "name": "Authenticate with OAuth2 Refresh Token", + "condition": { + "expression": "oauthMethod == 'REFRESH_TOKEN' && authType == 'oAuth2'" }, "show": [ { diff --git a/widgets/GoogleSheets-batchsource.json b/widgets/GoogleSheets-batchsource.json index ceebdc1b..e98ec48b 100644 --- a/widgets/GoogleSheets-batchsource.json +++ b/widgets/GoogleSheets-batchsource.json @@ -16,6 +16,11 @@ "widget-type": "textbox", "label": "Directory Identifier", "name": "directoryIdentifier" + }, + { + "widget-type": "textbox", + "label": "File Identifier", + "name": "fileIdentifier" } ] }, @@ -143,6 +148,33 @@ "label": "Service Account JSON", "name": "serviceAccountJSON" }, + { + "name": "oauthMethod", + "label": "OAuth Method", + "widget-type": "radio-group", + "widget-attributes": { + "layout": "inline", + "default": "REFRESH_TOKEN", + "options": [ + { + "id": "REFRESH_TOKEN", + "label": "Refresh Token" + }, + { + "id": "ACCESS_TOKEN", + "label": "Access Token" + } + ] + } + }, + { + "name": "accessToken", + "label": "Access Token", + "widget-type": "textbox", + "widget-attributes": { + "placeholder": "${oauthAccessToken(provider,credential)}" + } + }, { "widget-type": "textbox", "label": "Client ID", @@ -341,7 +373,7 @@ "label": "Number of Columns to Read", "name": "lastDataColumn", "widget-attributes": { - "default": "26", + "default": "0", "min": "0" } }, @@ -350,7 +382,7 @@ "label": "Number of Rows to Read", "name": "lastDataRow", "widget-attributes": { - "default": "1000", + "default": "0", "min": "0" } }, @@ -366,7 +398,12 @@ ] } ], - "outputs": [], + "outputs": [ + { + "name": "schema", + "widget-type": "schema" + } + ], "filters": [ { "name": "Select modification date range", @@ -405,6 +442,30 @@ "operator": "equal to", "value": "oAuth2" }, + "show": [ + { + "name": "oauthMethod", + "type": "property" + } + ] + }, + { + "name": "Authenticate with OAuth2 Access Token", + "condition": { + "expression": "oauthMethod == 'ACCESS_TOKEN' && authType == 'oAuth2'" + }, + "show": [ + { + "name": "accessToken", + "type": "property" + } + ] + }, + { + "name": "Authenticate with OAuth2 Refresh Token", + "condition": { + "expression": "oauthMethod == 'REFRESH_TOKEN' && authType == 'oAuth2'" + }, "show": [ { "name": "clientId",