diff --git a/integ-test/build.gradle b/integ-test/build.gradle index b914cb0cbe..1a2c2dee31 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -213,6 +213,7 @@ dependencies { exclude group: 'org.hamcrest', module: 'hamcrest-core' } testImplementation('org.junit.jupiter:junit-jupiter-api:5.9.3') + testImplementation('org.junit.jupiter:junit-jupiter-params:5.9.3') testRuntimeOnly('org.junit.jupiter:junit-jupiter-engine:5.9.3') testRuntimeOnly('org.junit.platform:junit-platform-launcher:1.9.3') diff --git a/integ-test/src/test/java/org/opensearch/sql/security/FGACIndexScanningIT.java b/integ-test/src/test/java/org/opensearch/sql/security/FGACIndexScanningIT.java new file mode 100644 index 0000000000..3bbf3937e5 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/security/FGACIndexScanningIT.java @@ -0,0 +1,666 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.security; + +import static org.opensearch.sql.util.MatcherUtils.columnName; +import static org.opensearch.sql.util.MatcherUtils.verifyColumn; + +import java.io.IOException; +import java.util.Locale; +import lombok.SneakyThrows; +import org.apache.hc.core5.http.ParseException; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.jetbrains.annotations.NotNull; +import org.json.JSONObject; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.opensearch.client.Request; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; + +/** + * Integration tests for Fine-Grained Access Control (FGAC) across indices. + * + *

These tests verify all three levels of access control: 1. Index-level: Can users access the + * index? 2. Column-level (Field-level): Can users see specific fields? 3. Row-level + * (Document-level): Can users see specific documents? + */ +@TestInstance(Lifecycle.PER_CLASS) +public class FGACIndexScanningIT extends SecurityTestBase { + private static final String PUBLIC_USER = "public_user"; + private static final String PUBLIC_ROLE = "public_role"; + private static final String LIMITED_USER = "limited_user"; + private static final String LIMITED_ROLE = "limited_role"; + private static final String SENSITIVE_USER = "sensitive_user"; + private static final String SENSITIVE_ROLE = "sensitive_role"; + private static final String MANAGER_USER = "manager_user"; + private static final String MANAGER_ROLE = "manager_role"; + private static final String HR_USER = "hr_user"; + private static final String HR_ROLE = "hr_role"; + private static final String[] RECORDS_INDEX_COLUMNS = { + "name", "department", "salary", "email", "employee_id" + }; + + // Indices for testing + private static final String PUBLIC_LOGS = "public_logs_fgac"; + private static final String SENSITIVE_LOGS = "sensitive_logs_fgac"; + private static final String SECURE_LOGS = "secure_logs_fgac"; + private static final String EMPLOYEE_RECORDS = "employee_records_fgac"; + + private static final int LARGE_DATASET_SIZE = 100; + + @SneakyThrows + @BeforeAll + public void initialize() { + setUpIndices(); + setupTestIndices(); + createSecurityRolesAndUsers(); + } + + @Override + protected void init() throws Exception { + super.init(); + allowCalciteFallback(); + } + + /** + * Configures the query engine for the test. + * + * @param useCalcite true to use V3 (Calcite) engine, false to use V2 (legacy) engine + */ + private void configureEngine(boolean useCalcite) throws IOException { + if (useCalcite) { + enableCalcite(); + } else { + disableCalcite(); + } + } + + private void setupTestIndices() throws IOException, ParseException { + createPublicLogsIndex(); + createSensitiveLogsIndex(); + createEmployeeRecordsIndex(); + createSecureLogsIndex(); + } + + /** Creates public_logs index with test documents. */ + private void createPublicLogsIndex() throws IOException, ParseException { + Request request = new Request("PUT", "/" + PUBLIC_LOGS); + request.setJsonEntity( + """ + { + "settings": { + "number_of_shards": 1, + "number_of_replicas": 0 + }, + "mappings": { + "properties": { + "message": { "type": "text" }, + "level": { "type": "keyword" }, + "timestamp": { "type": "date" } + } + } + } + """); + client().performRequest(request); + + bulkInsertDocs(PUBLIC_LOGS, "public"); + } + + /** Creates sensitive_logs index with test documents. */ + private void createSensitiveLogsIndex() throws IOException, ParseException { + Request request = new Request("PUT", "/" + SENSITIVE_LOGS); + request.setJsonEntity( + """ + { + "settings": { + "number_of_shards": 1, + "number_of_replicas": 0 + }, + "mappings": { + "properties": { + "message": { "type": "text" }, + "level": { "type": "keyword" }, + "timestamp": { "type": "date" } + } + } + } + """); + client().performRequest(request); + + bulkInsertDocs(SENSITIVE_LOGS, "sensitive"); + } + + /** + * Creates employee_records index with sensitive fields for field-level security testing. Contains + * fields: employee_id, name, department, salary, ssn + */ + private void createEmployeeRecordsIndex() throws IOException, ParseException { + Request request = new Request("PUT", "/" + EMPLOYEE_RECORDS); + request.setJsonEntity( + """ + { + "settings": { + "number_of_shards": 1, + "number_of_replicas": 0 + }, + "mappings": { + "properties": { + "employee_id": { "type": "keyword" }, + "name": { "type": "text" }, + "department": { "type": "keyword" }, + "salary": { "type": "integer" }, + "ssn": { "type": "keyword" }, + "email": { "type": "keyword" } + } + } + } + """); + client().performRequest(request); + + bulkInsertEmployeeRecords(); + } + + /** + * Creates secure_logs index with mixed security levels. This index contains documents with + * different security_level values to test row-level filtering. + */ + private void createSecureLogsIndex() throws IOException, ParseException { + Request request = new Request("PUT", "/" + SECURE_LOGS); + request.setJsonEntity( + """ + { + "settings": { + "number_of_shards": 1, + "number_of_replicas": 0 + }, + "mappings": { + "properties": { + "message": { "type": "text" }, + "security_level": { "type": "keyword" }, + "timestamp": { "type": "date" } + } + } + } + """); + client().performRequest(request); + + bulkInsertDocsWithSecurityLevel(); + } + + /** Bulk inserts documents to trigger background scanning. */ + private void bulkInsertDocs(String indexName, String prefix) throws IOException, ParseException { + StringBuilder bulk = new StringBuilder(); + for (int i = 0; i < FGACIndexScanningIT.LARGE_DATASET_SIZE; i++) { + bulk.append( + String.format( + Locale.ROOT, + """ + { "index": { "_index": "%s" } } + { "message": "%s message %d", "level": "info", "timestamp": "2025-01-01T00:00:00Z" } + """, + indexName, + prefix, + i)); + } + + Request request = new Request("POST", "/_bulk"); + request.addParameter("refresh", "true"); + request.setJsonEntity(bulk.toString()); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/x-ndjson"); + request.setOptions(restOptionsBuilder); + + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + String body = EntityUtils.toString(response.getEntity()); + JSONObject json = new JSONObject(body); + assertFalse("Bulk indexing reported errors: " + body, json.getBoolean("errors")); + } + + /** Bulk inserts employee records with sensitive fields for FLS testing. */ + private void bulkInsertEmployeeRecords() throws IOException, ParseException { + String bulk = getBulkEmployeeIndexRequest(); + + Request request = new Request("POST", "/_bulk"); + request.addParameter("refresh", "true"); + request.setJsonEntity(bulk); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/x-ndjson"); + request.setOptions(restOptionsBuilder); + + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + String body = EntityUtils.toString(response.getEntity()); + JSONObject json = new JSONObject(body); + assertFalse("Bulk indexing reported errors: " + body, json.getBoolean("errors")); + } + + @NotNull + private static String getBulkEmployeeIndexRequest() { + StringBuilder bulk = new StringBuilder(); + String[] departments = {"engineering", "finance", "hr", "sales", "marketing"}; + + for (int i = 0; i < LARGE_DATASET_SIZE; i++) { + String dept = departments[i % departments.length]; + bulk.append( + String.format( + Locale.ROOT, + """ + { "index": { "_index": "%s" } } + { "employee_id": "EMP%04d", "name": "Employee %d", "department": "%s", "salary": %d, "ssn": "XXX-XX-%04d", "email": "emp%d@company.com" } + """, + FGACIndexScanningIT.EMPLOYEE_RECORDS, + i, + i, + dept, + 50000 + (i * 1000), + i, + i)); + } + return bulk.toString(); + } + + /** Bulk inserts documents with different security levels for row-level testing. */ + private void bulkInsertDocsWithSecurityLevel() throws IOException, ParseException { + StringBuilder bulk = new StringBuilder(); + + int publicCount = LARGE_DATASET_SIZE / 2; + int internalStart = publicCount; + int internalCount = LARGE_DATASET_SIZE / 4; + int confidentialStart = internalStart + internalCount; + + for (int i = 0; i < publicCount; i++) { + bulk.append( + String.format( + Locale.ROOT, + """ + { "index": { "_index": "%s" } } + { "message": "public message %d", "security_level": "public", "timestamp": "2025-01-01T00:00:00Z" } + """, + FGACIndexScanningIT.SECURE_LOGS, + i)); + } + + for (int i = internalStart; i < confidentialStart; i++) { + bulk.append( + String.format( + Locale.ROOT, + """ + { "index": { "_index": "%s" } } + { "message": "internal message %d", "security_level": "internal", "timestamp": "2025-01-01T00:00:00Z" } + """, + FGACIndexScanningIT.SECURE_LOGS, + i)); + } + + for (int i = confidentialStart; i < LARGE_DATASET_SIZE; i++) { + bulk.append( + String.format( + Locale.ROOT, + """ + { "index": { "_index": "%s" } } + { "message": "confidential message %d", "security_level": "confidential", "timestamp": "2025-01-01T00:00:00Z" } + """, + FGACIndexScanningIT.SECURE_LOGS, + i)); + } + + // Add document with null security_level to test DLS behavior with null values + bulk.append( + String.format( + Locale.ROOT, + """ + { "index": { "_index": "%s" } } + { "message": "null security level", "security_level": null, "timestamp": "2025-01-01T00:00:00Z" } + """, + FGACIndexScanningIT.SECURE_LOGS)); + + // Add document with missing security_level field to test DLS behavior with missing fields + bulk.append( + String.format( + Locale.ROOT, + """ + { "index": { "_index": "%s" } } + { "message": "missing security level", "timestamp": "2025-01-01T00:00:00Z" } + """, + FGACIndexScanningIT.SECURE_LOGS)); + + Request request = new Request("POST", "/_bulk"); + request.addParameter("refresh", "true"); + request.setJsonEntity(bulk.toString()); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/x-ndjson"); + request.setOptions(restOptionsBuilder); + + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + String body = EntityUtils.toString(response.getEntity()); + JSONObject json = new JSONObject(body); + assertFalse("Bulk indexing reported errors: " + body, json.getBoolean("errors")); + } + + /** Creates security roles and users for testing. */ + private void createSecurityRolesAndUsers() throws IOException { + // Role for public_user: can only access PUBLIC_LOGS + createRoleWithIndexAccess(PUBLIC_ROLE, PUBLIC_LOGS); + createUser(PUBLIC_USER, PUBLIC_ROLE); + + // Role for sensitive_user: can only access SENSITIVE_LOGS + createRoleWithIndexAccess(SENSITIVE_ROLE, SENSITIVE_LOGS); + createUser(SENSITIVE_USER, SENSITIVE_ROLE); + + // Role for limited_user: can access SECURE_LOGS but with document-level filtering + // Only allow documents with security_level="public" + createRoleWithDocumentLevelSecurity(); + createUser(LIMITED_USER, LIMITED_ROLE); + + // Roles for Scenario 2: Field-level security + // manager_user: can see name, department, salary, email BUT NOT ssn + createRoleWithFieldLevelSecurity(); + createUser(MANAGER_USER, MANAGER_ROLE); + + // hr_user: can see ALL fields including ssn + createRoleWithIndexAccess(HR_ROLE, EMPLOYEE_RECORDS); + createUser(HR_USER, HR_ROLE); + } + + /** + * Creates a role with document-level security (DLS) - only documents matching the query are + * visible. + */ + private void createRoleWithDocumentLevelSecurity() throws IOException { + createRoleWithDLS( + LIMITED_ROLE, SECURE_LOGS, "{\\\"match\\\":{\\\"security_level\\\":\\\"public\\\"}}"); + } + + /** Creates a role with field-level security (FLS) - only specific fields are accessible. */ + private void createRoleWithFieldLevelSecurity() throws IOException { + createRoleWithFLS(MANAGER_ROLE, EMPLOYEE_RECORDS, RECORDS_INDEX_COLUMNS); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testPublicUserCanAccessPublicLogs(boolean useCalcite) throws IOException { + configureEngine(useCalcite); + // public_user can access public_logs + JSONObject result = + executeQueryAsUser( + String.format("search source=%s | fields message | head 10", PUBLIC_LOGS), PUBLIC_USER); + verifyColumn(result, columnName("message")); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testPublicUserCannotAccessSensitiveLogs(boolean useCalcite) throws IOException { + configureEngine(useCalcite); + // public_user cannot access sensitive_logs (should fail at planning stage) + try { + executeQueryAsUser( + String.format("search source=%s | fields message", SENSITIVE_LOGS), PUBLIC_USER); + fail("Expected security exception when public_user accesses sensitive_logs"); + } catch (ResponseException e) { + String responseBody = + org.opensearch.sql.legacy.TestUtils.getResponseBody(e.getResponse(), false); + assertTrue( + "Response should contain permission error", + responseBody.contains("no permissions") || responseBody.contains("Forbidden")); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testSensitiveUserCanAccessSensitiveLogs(boolean useCalcite) throws IOException { + configureEngine(useCalcite); + // sensitive_user can access sensitive_logs + JSONObject result = + executeQueryAsUser( + String.format("search source=%s | fields message | head 10", SENSITIVE_LOGS), + SENSITIVE_USER); + verifyColumn(result, columnName("message")); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testSensitiveUserCannotAccessPublicLogs(boolean useCalcite) throws IOException { + configureEngine(useCalcite); + // sensitive_user cannot access public_logs + try { + executeQueryAsUser( + String.format("search source=%s | fields message", PUBLIC_LOGS), SENSITIVE_USER); + fail("Expected security exception when sensitive_user accesses public_logs"); + } catch (ResponseException e) { + String responseBody = + org.opensearch.sql.legacy.TestUtils.getResponseBody(e.getResponse(), false); + assertTrue( + "Response should contain permission error", + responseBody.contains("no permissions") || responseBody.contains("Forbidden")); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testHrUserCanSeeAllFieldsIncludingSensitiveData(boolean useCalcite) + throws IOException { + configureEngine(useCalcite); + // hr_user can see ALL fields including sensitive ssn + String queryAllFields = + String.format( + "search source=%s | fields name, department, salary, ssn | head 10", EMPLOYEE_RECORDS); + JSONObject hrResult = executeQueryAsUser(queryAllFields, HR_USER); + + var hrSchema = hrResult.getJSONArray("schema"); + boolean hrHasName = false, hrHasSalary = false, hrHasSSN = false, hrHasDepartment = false; + + for (int i = 0; i < hrSchema.length(); i++) { + String fieldName = hrSchema.getJSONObject(i).getString("name"); + if ("name".equals(fieldName)) hrHasName = true; + if ("salary".equals(fieldName)) hrHasSalary = true; + if ("ssn".equals(fieldName)) hrHasSSN = true; + if ("department".equals(fieldName)) hrHasDepartment = true; + } + + assertTrue("hr_user should see 'name' field", hrHasName); + assertTrue("hr_user should see 'salary' field", hrHasSalary); + assertTrue("hr_user should see 'ssn' field (sensitive)", hrHasSSN); + assertTrue("hr_user should see 'department' field", hrHasDepartment); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testManagerUserCannotSeeSensitiveFields(boolean useCalcite) throws IOException { + configureEngine(useCalcite); + // manager_user can see most fields but NOT ssn + String queryAllowedFields = + String.format( + "search source=%s | fields name, department, salary | head 10", EMPLOYEE_RECORDS); + JSONObject managerResult = executeQueryAsUser(queryAllowedFields, MANAGER_USER); + + var managerSchema = managerResult.getJSONArray("schema"); + boolean managerHasName = false, + managerHasSalary = false, + managerHasSSN = false, + managerHasDepartment = false; + + for (int i = 0; i < managerSchema.length(); i++) { + String fieldName = managerSchema.getJSONObject(i).getString("name"); + if ("name".equals(fieldName)) managerHasName = true; + if ("salary".equals(fieldName)) managerHasSalary = true; + if ("ssn".equals(fieldName)) managerHasSSN = true; + if ("department".equals(fieldName)) managerHasDepartment = true; + } + + assertTrue("manager_user should see 'name' field", managerHasName); + assertTrue("manager_user should see 'salary' field", managerHasSalary); + assertTrue("manager_user should see 'department' field", managerHasDepartment); + assertFalse( + "SECURITY VIOLATION: manager_user should NOT see 'ssn' field. " + + "Field-level security should hide this sensitive field.", + managerHasSSN); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testManagerUserCannotQueryRestrictedField(boolean useCalcite) throws IOException { + configureEngine(useCalcite); + // Verify manager_user cannot even reference ssn in query (field is invisible) + try { + String queryWithSSN = + String.format("search source=%s | fields ssn | head 10", EMPLOYEE_RECORDS); + executeQueryAsUser(queryWithSSN, MANAGER_USER); + fail( + "SECURITY VIOLATION: manager_user should NOT be able to query 'ssn' field. " + + "Query should fail because field is invisible to this user."); + } catch (ResponseException e) { + String responseBody = + org.opensearch.sql.legacy.TestUtils.getResponseBody(e.getResponse(), false); + assertTrue( + "Error should indicate field not found", + responseBody.contains("Field [ssn] not found") || responseBody.contains("ssn")); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testFieldLevelSecurityEnforcedWithLargeDataset(boolean useCalcite) + throws IOException { + configureEngine(useCalcite); + // Verify with large result set that FLS is still enforced + // Query all data (not just count) to actually exercise FLS at scale + String queryLargeDataset = + String.format( + "search source=%s | fields name, salary, department, employee_id", EMPLOYEE_RECORDS); + JSONObject managerLargeResult = executeQueryAsUser(queryLargeDataset, MANAGER_USER); + + // Verify the schema contains allowed fields but not restricted fields + var largeSchema = managerLargeResult.getJSONArray("schema"); + boolean hasName = false; + boolean hasSalary = false; + boolean hasDepartment = false; + boolean hasEmployeeId = false; + boolean hasSSN = false; + + for (int i = 0; i < largeSchema.length(); i++) { + String fieldName = largeSchema.getJSONObject(i).getString("name"); + if ("name".equals(fieldName)) hasName = true; + if ("salary".equals(fieldName)) hasSalary = true; + if ("department".equals(fieldName)) hasDepartment = true; + if ("employee_id".equals(fieldName)) hasEmployeeId = true; + if ("ssn".equals(fieldName)) hasSSN = true; + } + + // Verify allowed fields are present + assertTrue("manager_user should see 'name' field in large dataset query", hasName); + assertTrue("manager_user should see 'salary' field in large dataset query", hasSalary); + assertTrue("manager_user should see 'department' field in large dataset query", hasDepartment); + assertTrue("manager_user should see 'employee_id' field in large dataset query", hasEmployeeId); + + // Verify restricted field is NOT present + assertFalse( + "SECURITY VIOLATION: manager_user should NOT see 'ssn' even with large dataset. " + + "Field-level security must be enforced.", + hasSSN); + + // Verify we actually got data back (not just an empty result) + var datarows = managerLargeResult.getJSONArray("datarows"); + assertTrue("Expected to receive data from large dataset query", datarows.length() > 0); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testRowLevelSecurity(boolean useCalcite) throws IOException { + configureEngine(useCalcite); + String engineLabel = useCalcite ? "V3" : "V2"; + + // limited_user should only see "public" documents + + // Execute query as limited_user + String query = + String.format( + "search source=%s | fields security_level, message | stats count() by security_level", + SECURE_LOGS); + JSONObject result = executeQueryAsUser(query, LIMITED_USER); + + // Extract the datarows for validation + var datarows = result.getJSONArray("datarows"); + + // Derive column indexes from schema to support both V2 and V3 engines + var schema = result.getJSONArray("schema"); + int countIdx = -1; + int levelIdx = -1; + for (int i = 0; i < schema.length(); i++) { + String name = schema.getJSONObject(i).getString("name"); + if ("security_level".equals(name)) { + levelIdx = i; + } else if ("count()".equalsIgnoreCase(name) || "count".equalsIgnoreCase(name)) { + countIdx = i; + } + } + assertTrue("Expected count() in schema", countIdx >= 0); + assertTrue("Expected security_level in schema", levelIdx >= 0); + + // Count total documents visible + int totalDocs = 0; + boolean sawConfidential = false; + boolean sawInternal = false; + int publicDocs = 0; + + for (int i = 0; i < datarows.length(); i++) { + var row = datarows.getJSONArray(i); + int count = row.getInt(countIdx); + String securityLevel = row.getString(levelIdx); + totalDocs += count; + + if ("confidential".equals(securityLevel)) { + sawConfidential = true; + } else if ("internal".equals(securityLevel)) { + sawInternal = true; + } else if ("public".equals(securityLevel)) { + publicDocs = count; + } + } + + assertFalse( + String.format( + "[%s] SECURITY VIOLATION: limited_user should NOT see 'confidential' documents. " + + "This indicates ThreadContext is not being copied to async worker threads, " + + "causing queries to run with admin permissions and bypass row-level security.", + engineLabel), + sawConfidential); + + assertFalse( + String.format( + "[%s] SECURITY VIOLATION: limited_user should NOT see 'internal' documents. " + + "This indicates ThreadContext is not being copied to async worker threads, " + + "causing queries to run with admin permissions and bypass row-level security.", + engineLabel), + sawInternal); + + int expectedPublicDocs = LARGE_DATASET_SIZE / 2; + assertEquals( + String.format( + "[%s] limited_user should ONLY see 'public' documents (half of dataset). " + + "Seeing more indicates row-level security is being bypassed.", + engineLabel), + expectedPublicDocs, + publicDocs); + + assertEquals( + String.format( + "[%s] Total visible documents should match public documents only. " + + "Seeing all documents indicates row-level security is completely bypassed.", + engineLabel), + expectedPublicDocs, + totalDocs); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/security/PPLPermissionsIT.java b/integ-test/src/test/java/org/opensearch/sql/security/PPLPermissionsIT.java index d84ca2af1b..4b8905f4be 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/PPLPermissionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/PPLPermissionsIT.java @@ -13,14 +13,9 @@ import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import java.io.IOException; -import java.util.Locale; import org.json.JSONObject; import org.junit.Test; -import org.opensearch.client.Request; -import org.opensearch.client.RequestOptions; -import org.opensearch.client.Response; import org.opensearch.client.ResponseException; -import org.opensearch.sql.ppl.PPLIntegTestCase; /** * Integration tests for PPL permissions issue fix. Tests that PPL queries work correctly when users @@ -31,13 +26,12 @@ * for all indices instead of just the requested index when no indices were specified in the * SearchRequest. */ -public class PPLPermissionsIT extends PPLIntegTestCase { +public class PPLPermissionsIT extends SecurityTestBase { private static final String BANK_USER = "bank_user"; private static final String BANK_ROLE = "bank_role"; private static final String DOG_USER = "dog_user"; private static final String DOG_ROLE = "dog_role"; - private static final String STRONG_PASSWORD = "StrongPassword123!"; // Users for testing missing permissions private static final String NO_PPL_USER = "no_ppl_user"; @@ -78,10 +72,10 @@ protected void init() throws Exception { private void createSecurityRolesAndUsers() throws IOException { if (!initialized) { // Create role for bank index access - createRole(BANK_ROLE, TEST_INDEX_BANK); + createRoleWithIndexAccess(BANK_ROLE, TEST_INDEX_BANK); // Create role for dog index access - createRole(DOG_ROLE, TEST_INDEX_DOG); + createRoleWithIndexAccess(DOG_ROLE, TEST_INDEX_DOG); // Create users and map them to roles createUser(BANK_USER, BANK_ROLE); @@ -99,95 +93,10 @@ private void createSecurityRolesAndUsers() throws IOException { } } - private void createRole(String roleName, String indexPattern) throws IOException { - Request request = new Request("PUT", "/_plugins/_security/api/roles/" + roleName); - request.setJsonEntity( - String.format( - Locale.ROOT, - """ - { - "cluster_permissions": [ - "cluster:admin/opensearch/ppl" - ], - "index_permissions": [{ - "index_patterns": [ - "%s" - ], - "allowed_actions": [ - "indices:data/read/search*", - "indices:admin/mappings/get", - "indices:monitor/settings/get", - "indices:data/read/point_in_time/create", - "indices:data/read/point_in_time/delete" - ] - }] - } - """, - indexPattern)); - - RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); - restOptionsBuilder.addHeader("Content-Type", "application/json"); - request.setOptions(restOptionsBuilder); - - Response response = client().performRequest(request); - // Role creation returns 201 (Created) for new roles or 200 (OK) for updates - assertTrue( - response.getStatusLine().getStatusCode() == 200 - || response.getStatusLine().getStatusCode() == 201); - } - - private void createUser(String username, String roleName) throws IOException { - // Create user with password - Request userRequest = new Request("PUT", "/_plugins/_security/api/internalusers/" + username); - userRequest.setJsonEntity( - String.format( - Locale.ROOT, - """ - { - "password": "%s", - "backend_roles": [], - "attributes": {} - } - """, - STRONG_PASSWORD)); - - RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); - restOptionsBuilder.addHeader("Content-Type", "application/json"); - userRequest.setOptions(restOptionsBuilder); - - Response userResponse = client().performRequest(userRequest); - // User creation returns 201 (Created) for new users or 200 (OK) for updates - assertTrue( - userResponse.getStatusLine().getStatusCode() == 200 - || userResponse.getStatusLine().getStatusCode() == 201); - - // Map user to role - Request mappingRequest = new Request("PUT", "/_plugins/_security/api/rolesmapping/" + roleName); - mappingRequest.setJsonEntity( - String.format( - Locale.ROOT, - """ - { - "backend_roles": [], - "hosts": [], - "users": ["%s"] - } - """, - username)); - - mappingRequest.setOptions(restOptionsBuilder); - - Response mappingResponse = client().performRequest(mappingRequest); - // Role mapping returns 201 (Created) for new mappings or 200 (OK) for updates - assertTrue( - mappingResponse.getStatusLine().getStatusCode() == 200 - || mappingResponse.getStatusLine().getStatusCode() == 201); - } - /** Creates roles with missing permissions for negative testing. */ private void createRoleWithMissingPermissions() throws IOException { // Role missing PPL cluster permission - createRoleWithSpecificPermissions( + createRoleWithPermissions( NO_PPL_ROLE, TEST_INDEX_BANK, new String[] {}, // No cluster permissions @@ -201,7 +110,7 @@ private void createRoleWithMissingPermissions() throws IOException { createUser(NO_PPL_USER, NO_PPL_ROLE); // Role missing search permissions - createRoleWithSpecificPermissions( + createRoleWithPermissions( NO_SEARCH_ROLE, TEST_INDEX_BANK, new String[] {"cluster:admin/opensearch/ppl"}, @@ -214,7 +123,7 @@ private void createRoleWithMissingPermissions() throws IOException { createUser(NO_SEARCH_USER, NO_SEARCH_ROLE); // Role missing mapping permissions - createRoleWithSpecificPermissions( + createRoleWithPermissions( NO_MAPPING_ROLE, TEST_INDEX_BANK, new String[] {"cluster:admin/opensearch/ppl"}, @@ -227,7 +136,7 @@ private void createRoleWithMissingPermissions() throws IOException { createUser(NO_MAPPING_USER, NO_MAPPING_ROLE); // Role missing settings permissions - createRoleWithSpecificPermissions( + createRoleWithPermissions( NO_SETTINGS_ROLE, TEST_INDEX_BANK, new String[] {"cluster:admin/opensearch/ppl"}, @@ -240,55 +149,11 @@ private void createRoleWithMissingPermissions() throws IOException { createUser(NO_SETTINGS_USER, NO_SETTINGS_ROLE); } - /** Creates a role with specific permissions for testing. */ - private void createRoleWithSpecificPermissions( - String roleName, String indexPattern, String[] clusterPermissions, String[] indexPermissions) - throws IOException { - Request request = new Request("PUT", "/_plugins/_security/api/roles/" + roleName); - - StringBuilder clusterPermsJson = new StringBuilder(); - for (int i = 0; i < clusterPermissions.length; i++) { - clusterPermsJson.append("\"").append(clusterPermissions[i]).append("\""); - if (i < clusterPermissions.length - 1) clusterPermsJson.append(","); - } - - StringBuilder indexPermsJson = new StringBuilder(); - for (int i = 0; i < indexPermissions.length; i++) { - indexPermsJson.append("\"").append(indexPermissions[i]).append("\""); - if (i < indexPermissions.length - 1) indexPermsJson.append(","); - } - - request.setJsonEntity( - String.format( - Locale.ROOT, - """ - { - "cluster_permissions": [%s], - "index_permissions": [{ - "index_patterns": ["%s"], - "allowed_actions": [%s] - }] - } - """, - clusterPermsJson, - indexPattern, - indexPermsJson)); - - RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); - restOptionsBuilder.addHeader("Content-Type", "application/json"); - request.setOptions(restOptionsBuilder); - - Response response = client().performRequest(request); - assertTrue( - response.getStatusLine().getStatusCode() == 200 - || response.getStatusLine().getStatusCode() == 201); - } - /** Creates a user with minimal permissions for testing plugin-based PIT functionality. */ private void createMinimalUserForPitTesting() throws IOException { // Create role with minimal permissions needed for plugin-based PIT testing // This role has all required permissions (PPL, search, mapping, settings, and PIT) - createRoleWithSpecificPermissions( + createRoleWithPermissions( MINIMAL_ROLE, TEST_INDEX_BANK, new String[] {"cluster:admin/opensearch/ppl"}, // PPL permission @@ -306,7 +171,7 @@ private void createMinimalUserForPitTesting() throws IOException { private void createNoPitUserForTesting() throws IOException { // Create role with all permissions EXCEPT PIT create/delete permissions // This role has PPL, search, mapping, settings permissions but NO PIT permissions - createRoleWithSpecificPermissions( + createRoleWithPermissions( NO_PIT_ROLE, TEST_INDEX_BANK, new String[] {"cluster:admin/opensearch/ppl"}, // PPL permission @@ -320,25 +185,6 @@ private void createNoPitUserForTesting() throws IOException { createUser(NO_PIT_USER, NO_PIT_ROLE); } - /** Executes a PPL query as a specific user with basic authentication. */ - private JSONObject executeQueryAsUser(String query, String username) throws IOException { - Request request = new Request("POST", "/_plugins/_ppl"); - request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); - - RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); - restOptionsBuilder.addHeader("Content-Type", "application/json"); - restOptionsBuilder.addHeader( - "Authorization", - "Basic " - + java.util.Base64.getEncoder() - .encodeToString((username + ":" + STRONG_PASSWORD).getBytes())); - request.setOptions(restOptionsBuilder); - - Response response = client().performRequest(request); - assertEquals(200, response.getStatusLine().getStatusCode()); - return new JSONObject(org.opensearch.sql.legacy.TestUtils.getResponseBody(response, true)); - } - @Test public void testUserWithBankPermissionCanAccessBankIndex() throws IOException { // Test that bank_user can access bank index - this should work with the fix diff --git a/integ-test/src/test/java/org/opensearch/sql/security/SecurityTestBase.java b/integ-test/src/test/java/org/opensearch/sql/security/SecurityTestBase.java new file mode 100644 index 0000000000..d64b69d4f2 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/security/SecurityTestBase.java @@ -0,0 +1,354 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.security; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import org.json.JSONObject; +import org.opensearch.client.Request; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.Response; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Base class for security-related integration tests. Provides common utilities for creating users, + * roles, and executing queries with authentication. + */ +public abstract class SecurityTestBase extends PPLIntegTestCase { + + protected static final String STRONG_PASSWORD = "correcthorsebatterystaple"; + + /** + * Creates a role with access to a specific index pattern and standard permissions. + * + * @param roleName the name of the role + * @param indexPattern the index pattern to grant access to + */ + protected void createRoleWithIndexAccess(String roleName, String indexPattern) + throws IOException { + createRoleWithPermissions( + roleName, + indexPattern, + new String[] {"cluster:admin/opensearch/ppl"}, + new String[] { + "indices:data/read/search*", + "indices:admin/mappings/get", + "indices:monitor/settings/get", + "indices:data/read/point_in_time/create", + "indices:data/read/point_in_time/delete" + }); + } + + /** + * Creates a role with specific cluster and index permissions. + * + * @param roleName the name of the role + * @param indexPattern the index pattern to grant access to + * @param clusterPermissions array of cluster-level permissions + * @param indexPermissions array of index-level permissions + */ + protected void createRoleWithPermissions( + String roleName, String indexPattern, String[] clusterPermissions, String[] indexPermissions) + throws IOException { + + Request request = new Request("PUT", "/_plugins/_security/api/roles/" + roleName); + + StringBuilder clusterPermsJson = new StringBuilder(); + for (int i = 0; i < clusterPermissions.length; i++) { + clusterPermsJson.append("\"").append(clusterPermissions[i]).append("\""); + if (i < clusterPermissions.length - 1) clusterPermsJson.append(","); + } + + StringBuilder indexPermsJson = new StringBuilder(); + for (int i = 0; i < indexPermissions.length; i++) { + indexPermsJson.append("\"").append(indexPermissions[i]).append("\""); + if (i < indexPermissions.length - 1) indexPermsJson.append(","); + } + + request.setJsonEntity( + String.format( + Locale.ROOT, + """ + { + "cluster_permissions": [%s], + "index_permissions": [{ + "index_patterns": ["%s"], + "allowed_actions": [%s] + }] + } + """, + clusterPermsJson, + indexPattern, + indexPermsJson)); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + + Response response = client().performRequest(request); + assertTrue( + response.getStatusLine().getStatusCode() == 200 + || response.getStatusLine().getStatusCode() == 201); + } + + /** + * Creates a role with document-level security (DLS) filtering. + * + * @param roleName the name of the role + * @param indexPattern the index pattern to grant access to + * @param dlsQuery the document-level security query in escaped JSON string format + */ + protected void createRoleWithDLS(String roleName, String indexPattern, String dlsQuery) + throws IOException { + Request request = new Request("PUT", "/_plugins/_security/api/roles/" + roleName); + request.setJsonEntity( + String.format( + Locale.ROOT, + """ + { + "cluster_permissions": [ + "cluster:admin/opensearch/ppl" + ], + "index_permissions": [{ + "index_patterns": [ + "%s" + ], + "allowed_actions": [ + "indices:data/read/search*", + "indices:admin/mappings/get", + "indices:monitor/settings/get", + "indices:data/read/point_in_time/create", + "indices:data/read/point_in_time/delete" + ], + "dls": "%s" + }] + } + """, + indexPattern, + dlsQuery)); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + + Response response = client().performRequest(request); + assertTrue( + response.getStatusLine().getStatusCode() == 200 + || response.getStatusLine().getStatusCode() == 201); + } + + /** + * Creates a role with field-level security (FLS) restrictions. + * + * @param roleName the name of the role + * @param indexPattern the index pattern to grant access to + * @param allowedFields array of field names that the role can access + */ + protected void createRoleWithFLS(String roleName, String indexPattern, String[] allowedFields) + throws IOException { + StringBuilder fieldsJson = new StringBuilder(); + for (int i = 0; i < allowedFields.length; i++) { + if (i > 0) fieldsJson.append(", "); + fieldsJson.append("\"").append(allowedFields[i]).append("\""); + } + + Request request = new Request("PUT", "/_plugins/_security/api/roles/" + roleName); + request.setJsonEntity( + String.format( + Locale.ROOT, + """ + { + "cluster_permissions": [ + "cluster:admin/opensearch/ppl" + ], + "index_permissions": [{ + "index_patterns": [ + "%s" + ], + "allowed_actions": [ + "indices:data/read/search*", + "indices:admin/mappings/get", + "indices:monitor/settings/get", + "indices:data/read/point_in_time/create", + "indices:data/read/point_in_time/delete" + ], + "fls": [%s] + }] + } + """, + indexPattern, + fieldsJson)); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + + Response response = client().performRequest(request); + assertTrue( + response.getStatusLine().getStatusCode() == 200 + || response.getStatusLine().getStatusCode() == 201); + } + + /** + * Creates a user and maps them to a role. + * + * @param username the username + * @param roleName the role to map the user to + */ + protected void createUser(String username, String roleName) throws IOException { + // Create user with password + Request userRequest = new Request("PUT", "/_plugins/_security/api/internalusers/" + username); + userRequest.setJsonEntity( + String.format( + Locale.ROOT, + """ + { + "password": "%s", + "backend_roles": [], + "attributes": {} + } + """, + STRONG_PASSWORD)); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + userRequest.setOptions(restOptionsBuilder); + + Response userResponse = client().performRequest(userRequest); + assertTrue( + userResponse.getStatusLine().getStatusCode() == 200 + || userResponse.getStatusLine().getStatusCode() == 201); + + // Map user to role + Request mappingRequest = new Request("PUT", "/_plugins/_security/api/rolesmapping/" + roleName); + mappingRequest.setJsonEntity( + String.format( + Locale.ROOT, + """ + { + "backend_roles": [], + "hosts": [], + "users": ["%s"] + } + """, + username)); + + mappingRequest.setOptions(restOptionsBuilder); + + Response mappingResponse = client().performRequest(mappingRequest); + assertTrue( + mappingResponse.getStatusLine().getStatusCode() == 200 + || mappingResponse.getStatusLine().getStatusCode() == 201); + } + + /** + * Executes a PPL query as a specific user with basic authentication. + * + * @param query the PPL query to execute + * @param username the username to authenticate as + * @return the JSON response from the query + */ + protected JSONObject executeQueryAsUser(String query, String username) throws IOException { + Request request = new Request("POST", "/_plugins/_ppl"); + request.setJsonEntity( + String.format( + Locale.ROOT, + """ + { + "query": "%s" + } + """, + query)); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + restOptionsBuilder.addHeader("Authorization", createBasicAuthHeader(username, STRONG_PASSWORD)); + request.setOptions(restOptionsBuilder); + + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + return new JSONObject(org.opensearch.sql.legacy.TestUtils.getResponseBody(response, true)); + } + + /** + * Creates a Basic authentication header value. + * + * @param username the username + * @param password the password + * @return the Basic auth header value + */ + protected String createBasicAuthHeader(String username, String password) { + return "Basic " + + java.util.Base64.getEncoder().encodeToString((username + ":" + password).getBytes()); + } + + /** + * Helper to build bulk insert request body for multiple documents. + * + * @param indexName the index to insert into + * @param documents list of document maps (field name -> value) + * @return the bulk request body as a string + */ + protected String buildBulkInsertRequest(String indexName, List> documents) { + StringBuilder bulk = new StringBuilder(); + for (Map doc : documents) { + bulk.append(String.format(Locale.ROOT, "{ \"index\": { \"_index\": \"%s\" } }\n", indexName)); + bulk.append(new JSONObject(doc).toString()); + bulk.append("\n"); + } + return bulk.toString(); + } + + /** + * Performs a bulk insert operation with automatic refresh. + * + * @param bulkRequestBody the bulk request body (NDJSON format) + * @return the response from the bulk operation + */ + protected Response performBulkInsert(String bulkRequestBody) throws IOException { + Request request = new Request("POST", "/_bulk"); + request.addParameter("refresh", "true"); + request.setJsonEntity(bulkRequestBody); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/x-ndjson"); + request.setOptions(restOptionsBuilder); + + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + return response; + } + + /** + * Creates an index with a simple mapping. + * + * @param indexName the name of the index + * @param mappingJson the mapping definition as JSON string + */ + protected void createIndexWithMapping(String indexName, String mappingJson) throws IOException { + Request request = new Request("PUT", "/" + indexName); + request.setJsonEntity(mappingJson); + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + } + + /** Simple builder for creating bulk insert documents. */ + protected static class BulkDocumentBuilder { + private final List> documents = new ArrayList<>(); + + public BulkDocumentBuilder addDocument(Map doc) { + documents.add(doc); + return this; + } + + public List> build() { + return documents; + } + } +}