From 2975072ba58f61f6a424c06ddb6abed08d4a920c Mon Sep 17 00:00:00 2001 From: Jarek Jarcec Cecho Date: Fri, 25 Jul 2014 14:42:13 -0700 Subject: [PATCH 1/6] SENTRY-336: Fix test failures on real cluster (Sravya Tirukkovalur via Jarek Jarcec Cecho) --- .../AbstractTestWithStaticConfiguration.java | 22 ++++++++++- .../e2e/hive/TestExportImportPrivileges.java | 14 +++---- .../sentry/tests/e2e/hive/TestOperations.java | 10 ++++- .../hive/TestPrivilegesAtFunctionScope.java | 30 +++++++------- .../sentry/tests/e2e/hive/TestSandboxOps.java | 35 +++++++---------- .../tests/e2e/hive/TestUriPermissions.java | 39 +++++++++++-------- .../hive/hiveserver/HiveServerFactory.java | 2 +- 7 files changed, 89 insertions(+), 63 deletions(-) diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java index dbc01f911..31d8172e4 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/AbstractTestWithStaticConfiguration.java @@ -105,6 +105,7 @@ public abstract class AbstractTestWithStaticConfiguration { protected static final String SERVER_HOST = "localhost"; private static final String EXTERNAL_SENTRY_SERVICE = "sentry.e2etest.external.sentry"; + protected static final String EXTERNAL_HIVE_LIB = "sentry.e2etest.hive.lib"; protected static boolean policy_on_hdfs = false; protected static boolean useSentryService = false; @@ -119,6 +120,7 @@ public abstract class AbstractTestWithStaticConfiguration { protected static File policyFileLocation; protected static HiveServer hiveServer; protected static FileSystem fileSystem; + protected static HiveServerFactory.HiveServer2Type hiveServer2Type; protected static DFS dfs; protected static Map properties; protected static SentryService sentryServer; @@ -196,9 +198,12 @@ public static void setupTestStaticConfiguration() throws Exception { fileSystem = dfs.getFileSystem(); String policyURI; + + //TODO: We can probably get rid of this. PolicyFile policyFile = PolicyFile.setAdminOnServer1(ADMIN1) .setUserGroupMapping(StaticUserGroup.getStaticMapping()); policyFile.write(policyFileLocation); + if (policy_on_hdfs) { String dfsUri = fileSystem.getDefaultUri(fileSystem.getConf()).toString(); LOGGER.error("dfsUri " + dfsUri); @@ -213,11 +218,26 @@ public static void setupTestStaticConfiguration() throws Exception { setupSentryService(); } - hiveServer = HiveServerFactory.create(properties, baseDir, confDir, logDir, policyURI, fileSystem); + hiveServer = create(properties, baseDir, confDir, logDir, policyURI, fileSystem); hiveServer.start(); createContext(); } + public static HiveServer create(Map properties, + File baseDir, File confDir, File logDir, String policyFile, + FileSystem fileSystem) throws Exception { + String type = properties.get(HiveServerFactory.HIVESERVER2_TYPE); + if(type == null) { + type = System.getProperty(HiveServerFactory.HIVESERVER2_TYPE); + } + if(type == null) { + type = HiveServerFactory.HiveServer2Type.InternalHiveServer2.name(); + } + hiveServer2Type = HiveServerFactory.HiveServer2Type.valueOf(type.trim()); + return HiveServerFactory.create(hiveServer2Type, properties, + baseDir, confDir, logDir, policyFile, fileSystem); + } + protected void writePolicyFile(PolicyFile policyFile) throws Exception{ policyFile.write(context.getPolicyFile()); diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestExportImportPrivileges.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestExportImportPrivileges.java index b6c985e25..eaf381682 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestExportImportPrivileges.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestExportImportPrivileges.java @@ -41,6 +41,8 @@ public void setup() throws Exception { Resources.copy(Resources.getResource(SINGLE_TYPE_DATA_FILE_NAME), to); to.close(); policyFile = PolicyFile.setAdminOnServer1(ADMINGROUP); + policyFile.setUserGroupMapping(StaticUserGroup.getStaticMapping()); + writePolicyFile(policyFile); } @Test @@ -54,7 +56,6 @@ public void testInsertToDirPrivileges() throws Exception { .addRolesToGroup(USERGROUP2, "db1_read", "db1_write") .addPermissionsToRole("db1_write", "server=server1->db=" + DB1 + "->table=" + TBL1 + "->action=INSERT") .addPermissionsToRole("db1_read", "server=server1->db=" + DB1 + "->table=" + TBL1 + "->action=SELECT") - .setUserGroupMapping(StaticUserGroup.getStaticMapping()) .addPermissionsToRole("data_dump", "server=server1->URI=" + dumpDir); writePolicyFile(policyFile); @@ -93,6 +94,10 @@ public void testExportImportPrivileges() throws Exception { Connection connection = null; Statement statement = null; String exportDir = dfs.getBaseDir() + "/hive_export1"; + dropDb(ADMIN1, DB1); + createDb(ADMIN1, DB1); + createTable(ADMIN1, DB1, dataFile, TBL1); + policyFile .addRolesToGroup(USERGROUP1, "tab1_read", "tab1_write", "db1_all", "data_read", "data_export") .addRolesToGroup(USERGROUP2, "tab1_write", "tab1_read") @@ -100,14 +105,9 @@ public void testExportImportPrivileges() throws Exception { .addPermissionsToRole("tab1_read", "server=server1->db=" + DB1 + "->table=" + TBL1 + "->action=SELECT") .addPermissionsToRole("db1_all", "server=server1->db=" + DB1) .addPermissionsToRole("data_read", "server=server1->URI=file://" + dataFile.getPath()) - .addPermissionsToRole("data_export", "server=server1->URI=" + exportDir) - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); + .addPermissionsToRole("data_export", "server=server1->URI=" + exportDir); writePolicyFile(policyFile); - dropDb(ADMIN1, DB1); - createDb(ADMIN1, DB1); - createTable(ADMIN1, DB1, dataFile, TBL1); - // Negative test, user2 doesn't have access to the file being loaded connection = context.createConnection(USER2_1); statement = context.createStatement(connection); diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperations.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperations.java index 23e91f093..ae6b33db7 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperations.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperations.java @@ -270,6 +270,7 @@ public void testAllOnTable() throws Exception{ statement.execute("ALTER TABLE tb1 CLUSTERED BY (a) SORTED BY (a) INTO 1 BUCKETS"); statement.execute("ALTER TABLE tb1 TOUCH"); statement.execute("ALTER TABLE tb1 ENABLE NO_DROP"); + statement.execute("ALTER TABLE tb1 DISABLE NO_DROP"); statement.execute("ALTER TABLE tb1 DISABLE OFFLINE"); statement.execute("ALTER TABLE tb1 SET FILEFORMAT RCFILE"); @@ -292,7 +293,14 @@ public void testAllOnTable() throws Exception{ statement.execute("CREATE INDEX tb1_index ON TABLE tb1 (a) AS 'COMPACT' WITH DEFERRED REBUILD"); //statement.execute("ALTER INDEX tb1_index ON tb1 REBUILD"); statement.execute("ALTER TABLE tb1 RENAME TO tb2"); - statement.execute("DROP TABLE db1.tb1"); + + //Drop of the new tablename works only when Hive meta store syncs the alters with the sentry privileges. + //This is currently not set for pseudo cluster runs + if( hiveServer2Type.equals(HiveServerFactory.HiveServer2Type.UnmanagedHiveServer2)) { + statement.execute("DROP TABLE db1.tb2"); + } else { + statement.execute("DROP TABLE db1.tb1"); + } statement.close(); connection.close(); diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java index ccb5550e7..1e1978d1b 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestPrivilegesAtFunctionScope.java @@ -46,7 +46,8 @@ public void setup() throws Exception { Resources.copy(Resources.getResource(SINGLE_TYPE_DATA_FILE_NAME), to); to.close(); policyFile = PolicyFile.setAdminOnServer1(ADMINGROUP); - + policyFile.setUserGroupMapping(StaticUserGroup.getStaticMapping()); + writePolicyFile(policyFile); } /** @@ -61,22 +62,15 @@ public void testFuncPrivileges1() throws Exception { String tableName1 = "tb_1"; String udfClassName = "org.apache.hadoop.hive.ql.udf.generic.GenericUDFPrintf"; CodeSource udfSrc = Class.forName(udfClassName).getProtectionDomain().getCodeSource(); - policyFile - .addRolesToGroup(USERGROUP1, "db1_all", "UDF_JAR") - .addRolesToGroup(USERGROUP2, "db1_tab1", "UDF_JAR") - .addRolesToGroup(USERGROUP3, "db1_tab1") - .addPermissionsToRole("db1_all", "server=server1->db=" + dbName1) - .addPermissionsToRole("db1_tab1", "server=server1->db=" + dbName1 + "->table=" + tableName1) - .addPermissionsToRole("UDF_JAR", "server=server1->uri=file://" + udfSrc.getLocation().getPath()) - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); - writePolicyFile(policyFile); - + String udfLocation = System.getProperty(EXTERNAL_HIVE_LIB); + if(udfLocation == null) { + udfLocation = udfSrc.getLocation().getPath(); + } Connection connection = context.createConnection(ADMIN1); Statement statement = context.createStatement(connection); statement.execute("DROP DATABASE IF EXISTS " + dbName1 + " CASCADE"); statement.execute("CREATE DATABASE " + dbName1); statement.execute("USE " + dbName1); - statement.execute("DROP TABLE IF EXISTS " + dbName1 + "." + tableName1); statement.execute("create table " + dbName1 + "." + tableName1 + " (under_col int comment 'the under column', value string)"); statement.execute("LOAD DATA LOCAL INPATH '" + dataFile.getPath() + "' INTO TABLE " @@ -85,6 +79,15 @@ public void testFuncPrivileges1() throws Exception { statement.execute("DROP TEMPORARY FUNCTION IF EXISTS printf_test_2"); context.close(); + policyFile + .addRolesToGroup(USERGROUP1, "db1_all", "UDF_JAR") + .addRolesToGroup(USERGROUP2, "db1_tab1", "UDF_JAR") + .addRolesToGroup(USERGROUP3, "db1_tab1") + .addPermissionsToRole("db1_all", "server=server1->db=" + dbName1) + .addPermissionsToRole("db1_tab1", "server=server1->db=" + dbName1 + "->table=" + tableName1) + .addPermissionsToRole("UDF_JAR", "server=server1->uri=file://" + udfLocation); + writePolicyFile(policyFile); + // user1 should be able create/drop temp functions connection = context.createConnection(USER1_1); statement = context.createStatement(connection); @@ -144,8 +147,7 @@ public void testUdfWhiteList () throws Exception { .addRolesToGroup(USERGROUP3, "db1_tab1") .addPermissionsToRole("db1_all", "server=server1->db=" + dbName1) .addPermissionsToRole("db1_tab1", "server=server1->db=" + dbName1 + "->table=" + tableName1) - .addPermissionsToRole("UDF_JAR", "server=server1->uri=file://${user.home}/.m2") - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); + .addPermissionsToRole("UDF_JAR", "server=server1->uri=file://${user.home}/.m2"); writePolicyFile(policyFile); Connection connection = context.createConnection(ADMIN1); diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestSandboxOps.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestSandboxOps.java index 021f5d22f..8710cd3b9 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestSandboxOps.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestSandboxOps.java @@ -48,6 +48,8 @@ public void setup() throws Exception { Resources.copy(Resources.getResource(SINGLE_TYPE_DATA_FILE_NAME), to); to.close(); policyFile = PolicyFile.setAdminOnServer1(ADMINGROUP); + policyFile.setUserGroupMapping(StaticUserGroup.getStaticMapping()); + writePolicyFile(policyFile); loadData = "server=server1->uri=file://" + dataFile.getPath(); } @@ -55,8 +57,7 @@ private PolicyFile addTwoUsersWithAllDb() throws Exception { policyFile .addPermissionsToRole("db1_all", "server=server1->db=db1") .addPermissionsToRole("db2_all", "server=server1->db=db2") - .addRolesToGroup(USERGROUP1, "db1_all", "db2_all") - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); + .addRolesToGroup(USERGROUP1, "db1_all", "db2_all"); return policyFile; } /** @@ -101,9 +102,6 @@ public void testDbPrivileges() throws Exception { */ @Test public void testAdminDbPrivileges() throws Exception { - policyFile - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); - writePolicyFile(policyFile); Connection adminCon = context.createConnection(ADMIN1); Statement adminStmt = context.createStatement(adminCon); String dbName = "db1"; @@ -136,8 +134,7 @@ public void testAdminDbPrivileges() throws Exception { public void testNegativeUserDMLPrivileges() throws Exception { policyFile .addPermissionsToRole("db1_tab2_all", "server=server1->db=db1->table=table_2") - .addRolesToGroup(USERGROUP1, "db1_tab2_all") - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); + .addRolesToGroup(USERGROUP1, "db1_tab2_all"); writePolicyFile(policyFile); Connection adminCon = context.createConnection(ADMIN1); Statement adminStmt = context.createStatement(adminCon); @@ -182,8 +179,7 @@ public void testNegUserPrivilegesAll() throws Exception { .addRolesToGroup(USERGROUP1, "db1_all") .addRolesToGroup(USERGROUP2, "db1_tab1_select") .addPermissionsToRole("db1_tab1_select", "server=server1->db=db1->table=table_1->action=select") - .addPermissionsToRole("db1_all", "server=server1->db=db1") - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); + .addPermissionsToRole("db1_all", "server=server1->db=db1"); writePolicyFile(policyFile); // create dbs Connection adminCon = context.createConnection(ADMIN1); @@ -257,8 +253,7 @@ public void testSandboxOpt9() throws Exception { policyFile .addPermissionsToRole(GROUP1_ROLE, ALL_DB1, ALL_DB2, loadData) - .addRolesToGroup(USERGROUP1, GROUP1_ROLE) - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); + .addRolesToGroup(USERGROUP1, GROUP1_ROLE); writePolicyFile(policyFile); dropDb(ADMIN1, DB1, DB2); @@ -327,8 +322,7 @@ public void testSandboxOpt13() throws Exception { // unrelated permission to allow user1 to connect to db1 policyFile .addPermissionsToRole(GROUP1_ROLE, SELECT_DB1_TBL2) - .addRolesToGroup(USERGROUP1, GROUP1_ROLE) - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); + .addRolesToGroup(USERGROUP1, GROUP1_ROLE); writePolicyFile(policyFile); dropDb(ADMIN1, DB1); createDb(ADMIN1, DB1); @@ -382,16 +376,16 @@ public void testSandboxOpt13() throws Exception { @Test public void testSandboxOpt17() throws Exception { + dropDb(ADMIN1, DB1); + createDb(ADMIN1, DB1); + policyFile .addRolesToGroup(USERGROUP1, "all_db1", "load_data") .addRolesToGroup(USERGROUP2, "select_tb1") .addPermissionsToRole("select_tb1", "server=server1->db=db_1->table=tbl_1->action=select") .addPermissionsToRole("all_db1", "server=server1->db=db_1") - .addPermissionsToRole("load_data", "server=server1->uri=file://" + dataFile.toString()) - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); + .addPermissionsToRole("load_data", "server=server1->uri=file://" + dataFile.toString()); writePolicyFile(policyFile); - dropDb(ADMIN1, DB1); - createDb(ADMIN1, DB1); createTable(USER1_1, DB1, dataFile, TBL1, TBL2); Connection connection = context.createConnection(USER1_1); @@ -450,8 +444,7 @@ public void testInsertOverwriteAndLoadData() throws Exception { .addPermissionsToRole("all_db1", "server=server1->db=db_1") .addPermissionsToRole("load_data", "server=server1->uri=file://" + allowedDir.getPath() + ", server=server1->uri=file://" + allowedDir.getPath() + - ", server=server1->uri=" + allowedDfsDir.toString()) - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); + ", server=server1->uri=" + allowedDfsDir.toString()); writePolicyFile(policyFile); dropDb(ADMIN1, DB1); @@ -484,8 +477,7 @@ public void testSandboxOpt10() throws Exception { policyFile .addPermissionsToRole(GROUP1_ROLE, ALL_DB1, SELECT_DB2_TBL2, loadData) - .addRolesToGroup(USERGROUP1, GROUP1_ROLE) - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); + .addRolesToGroup(USERGROUP1, GROUP1_ROLE); writePolicyFile(policyFile); dropDb(ADMIN1, DB1, DB2); @@ -514,7 +506,6 @@ public void testPerDbPolicyOnDFS() throws Exception { .addRolesToGroup(USERGROUP1, "select_tbl1") .addRolesToGroup(USERGROUP2, "select_tbl2") .addPermissionsToRole("select_tbl1", "server=server1->db=db1->table=tbl1->action=select") - .setUserGroupMapping(StaticUserGroup.getStaticMapping()) .addDatabase("db2", dfs.getBaseDir().toUri().toString() + "/" + DB2_POLICY_FILE); writePolicyFile(policyFile); diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestUriPermissions.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestUriPermissions.java index c633cc292..8a857e0fe 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestUriPermissions.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestUriPermissions.java @@ -39,6 +39,8 @@ public class TestUriPermissions extends AbstractTestWithStaticConfiguration { @Before public void setup() throws Exception { policyFile = PolicyFile.setAdminOnServer1(ADMINGROUP); + policyFile.setUserGroupMapping(StaticUserGroup.getStaticMapping()); + writePolicyFile(policyFile); } @@ -61,8 +63,7 @@ public void testLoadPrivileges() throws Exception { .addRolesToGroup(USERGROUP2, "db1_write") .addPermissionsToRole("db1_write", "server=server1->db=" + dbName + "->table=" + tabName + "->action=INSERT") .addPermissionsToRole("db1_read", "server=server1->db=" + dbName + "->table=" + tabName + "->action=SELECT") - .addPermissionsToRole("data_read", loadData) - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); + .addPermissionsToRole("data_read", loadData); writePolicyFile(policyFile); // create dbs @@ -108,16 +109,6 @@ public void testAlterPartitionLocationPrivileges() throws Exception { Connection userConn = null; Statement userStmt = null; - policyFile - .addRolesToGroup(USERGROUP1, "db1_all", "data_read") - .addRolesToGroup(USERGROUP2, "db1_all") - .addRolesToGroup(USERGROUP3, "db1_tab1_all", "data_read") - .addPermissionsToRole("db1_all", "server=server1->db=" + dbName) - .addPermissionsToRole("db1_tab1_all", "server=server1->db=" + dbName + "->table=" + tabName) - .addPermissionsToRole("data_read", "server=server1->uri=" + tabDir) - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); - writePolicyFile(policyFile); - // create dbs Connection adminCon = context.createConnection(ADMIN1); Statement adminStmt = context.createStatement(adminCon); @@ -128,6 +119,16 @@ public void testAlterPartitionLocationPrivileges() throws Exception { adminStmt.execute("CREATE TABLE " + tabName + " (id int) PARTITIONED BY (dt string)"); adminCon.close(); + policyFile + .addRolesToGroup(USERGROUP1, "db1_all", "data_read") + .addRolesToGroup(USERGROUP2, "db1_all") + .addRolesToGroup(USERGROUP3, "db1_tab1_all", "data_read") + .addPermissionsToRole("db1_all", "server=server1->db=" + dbName) + .addPermissionsToRole("db1_tab1_all", "server=server1->db=" + dbName + "->table=" + tabName) + .addPermissionsToRole("data_read", "server=server1->uri=" + tabDir); + writePolicyFile(policyFile); + + // positive test: user1 has privilege to alter table add partition but not set location userConn = context.createConnection(USER1_1); userStmt = context.createStatement(userConn); @@ -183,8 +184,7 @@ public void testAlterTableLocationPrivileges() throws Exception { .addRolesToGroup(USERGROUP2, "db1_all, data_read") .addPermissionsToRole("db1_all", "server=server1->db=" + dbName) .addPermissionsToRole("data_read", "server=server1->URI=" + tabDir) - .addPermissionsToRole("server1_all", "server=server1") - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); + .addPermissionsToRole("server1_all", "server=server1"); writePolicyFile(policyFile); // create dbs @@ -219,14 +219,19 @@ public void testExternalTablePrivileges() throws Exception { String dbName = "db1"; Connection userConn = null; Statement userStmt = null; - String tableDir = "file://" + context.getDataDir() + "/" + Math.random(); + + String dataDirPath = "file://" + dataDir; + String tableDir = dataDirPath + "/" + Math.random(); + + //Hive needs write permissions on this local directory + baseDir.setWritable(true, false); + dataDir.setWritable(true, false); policyFile .addRolesToGroup(USERGROUP1, "db1_all", "data_read") .addRolesToGroup(USERGROUP2, "db1_all") .addPermissionsToRole("db1_all", "server=server1->db=" + dbName) - .addPermissionsToRole("data_read", "server=server1->URI=" + tableDir) - .setUserGroupMapping(StaticUserGroup.getStaticMapping()); + .addPermissionsToRole("data_read", "server=server1->URI=" + dataDirPath); writePolicyFile(policyFile); // create dbs diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java index 44331f65a..5ebe4d7d6 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/hiveserver/HiveServerFactory.java @@ -89,7 +89,7 @@ public static HiveServer create(Map properties, return create(HiveServer2Type.valueOf(type.trim()), properties, baseDir, confDir, logDir, policyFile, fileSystem); } - private static HiveServer create(HiveServer2Type type, + public static HiveServer create(HiveServer2Type type, Map properties, File baseDir, File confDir, File logDir, String policyFile, FileSystem fileSystem) throws Exception { From 484d4a405af91dab0df037ab39b0e42e19a61005 Mon Sep 17 00:00:00 2001 From: Sravya Tirukkovalur Date: Fri, 25 Jul 2014 14:51:46 -0700 Subject: [PATCH 2/6] SENTRY-365: Create release branch for 1.4.0 (Tuong Truong via Sravya Tirukkovalur) --- pom.xml | 2 +- sentry-binding/pom.xml | 2 +- sentry-binding/sentry-binding-hive/pom.xml | 2 +- sentry-binding/sentry-binding-solr/pom.xml | 2 +- sentry-core/pom.xml | 2 +- sentry-core/sentry-core-common/pom.xml | 2 +- sentry-core/sentry-core-model-db/pom.xml | 2 +- sentry-core/sentry-core-model-search/pom.xml | 2 +- sentry-dist/pom.xml | 2 +- sentry-policy/pom.xml | 2 +- sentry-policy/sentry-policy-common/pom.xml | 2 +- sentry-policy/sentry-policy-db/pom.xml | 2 +- sentry-policy/sentry-policy-search/pom.xml | 2 +- sentry-provider/pom.xml | 2 +- sentry-provider/sentry-provider-cache/pom.xml | 2 +- sentry-provider/sentry-provider-common/pom.xml | 2 +- sentry-provider/sentry-provider-db/pom.xml | 2 +- sentry-provider/sentry-provider-file/pom.xml | 2 +- sentry-tests/pom.xml | 2 +- sentry-tests/sentry-tests-hive/pom.xml | 2 +- sentry-tests/sentry-tests-solr/pom.xml | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) diff --git a/pom.xml b/pom.xml index 2833cf00f..f615bd69b 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ limitations under the License. org.apache.sentry sentry - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating Sentry component Sentry pom diff --git a/sentry-binding/pom.xml b/sentry-binding/pom.xml index 6cfd7781f..b35e5f6fd 100644 --- a/sentry-binding/pom.xml +++ b/sentry-binding/pom.xml @@ -22,7 +22,7 @@ limitations under the License. org.apache.sentry sentry - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-binding diff --git a/sentry-binding/sentry-binding-hive/pom.xml b/sentry-binding/sentry-binding-hive/pom.xml index 5b822ba4c..071e2d1c6 100644 --- a/sentry-binding/sentry-binding-hive/pom.xml +++ b/sentry-binding/sentry-binding-hive/pom.xml @@ -22,7 +22,7 @@ limitations under the License. org.apache.sentry sentry-binding - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-binding-hive diff --git a/sentry-binding/sentry-binding-solr/pom.xml b/sentry-binding/sentry-binding-solr/pom.xml index 8e7cf1316..e9178410d 100644 --- a/sentry-binding/sentry-binding-solr/pom.xml +++ b/sentry-binding/sentry-binding-solr/pom.xml @@ -22,7 +22,7 @@ limitations under the License. org.apache.sentry sentry-binding - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-binding-solr diff --git a/sentry-core/pom.xml b/sentry-core/pom.xml index c7b107422..a413b5b66 100644 --- a/sentry-core/pom.xml +++ b/sentry-core/pom.xml @@ -21,7 +21,7 @@ limitations under the License. org.apache.sentry sentry - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-core diff --git a/sentry-core/sentry-core-common/pom.xml b/sentry-core/sentry-core-common/pom.xml index d1785b8d4..772d292a1 100644 --- a/sentry-core/sentry-core-common/pom.xml +++ b/sentry-core/sentry-core-common/pom.xml @@ -21,7 +21,7 @@ limitations under the License. org.apache.sentry sentry-core - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-core-common diff --git a/sentry-core/sentry-core-model-db/pom.xml b/sentry-core/sentry-core-model-db/pom.xml index 652b8a2f2..0b60846c6 100644 --- a/sentry-core/sentry-core-model-db/pom.xml +++ b/sentry-core/sentry-core-model-db/pom.xml @@ -21,7 +21,7 @@ limitations under the License. org.apache.sentry sentry-core - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-core-model-db diff --git a/sentry-core/sentry-core-model-search/pom.xml b/sentry-core/sentry-core-model-search/pom.xml index ca20293e3..974e42b54 100644 --- a/sentry-core/sentry-core-model-search/pom.xml +++ b/sentry-core/sentry-core-model-search/pom.xml @@ -21,7 +21,7 @@ limitations under the License. org.apache.sentry sentry-core - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-core-model-search diff --git a/sentry-dist/pom.xml b/sentry-dist/pom.xml index fa03d4e8e..f97d86f53 100644 --- a/sentry-dist/pom.xml +++ b/sentry-dist/pom.xml @@ -20,7 +20,7 @@ limitations under the License. org.apache.sentry sentry - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-dist Sentry Distribution diff --git a/sentry-policy/pom.xml b/sentry-policy/pom.xml index 560ad9941..c36094ed6 100644 --- a/sentry-policy/pom.xml +++ b/sentry-policy/pom.xml @@ -22,7 +22,7 @@ limitations under the License. org.apache.sentry sentry - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-policy diff --git a/sentry-policy/sentry-policy-common/pom.xml b/sentry-policy/sentry-policy-common/pom.xml index 168b17511..b49a04107 100644 --- a/sentry-policy/sentry-policy-common/pom.xml +++ b/sentry-policy/sentry-policy-common/pom.xml @@ -21,7 +21,7 @@ limitations under the License. org.apache.sentry sentry-policy - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-policy-common diff --git a/sentry-policy/sentry-policy-db/pom.xml b/sentry-policy/sentry-policy-db/pom.xml index 315eeea53..90a8d3aee 100644 --- a/sentry-policy/sentry-policy-db/pom.xml +++ b/sentry-policy/sentry-policy-db/pom.xml @@ -21,7 +21,7 @@ limitations under the License. org.apache.sentry sentry-policy - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-policy-db diff --git a/sentry-policy/sentry-policy-search/pom.xml b/sentry-policy/sentry-policy-search/pom.xml index 2feb170e5..91fdbfa0b 100644 --- a/sentry-policy/sentry-policy-search/pom.xml +++ b/sentry-policy/sentry-policy-search/pom.xml @@ -21,7 +21,7 @@ limitations under the License. org.apache.sentry sentry-policy - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-policy-search diff --git a/sentry-provider/pom.xml b/sentry-provider/pom.xml index 74e3c0ff2..6726b3c22 100644 --- a/sentry-provider/pom.xml +++ b/sentry-provider/pom.xml @@ -22,7 +22,7 @@ limitations under the License. org.apache.sentry sentry - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-provider diff --git a/sentry-provider/sentry-provider-cache/pom.xml b/sentry-provider/sentry-provider-cache/pom.xml index c791eb206..5945c8a71 100644 --- a/sentry-provider/sentry-provider-cache/pom.xml +++ b/sentry-provider/sentry-provider-cache/pom.xml @@ -21,7 +21,7 @@ limitations under the License. org.apache.sentry sentry-provider - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-provider-cache diff --git a/sentry-provider/sentry-provider-common/pom.xml b/sentry-provider/sentry-provider-common/pom.xml index 4c778a2aa..d6487fffb 100644 --- a/sentry-provider/sentry-provider-common/pom.xml +++ b/sentry-provider/sentry-provider-common/pom.xml @@ -21,7 +21,7 @@ limitations under the License. org.apache.sentry sentry-provider - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-provider-common diff --git a/sentry-provider/sentry-provider-db/pom.xml b/sentry-provider/sentry-provider-db/pom.xml index 16f4e8e6b..d7d9c173a 100644 --- a/sentry-provider/sentry-provider-db/pom.xml +++ b/sentry-provider/sentry-provider-db/pom.xml @@ -21,7 +21,7 @@ limitations under the License. org.apache.sentry sentry-provider - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-provider-db diff --git a/sentry-provider/sentry-provider-file/pom.xml b/sentry-provider/sentry-provider-file/pom.xml index b834c8e4f..44276d5eb 100644 --- a/sentry-provider/sentry-provider-file/pom.xml +++ b/sentry-provider/sentry-provider-file/pom.xml @@ -21,7 +21,7 @@ limitations under the License. org.apache.sentry sentry-provider - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-provider-file diff --git a/sentry-tests/pom.xml b/sentry-tests/pom.xml index a18f912d7..a392001ee 100644 --- a/sentry-tests/pom.xml +++ b/sentry-tests/pom.xml @@ -20,7 +20,7 @@ limitations under the License. org.apache.sentry sentry - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-tests Sentry Tests diff --git a/sentry-tests/sentry-tests-hive/pom.xml b/sentry-tests/sentry-tests-hive/pom.xml index 651252abb..3602620fb 100644 --- a/sentry-tests/sentry-tests-hive/pom.xml +++ b/sentry-tests/sentry-tests-hive/pom.xml @@ -21,7 +21,7 @@ limitations under the License. org.apache.sentry sentry-tests - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-tests-hive Sentry Hive Tests diff --git a/sentry-tests/sentry-tests-solr/pom.xml b/sentry-tests/sentry-tests-solr/pom.xml index fdca89baa..75dc2009b 100644 --- a/sentry-tests/sentry-tests-solr/pom.xml +++ b/sentry-tests/sentry-tests-solr/pom.xml @@ -22,7 +22,7 @@ limitations under the License. org.apache.sentry sentry-tests - 1.4.0-incubating-SNAPSHOT + 1.4.0-incubating sentry-tests-solr From e6d58dca00702bbe17cb1054a9ff6b8db4005186 Mon Sep 17 00:00:00 2001 From: Vamsee Date: Fri, 25 Jul 2014 15:03:17 -0700 Subject: [PATCH 3/6] SENTRY-364: Bump up hive and hadoop versions from SNAPSHOT to released bits (Sravya Tirukkovalur via Vamsee Yarlagadda) --- pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index f615bd69b..78a09158f 100644 --- a/pom.xml +++ b/pom.xml @@ -67,8 +67,8 @@ limitations under the License. 3.0.1 10.4.2.0 1.2 - 0.12.0-cdh5.1.0-SNAPSHOT - 2.3.0-cdh5.1.0-SNAPSHOT + 0.12.0-cdh5.1.0 + 2.3.0-cdh5.1.0 1.4.1 11.0.2 4.9 @@ -79,9 +79,9 @@ limitations under the License. 1.2.1 1.6.1 4.8.0 - 4.4.0-cdh5.1.0-SNAPSHOT - 3.4.5-cdh5.1.0-SNAPSHOT - 0.12.0-cdh5.1.0-SNAPSHOT + 4.4.0-cdh5.1.0 + 3.4.5-cdh5.1.0 + 0.12.0-cdh5.1.0 From 1f1252e10681655728d71c716a6bbe860fa4bd80 Mon Sep 17 00:00:00 2001 From: Jarek Jarcec Cecho Date: Sat, 26 Jul 2014 07:42:54 -0700 Subject: [PATCH 4/6] SENTRY-363: CTAS from view is requiring select on underlying table (Sravya Tirukkovalur via Jarek Jarcec Cecho) --- .../binding/hive/HiveAuthzBindingHook.java | 6 +++- .../sentry/tests/e2e/hive/TestOperations.java | 28 +++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java index 44c0d2007..98820e11d 100644 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java @@ -396,7 +396,11 @@ private void authorizeWithHiveBindings(HiveSemanticAnalyzerHookContext context, outputHierarchy.add(dbHierarchy); for(ReadEntity readEntity:inputs) { - // If this is a UDF, then check whether its allowed to be executed + // skip the tables/view that are part of expanded view definition. + if (isChildTabForView(readEntity)) { + continue; + } + // If this is a UDF, then check whether its allowed to be executed // TODO: when we support execute privileges on UDF, this can be removed. if (isUDF(readEntity)) { if (isBuiltinUDF(readEntity)) { diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperations.java b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperations.java index ae6b33db7..5db8546f3 100644 --- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperations.java +++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/hive/TestOperations.java @@ -49,6 +49,7 @@ public class TestOperations extends AbstractTestWithStaticConfiguration { privileges.put("select_db1_tb1", "server=server1->db=db1->table=tb1->action=select"); privileges.put("insert_db1_tb1", "server=server1->db=db1->table=tb1->action=insert"); privileges.put("insert_db2_tb2", "server=server1->db=db2->table=tb2->action=insert"); + privileges.put("select_db1_view1", "server=server1->db=db1->table=view1->action=select"); } @@ -607,16 +608,37 @@ public void testCTAS() throws Exception { adminCreate(dbName, tableName); adminCreate("db2", null); + Connection connection = context.createConnection(ADMIN1); + Statement statement = context.createStatement(connection); + statement.execute("Use db1"); + statement.execute("create view view1 as select a from db1.tb1"); + statement.close(); + connection.close(); + policyFile .addPermissionsToRole("select_db1_tb1", privileges.get("select_db1_tb1")) + .addPermissionsToRole("select_db1_view1", privileges.get("select_db1_view1")) .addPermissionsToRole("all_db2", privileges.get("all_db2")) - .addRolesToGroup(USERGROUP1, "select_db1_tb1", "all_db2"); + .addRolesToGroup(USERGROUP1, "select_db1_tb1", "all_db2") + .addRolesToGroup(USERGROUP2, "select_db1_view1", "all_db2"); writePolicyFile(policyFile); - Connection connection = context.createConnection(USER1_1); - Statement statement = context.createStatement(connection); + connection = context.createConnection(USER1_1); + statement = context.createStatement(connection); statement.execute("Use db2"); statement.execute("create table tb2 as select a from db1.tb1" ); + context.assertSentrySemanticException(statement, "create table tb3 as select a from db1.view1", + semanticException); + statement.close(); + connection.close(); + + connection = context.createConnection(USER2_1); + statement = context.createStatement(connection); + statement.execute("Use db2"); + statement.execute("create table tb3 as select a from db1.view1" ); + context.assertSentrySemanticException(statement, "create table tb4 as select a from db1.tb1", + semanticException); + statement.close(); connection.close(); } From 73dcf89677b2764d6ad842cca85798d56c4be985 Mon Sep 17 00:00:00 2001 From: Sravya Tirukkovalur Date: Tue, 5 Aug 2014 13:27:20 -0700 Subject: [PATCH 5/6] SENTRY-369: Update changelog.txt, notice.txt, etc... for 1.4.0 release ( Tuong Truong via Sravya Tirukkovalur) --- CHANGELOG.txt | 214 ++++++++++++++++++++++++++++++++++++++++---------- NOTICE.txt | 2 +- 2 files changed, 172 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 759ea942e..962ac73d6 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,53 +1,181 @@ -Release Notes - Sentry - Version v1.2.0 +Release Notes - Sentry - Version 1.4.0 + +** Sub-task + * [SENTRY-97] - Create service configuration properties + * [SENTRY-129] - Implement Hive Sentry Authz DDL Task Factory + * [SENTRY-134] - Use BoneCP, add unique constraint to GROUP_NAME, and expose jdo/datanucleus properties + * [SENTRY-137] - Validate privilege scope in sentry service + * [SENTRY-138] - Use server timestamp for createTime for role, privilege and group + * [SENTRY-142] - Create database backed ProviderBackend + * [SENTRY-143] - Merge db_policy_store branch into master + * [SENTRY-153] - Add Hive e2e test with grant/revoke statements + * [SENTRY-156] - Support local privilege validation APIs + * [SENTRY-160] - Class to table mapping in package.jdo is incorrect + * [SENTRY-364] - Bump up hive and hadoop versions from SNAPSHOT to released bits + * [SENTRY-365] - Create release branch for 1.4.0 + * [SENTRY-369] - Update changelog.txt, notice.txt, etc... for 1.4.0 release + + ** Bug - * [SENTRY-15] - log4j.properties file under sentry-tests references the old access package - * [SENTRY-1] - use default on HiveServer2 fails with invalid privileges exception - * [SENTRY-2] - Code cleanup in various poms - * [ACCESS-8] - Log warning if authorization is not used with strong authentication - * [ACCESS-49] - Modify test cases to restrict LOAD from specific locations - * [ACCESS-140] - malformatted policy is permitted conditionally - * [ACCESS-164] - policy file doesn't check non-exist entity mapping - * [ACCESS-174] - access only throw first error message in HiveServer2 log, and ignore the rest - * [ACCESS-180] - per DB policy file usability issues - * [ACCESS-197] - Child authorizeable objects are not inheriting permissions from parent - * [ACCESS-201] - Bad error message in HiveAuthzBinding - * [ACCESS-203] - Update trunk version to 1.1 and update dependencies - * [ACCESS-230] - CREATE TABLE AS works even if user does not have DB-level access - * [ACCESS-231] - ALTER TABLE SET TBLPROPERTIES allows updates to tables even when the user doesn't have the right privileges - * [ACCESS-232] - The per-db policy fies can't be accessed if they are not in the same file system as the global policy file. - * [ACCESS-233] - The URI permission checks should append path separator before checking the parent path - * [ACCESS-235] - Format unqualified URI as DFS uri by default + * [SENTRY-118] - cast udf should be added to sentry udf whitelist for hive + * [SENTRY-131] - bin/sentry script doesn't find config-tool.sh under some circumstances + * [SENTRY-133] - Alter table create partition if not exists - results in error + * [SENTRY-161] - Sentry master branch is trying to download Hadoop tarball from nonexisting URL + * [SENTRY-162] - Cleanup DB store privilege metadata on Hive DDL statements + * [SENTRY-166] - Sentry does not accept URIs with an equals sign (=) in path. Fails with llegalArgumentException: Invalid key value + * [SENTRY-169] - JAAS login options not compatible with IBM JDK + * [SENTRY-172] - config-tool.sh is missing from master branch + * [SENTRY-174] - Sentry should not package hadoop, hive and other jars + * [SENTRY-175] - sentry script throws error for the dbstore service invocation + * [SENTRY-176] - Not able to read policy files on HDFS (Regression) + * [SENTRY-177] - Sentry Policy Service does not treat role names as case insensitive + * [SENTRY-178] - Poor performance for Sentry Policy Service as #of privileges is scaled up + * [SENTRY-181] - Add a test case for duplicate privileges + * [SENTRY-182] - Granting ALL privileges to table does not seem to do the right thing when using the SimpleDbPolicyProvider + * [SENTRY-183] - Sentry Policy Service goes into an unusable state when granting privileges. Subsequent access fail with a DataNucleusException: "Iteration request failed: SELECT ..." + * [SENTRY-186] - e2e tests for solr document-level security + * [SENTRY-187] - Use invariants rather than default for specification of update index level authorization + * [SENTRY-188] - Reduce the logging level during per-db policy loading + * [SENTRY-190] - Support for getting set of roles from ProviderBackend + * [SENTRY-191] - Sentry Policy Service should not require passing the RPC requestor's user/group information + * [SENTRY-192] - Convert solr doc-level e2e test to be based on roles rather than groups + * [SENTRY-194] - Sentry script should note use Hive script by default for service and tool execution + * [SENTRY-195] - Sentry schema tool can't process comments inside statement + * [SENTRY-200] - Remove sentry-provider dependencies on hive + * [SENTRY-201] - TestDatabaseProvider tests fail after Sentry schema tool was added. + * [SENTRY-202] - Sentry end to end tests which use ClusterDFS will need to explicitly add the policy file to HDFS + * [SENTRY-203] - Column name mismatch causes DataNucleus to throw exceptions + * [SENTRY-204] - Test cases extending SentryServiceIntegrationBase are failing + * [SENTRY-205] - Sentry throws Exception when trying to revoke Table level privileges + * [SENTRY-206] - Sentry distribution should include a template config file for the service + * [SENTRY-207] - Sentry script should return non-zero exist status in error conditions + * [SENTRY-209] - Empty list returned when calling listPrivilegesByRoleName + * [SENTRY-210] - Exception Thrown When Trying to grantRoleToGroup + * [SENTRY-212] - Restrict access to hive config property hive.sentry.active.role.set which is set by Sentry Hive binding + * [SENTRY-213] - Sentry schema tool doesn't handle sentry.javax.jdo.* properties + * [SENTRY-214] - Sentry Service does not allow the same Privilege to be associated to multiple Roles + * [SENTRY-217] - Add Insert and URI tests for Sentry DB provider + * [SENTRY-218] - Use defaults for user, password and driver in SchemaTool + * [SENTRY-219] - Sentry Cache Backend Provider initialization does not work as expected + * [SENTRY-220] - Trivial fix to SentrySchemaTool to set default driver + * [SENTRY-221] - Privilege scope is case sensitive + * [SENTRY-222] - Privileges are sometimes granted to the wrong roles + * [SENTRY-224] - Provider resource should not be required for DB provider backend + * [SENTRY-229] - SentrySchemaTool initSchema does not work with postgres 8.1and oracle + * [SENTRY-231] - Fix JDK 6 build + * [SENTRY-235] - Change tests in TestSentryServerWithoutKerberos to use new Sentry service APIs + * [SENTRY-236] - Sentry PolicyFile provider incorrectly logs error messages when reading policy file + * [SENTRY-237] - Support log4j configuration for Sentry service + * [SENTRY-238] - Denied Show roles and show role grant throw thrift exception + * [SENTRY-239] - Setup in TestDatabaseProvider is flaky + * [SENTRY-241] - Sentry GrantRevokeTask should fire the sentry failure look + * [SENTRY-243] - The operation type needs to be set in the grant/revoke task context for the failure hook + * [SENTRY-244] - Sentry deprecated properties do not work + * [SENTRY-245] - Fix failing db provider tests + * [SENTRY-246] - Load command does not seem to work with filter push down + * [SENTRY-247] - Go back to using filter push down once the bugs are fixed + * [SENTRY-248] - The sentry-provider-cache dependency is not correctly set + * [SENTRY-249] - "Use default" should be allowed for all the users even when using filter push down + * [SENTRY-250] - Create external table fails with filter push down + * [SENTRY-251] - PolicyProviderForTest.addPrivilege breaks in some cases + * [SENTRY-252] - Per db policy files based tests should be updated for dbprovider usage + * [SENTRY-253] - Creating external table seems to be failing when using provider db. + * [SENTRY-254] - Privilege name in provider db has a limit of length 128 which might be very low for long uris. + * [SENTRY-255] - Revoke on Server privilege fails + * [SENTRY-256] - Fix TestDbEndToEnd.testEndToEnd1 + * [SENTRY-257] - Upgrade master to use version 1.4.0-incubating-SNAPSHOT + * [SENTRY-259] - Implement Hive metastore plugin + * [SENTRY-260] - Add support to use DB2 as database for sentry metastore + * [SENTRY-262] - Updating patch for SENTRY-178 + * [SENTRY-263] - Remove usage of getHostString() from AbstractTestWithStaticConfiguration + * [SENTRY-266] - Implement _HOST substitution in principal + * [SENTRY-268] - Allow only granted roles to be set in "SET ROLE " + * [SENTRY-269] - Add a test case for Denied Alter table, should fire SentryOnFailureHook + * [SENTRY-271] - Test TestSentryServiceIntegration is flaky + * [SENTRY-272] - Test TestSentryStoreToAuthorizable.testUri is failing on comparing URI string + * [SENTRY-273] - org.apache.sentry.tests.e2e.dbprovider.TestDbUriPermissions is failing + * [SENTRY-274] - MySQL init scripts contains invalid comments + * [SENTRY-275] - Fix compilation error in SentryService + * [SENTRY-276] - SentryService tests are currently timing out + * [SENTRY-277] - Add Pig+HCat test for Metastore auth plugin + * [SENTRY-278] - TestSearchModelAuthorizables.testTooManyKV and TestDBModelAuthorizables.testTooManyKV fail + * [SENTRY-279] - Revert back using lowercase for uri label + * [SENTRY-280] - Sentry-202 missing changes + * [SENTRY-281] - Revoking a parent privilege should revoke all child privileges + * [SENTRY-282] - Select on DB should give privileges to query tables within it. + * [SENTRY-283] - Secure connection from HS2 to Sentry service fails + * [SENTRY-284] - Create test for creating external partition + * [SENTRY-285] - privilege->action=all is not same as privilege + * [SENTRY-288] - Dissable MetastoreBinding for test cases that do not require it + * [SENTRY-289] - Kerberos based connection from HS2 and Metastore to Sentry service fails + * [SENTRY-290] - Handle null pointer in SentryPolicyProcessor + * [SENTRY-294] - The Sentry service client should execute UGI privilege action by default + * [SENTRY-297] - Increase privilege_name to 4000 in mysql to be consistent with other dbs + * [SENTRY-299] - Partial Revoke Fails under certain conditions + * [SENTRY-300] - HiveAuthzBinding checks for Hive server2 config which is not available when using Sentry with Hive meta store server + * [SENTRY-301] - Sentry plugin fails access service from secure Hive Metastore + * [SENTRY-302] - Partial revoke on Table fails if both ALL and a SELECT/INSERT grant exists + * [SENTRY-304] - Limit on index key in MYSQL (innoDB ) is 767 bytes + * [SENTRY-305] - SHOW CURRENT ROLES shouldn't require admin privileges + * [SENTRY-306] - Fix grant all on table in db based provider + * [SENTRY-307] - Unqualified URIs should be reconstructed in a standard way + * [SENTRY-309] - Metastore binding should use fully qualified URI for validating alter table operations + * [SENTRY-310] - Make Hive operation to required privileges more granular + * [SENTRY-311] - Metastore plugin needs to be changed to updated privilege model + * [SENTRY-312] - Add 'decimal' and 'date' to default UDF whitelist + * [SENTRY-313] - Fix some uri failing tests + * [SENTRY-314] - Metastore plugin should verify the storage descriptor before referencing + * [SENTRY-315] - SHOW CURRENT ROLE fails if the one of the groups doesn't have any roles granted + * [SENTRY-317] - Fix TestDbOperations.testLoad test + * [SENTRY-319] - group names should be case sensitive. + * [SENTRY-321] - SentryMetastorePostEventListener should use sentry config to create SentryClient + * [SENTRY-332] - A role may got empty privilege, although the role have some privileges + * [SENTRY-336] - Fix test failures on real cluster + * [SENTRY-337] - When the parameter sentry.metastore.service.users isn't set or set empty, starting metastore will throw java.lang.NullPointerException + * [SENTRY-363] - CTAS from view is requiring select on underlying table + + ** Improvement - * [SENTRY-5] - Normalize the usernames used in the end to end tests - * [ACCESS-100] - ResourceAuthzProvider should ensure the subject name is non-null before doing the group lookup - * [ACCESS-157] - Access hard codes hive authentication method none - * [ACCESS-211] - Add maven profile for compiling access with upstream Apache hadoop/hive - * [ACCESS-221] - Restrict the URI access granted from a per-database policy file + * [SENTRY-106] - Make solr testing work against apache 4.7 version + * [SENTRY-193] - Add schematool for creating Sentry store schema from the SQL scripts + * [SENTRY-211] - Do the user: group lookup in the Sentry db policy server + * [SENTRY-258] - Increase field PRIVILEGE_NAME to 4000 characters to enable long URIs + * [SENTRY-293] - Create a new mvn cluster test profile for provider db tests + * [SENTRY-303] - Allow users to grant/revoke SELECT/INSERT to ALL tables in a Database + * [SENTRY-333] - Add conf directory to sentry distribution + * [SENTRY-361] - Sentry server should use sentry-site.xml in conf directory by default + +** New Feature + * [SENTRY-3] - Create a diagnostics tool for configuration validation + * [SENTRY-37] - Implement a DB backed policy store + * [SENTRY-115] - Give bindings the ability to access the group mappings + * [SENTRY-157] - Support filter pushdown in DB Store client to reduce data transfer from DB Store service + * [SENTRY-158] - Hive bindings should enable MR level ACLs for session user + * [SENTRY-165] - Implement createShowRolesTask() in SentryHiveAuthorizationTaskFactoryImpl + * [SENTRY-184] - Add Sentry service APIs to query roles and privileges + * [SENTRY-199] - Create tool that will convert policy file into into DB store + * [SENTRY-215] - SHOW GRANT ROLE xxx ON [SERVER, DATABASE, TABLE, URI] xxx + * [SENTRY-216] - Support SHOW CURRENT ROLES + ** Task - * [ACCESS-16] - Implement the test cases in the test plan - * [ACCESS-34] - Analyze Path Security - * [ACCESS-115] - Format all files using a consistent code style formatter for the project - * [ACCESS-122] - Remove context.close() mid-test - * [ACCESS-123] - Fix confusing communication mechanism to request if ANY access is exists - * [ACCESS-125] - TestUserManagement major issues - * [ACCESS-127] - TestSandboxOps Major issues - * [ACCESS-130] - TestMovingToProduction major issues - * [ACCESS-136] - TestCrossDbOps major issues - * [ACCESS-145] - TestMetadataObjectRetrieval major issues - * [ACCESS-147] - TestPrivilegeAtTransform major issues - * [ACCESS-149] - TestPrivilegesAtDatabaseScope major issues - * [ACCESS-152] - TestPrivilegesAtTableScope minor issues - * [ACCESS-166] - Policy Engine should do expanded validation of policy file - * [ACCESS-194] - Explore options for metastore access restriction - * [ACCESS-195] - Support username mapping at access level + * [SENTRY-159] - Convert AbstractSolrSentryTestBase to use MiniSolrCloudCluster rather than Lucene test hierarchy + * [SENTRY-164] - Missing implementation for HiveAuthorizationTaskFactory: createShowRolesTask() + * [SENTRY-230] - e2e test for doc level security to cover failure scenarios around Index level auth + * [SENTRY-356] - Apache Sentry 1.4.0 Release + + +** Test + * [SENTRY-223] - Add a test for updates with doc-level security + * [SENTRY-233] - Disable hdfs blockcache during solr e2e tests + * [SENTRY-261] - Improve test coverage for grant/revoke statements in Hive e2e tests + * [SENTRY-287] - Add test case for giving select privieleges on a table in a non default database + * [SENTRY-291] - Remove duplicate testSameGrantTwice + + -** Sub-task - * [ACCESS-101] - Implement more test cases regarding subquery - * [ACCESS-209] - be able to run e2e test in cluster mode - * [ACCESS-225] - Update master branch version to 1.2.0-SNAPSHOT diff --git a/NOTICE.txt b/NOTICE.txt index 7232bcb81..9fde41957 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,5 +1,5 @@ Apache Sentry -Copyright 2013 The Apache Software Foundation +Copyright 2013-2014 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). From ef78f65eec2297687cb7ef0e8af3ee281a26829c Mon Sep 17 00:00:00 2001 From: Anne Yu Date: Mon, 25 Jan 2016 13:32:01 -0800 Subject: [PATCH 6/6] SENTRY-1003: Support "reload" by updating the classpath of Sentry function aux jar path during runtime. (Dapeng Sun via Anne Yu) --- .../apache/sentry/binding/hive/HiveAuthzBindingHook.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java index 98820e11d..9baf421b3 100644 --- a/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java +++ b/sentry-binding/sentry-binding-hive/src/main/java/org/apache/sentry/binding/hive/HiveAuthzBindingHook.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.ql.HiveDriverFilterHookResultImpl; import org.apache.hadoop.hive.ql.exec.SentryGrantRevokeTask; import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.hooks.Entity; import org.apache.hadoop.hive.ql.hooks.Entity.Type; import org.apache.hadoop.hive.ql.hooks.Hook; @@ -192,7 +193,9 @@ public ASTNode preAnalyze(HiveSemanticAnalyzerHookContext context, ASTNode ast) case HiveParser.TOK_CREATEFUNCTION: String udfClassName = BaseSemanticAnalyzer.unescapeSQLString(ast.getChild(1).getText()); try { - CodeSource udfSrc = Class.forName(udfClassName).getProtectionDomain().getCodeSource(); + CodeSource udfSrc = + Class.forName(udfClassName, true, Utilities.getSessionSpecifiedClassLoader()) + .getProtectionDomain().getCodeSource(); if (udfSrc == null) { throw new SemanticException("Could not resolve the jar for UDF class " + udfClassName); } @@ -203,7 +206,7 @@ public ASTNode preAnalyze(HiveSemanticAnalyzerHookContext context, ASTNode ast) } udfURI = parseURI(udfSrc.getLocation().toString(), true); } catch (ClassNotFoundException e) { - throw new SemanticException("Error retrieving udf class", e); + throw new SemanticException("Error retrieving udf class:" + e.getMessage(), e); } // create/drop function is allowed with any database currDB = Database.ALL;