Memory estimation

gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ddl/catalog/GfxdSystemProcedures.java

@@ -27,10 +27,7 @@
 import java.sql.SQLException;
 import java.sql.Types;
 import java.util.*;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
+import java.util.concurrent.*;
 import javax.annotation.Nonnull;
 
 import com.gemstone.gemfire.DataSerializer;
@@ -41,6 +38,7 @@
 import com.gemstone.gemfire.cache.TransactionException;
 import com.gemstone.gemfire.cache.control.RebalanceOperation;
 import com.gemstone.gemfire.cache.control.ResourceManager;
+import com.gemstone.gemfire.cache.execute.FunctionException;
 import com.gemstone.gemfire.cache.execute.FunctionService;
 import com.gemstone.gemfire.distributed.DistributedMember;
 import com.gemstone.gemfire.distributed.internal.ServerLocation;
@@ -70,6 +68,7 @@
 import com.pivotal.gemfirexd.internal.engine.ddl.catalog.messages.GfxdSystemProcedureMessage;
 import com.pivotal.gemfirexd.internal.engine.ddl.resolver.GfxdPartitionByExpressionResolver;
 import com.pivotal.gemfirexd.internal.engine.ddl.wan.messages.AbstractGfxdReplayableMessage;
+import com.pivotal.gemfirexd.internal.engine.diag.SnappyTableStatsVTI;
 import com.pivotal.gemfirexd.internal.engine.distributed.AckResultCollector;
 import com.pivotal.gemfirexd.internal.engine.distributed.ByteArrayDataOutput;
 import com.pivotal.gemfirexd.internal.engine.distributed.GfxdDistributionAdvisor;
@@ -86,6 +85,7 @@
 import com.pivotal.gemfirexd.internal.engine.store.CustomRowsResultSet;
 import com.pivotal.gemfirexd.internal.engine.store.GemFireContainer;
 import com.pivotal.gemfirexd.internal.engine.store.GemFireStore;
+import com.pivotal.gemfirexd.internal.engine.ui.SnappyRegionStats;
 import com.pivotal.gemfirexd.internal.iapi.db.PropertyInfo;
 import com.pivotal.gemfirexd.internal.iapi.error.PublicAPI;
 import com.pivotal.gemfirexd.internal.iapi.error.StandardException;
@@ -121,8 +121,12 @@
 import com.pivotal.gemfirexd.internal.shared.common.sanity.SanityManager;
 import com.pivotal.gemfirexd.internal.snappy.LeadNodeSmartConnectorOpContext;
 import com.pivotal.gemfirexd.load.Import;
+import com.pivotal.gemfirexd.internal.engine.ui.SnappyRegionStatsCollectorFunction;
+import com.pivotal.gemfirexd.internal.engine.ui.SnappyRegionStatsCollectorResult;
+//import com.pivotal.gemfirexd.internal.engine.distributed;
 import io.snappydata.thrift.ServerType;
 import io.snappydata.thrift.internal.ClientBlob;
+import org.apache.commons.collections.bag.SynchronizedSortedBag;
 
 /**
  * GemFireXD built-in system procedures that will get executed on every
@@ -690,7 +694,7 @@ public boolean getNext(DataValueDescriptor[] template)
 
   private static final ResultColumnDescriptor[] encryptColumnInfo = {
       EmbedResultSetMetaData.getResultColumnDescriptor("ENCRYPTED_PASSWORD", Types.VARCHAR,
-          false, Limits.DB2_VARCHAR_MAXWIDTH),
+          false, Limits.DB2_VARCHAR_MAXWIDTH)
   };
 
 
@@ -3242,4 +3246,114 @@ private static void rollBackAndThrowSQLException(Connection conn,
     }
     throw se;
   }
-}
+
+  /**
+   * given a row/column table, method returns size of table with samplePercentage=100%,
+   * if given an external table and sample percentage, method creates a column table with
+   * sample percentage amount and extrapolates memory for loading whole(100% data)
+   * external table as column table.
+   *
+   * Cases(Exceptions) not handled:
+   * Case 1: given a row/column table and isExternalTable=TRUE then it doesn't
+   * show an error instead it creates another column table with sample amount.
+   *
+   * Case 2: given an external table with isExternalTable=FALSE or
+   * given any table which is not present  then it gives error but need to show
+   * an error message as table not found.
+   *
+   * @param tableName table name with schema(fully qualified table name)
+   * @param isExternalTable true for external table else false
+   * @param samplePercentage sampling percentage (Integer :0 to 100)
+   * @param tableSize returns resultset
+   * @throws SQLException
+   */
+  public static void GET_TABLE_SIZE(String tableName, Boolean isExternalTable, int samplePercentage, ResultSet[] tableSize) throws SQLException {
+    if (GemFireXDUtils.TraceSysProcedures) {
+      SanityManager.DEBUG_PRINT(GfxdConstants.TRACE_SYS_PROCEDURES,
+          "executing GET_TABLE_SIZE ");
+    }
+    List result = new java.util.ArrayList<>();
+    Set<DistributedMember> dataServers = GfxdMessage.getAllDataStores();
+    try {
+
+      if (dataServers != null && dataServers.size() > 0) {
+        Map args = new HashMap<String, String>();
+        if (isExternalTable) {
+          float sample = (float) samplePercentage / 100;
+          String sampleTable = tableName + "_sample";
+          Connection conn = getDefaultConn();
+          conn.createStatement().execute("drop table if exists " + sampleTable + " ;");
+          conn.createStatement().execute("create table " + sampleTable + " using column as(select * from " + tableName + " where rand() < " + sample + " );");
+          args.put("TABLE_NAME", sampleTable);
+        } else {
+          args.put("TABLE_NAME", tableName);
+          samplePercentage = 100;
+        }
+        result = (ArrayList<SnappyRegionStatsCollectorResult>) FunctionService.onMembers(dataServers)
+            .withArgs(args)
+            .execute(SnappyRegionStatsCollectorFunction.ID).getResult(1, TimeUnit.SECONDS);
+
+        List<SnappyRegionStats> stats = ((SnappyRegionStatsCollectorResult) result.get(0)).getRegionStats();
+        SnappyRegionStats statsResult = stats.get(0);
+        long sampleTableSize = statsResult.getTotalSize();
+        long sampleInMemoryTableSize = statsResult.getSizeInMemory();
+
+        final long totalTableSize = (sampleTableSize * 100) / samplePercentage;
+        final long inMemoryTableSize = (sampleInMemoryTableSize * 100) / samplePercentage;
+
+        Boolean resultFetched = true;
+        final CustomRowsResultSet.FetchDVDRows fetchRows =
+            new CustomRowsResultSet.FetchDVDRows() {
+              Boolean resultFetched = false;
+
+              @Override
+              public boolean getNext(DataValueDescriptor[] template)
+                  throws SQLException, StandardException {
+                if (!resultFetched) {
+                  if (totalTableSize > 104857.6) {
+                    float totalTableSizeInMB = (float) totalTableSize / 1048576;
+                    template[0].setValue(tableName + " = " + totalTableSizeInMB + " MB");
+                  } else if (totalTableSize > 102.4) {
+                    float totalTableSizeInKB = (float) totalTableSize / 1024;
+                    template[0].setValue(tableName + " = " + totalTableSizeInKB + " KB");
+                  } else {
+                    template[0].setValue(tableName + " = " + totalTableSize + " Bytes");
+                  }
+
+                  if (inMemoryTableSize > 104857.6) {
+                    float inMemoryTableSizeInMB = (float) inMemoryTableSize / 1048576;
+                    template[1].setValue(tableName + " = " + inMemoryTableSizeInMB + " MB");
+                  } else if (inMemoryTableSize > 102.4) {
+                    float inMemoryTableSizeInKB = (float) inMemoryTableSize / 1024;
+                    template[1].setValue(tableName + " = " + inMemoryTableSizeInKB + " KB");
+                  } else {
+                    template[1].setValue(tableName + " = " + inMemoryTableSize + " Bytes");
+                  }
+                  resultFetched = true;
+                  return true;
+                }
+                return false;
+              }
+            };
+        if (!result.isEmpty()) {
+          tableSize[0] = new CustomRowsResultSet(fetchRows, tableSizeInfo);
+        } else {
+          tableSize[0] = null;
+        }
+      }
+    } catch (InterruptedException e) {
+      e.printStackTrace();
+    } catch (StandardException se) {
+      se.printStackTrace();
+    }
+  }
+
+  private static final ResultColumnDescriptor[] tableSizeInfo = {
+      EmbedResultSetMetaData.getResultColumnDescriptor("TOTAL TABLE SIZE", Types.VARCHAR,
+          false, Limits.DB2_VARCHAR_MAXWIDTH),
+      EmbedResultSetMetaData.getResultColumnDescriptor("IN MEMORY TABLE SIZE", Types.VARCHAR,
+          false, Limits.DB2_VARCHAR_MAXWIDTH)
+  };
+
+
+}

gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/ui/SnappyRegionStatsCollectorFunction.java

@@ -31,6 +31,9 @@
 import com.gemstone.gemfire.management.internal.SystemManagementService;
 import com.pivotal.gemfirexd.internal.engine.Misc;
 import com.pivotal.gemfirexd.internal.engine.access.index.GfxdIndexManager;
+import com.pivotal.gemfirexd.internal.engine.ddl.DDLConflatable;
+import com.pivotal.gemfirexd.internal.engine.ddl.callbacks.CallbackProcedures;
+import com.pivotal.gemfirexd.internal.engine.distributed.utils.GemFireXDUtils;
 import com.pivotal.gemfirexd.internal.engine.store.GemFireContainer;
 import com.pivotal.gemfirexd.internal.iapi.error.StandardException;
 import com.pivotal.gemfirexd.tools.sizer.GemFireXDInstrumentation;
@@ -66,78 +69,120 @@ public void execute(FunctionContext context) {
     com.pivotal.gemfirexd.internal.snappy.CallbackFactoryProvider.getClusterCallbacks().
         publishColumnTableStats();
     SnappyRegionStatsCollectorResult result = new SnappyRegionStatsCollectorResult();
-    Map<String, SnappyRegionStats> cachBatchStats = new HashMap<>();
+    Map<String, SnappyRegionStats> cachedBatchStats = new HashMap<>();
     ArrayList<SnappyRegionStats> otherStats = new ArrayList<>();
+    try {
+      final SystemManagementService managementService =
+          (SystemManagementService) ManagementService.getManagementService(Misc.getGemFireCache());
 
+      if (context.getArguments() != null && context.getArguments() instanceof HashMap
+          && ((HashMap<String, String>) context.getArguments()).containsKey("TABLE_NAME")
+          && !((HashMap<String, String>) context.getArguments()).get("TABLE_NAME").isEmpty()) {
+        String tname = ((HashMap<String, String>) context.getArguments()).get("TABLE_NAME");
+        String[] names = tname.split("\\.");
+        if (names.length == 2) {
 
-    try {
-      List<GemFireContainer> containers = Misc.getMemStore().getAllContainers();
+          GemFireContainer container = CallbackProcedures.getContainerForTable(names[0], names[1]);
+          getSnappyRegionStats(otherStats, cachedBatchStats, container, managementService);
+          StoreCallbacks callback = CallbackFactoryProvider.getStoreCallbacks();
+          String columnBatchTableName = callback.columnBatchTableName(otherStats.get(0).getTableName());
+          try {
+            container = CallbackProcedures.getContainerForTable(names[0],
+                columnBatchTableName.substring(columnBatchTableName.indexOf(".") + 1));
+            getSnappyRegionStats(otherStats, cachedBatchStats, container, managementService);
+          } catch (StandardException se) {
+          }
 
-      final SystemManagementService managementService =
-          (SystemManagementService)ManagementService.getManagementService(Misc.getGemFireCache());
-
-      for (GemFireContainer container : containers) {
-        if (container.isApplicationTable()) {
-          LocalRegion r = container.getRegion();
-          if (managementService != null && r != null ) {
-            RegionMXBean bean = managementService.getLocalRegionMBean(r.getFullPath());
-            if (bean != null && !(r.getFullPath().startsWith(
-                "/" + Misc.SNAPPY_HIVE_METASTORE + '/'))) {
-              SnappyRegionStats dataCollector = collectDataFromBean(r, bean);
-              if (dataCollector.isColumnTable()) {
-                cachBatchStats.put(dataCollector.getTableName(), dataCollector);
-              } else {
-                otherStats.add(dataCollector);
+        } // TODO else get default or current schemaname and proceed
+      } else {
+        List<GemFireContainer> containers = Misc.getMemStore().getAllContainers();
+
+        for (GemFireContainer container : containers) {
+          //TODO replace block with method  getSnappyRegionStats()
+          if (container.isApplicationTable()) {
+            LocalRegion r = container.getRegion();
+            if (managementService != null && r != null) {
+              RegionMXBean bean = managementService.getLocalRegionMBean(r.getFullPath());
+              if (bean != null && !(r.getFullPath().startsWith(
+                  "/" + Misc.SNAPPY_HIVE_METASTORE + '/'))) {
+                SnappyRegionStats dataCollector = collectDataFromBean(r, bean);
+                if (dataCollector.isColumnTable()) {
+                  cachedBatchStats.put(dataCollector.getTableName(), dataCollector);
+                } else {
+                  otherStats.add(dataCollector);
+                }
               }
             }
-          }
           /*
           if(!LocalRegion.isMetaTable(r.getFullPath())){
             result.addAllIndexStat(getIndexStatForContainer(container));
           }
           */
+          }
         }
       }
 
 
-
       if (Misc.reservoirRegionCreated) {
         for (SnappyRegionStats tableStats : otherStats) {
           String tableName = tableStats.getTableName();
           StoreCallbacks callback = CallbackFactoryProvider.getStoreCallbacks();
           String columnBatchTableName = callback.columnBatchTableName(tableName);
-          if (cachBatchStats.containsKey(columnBatchTableName)) {
+          if (cachedBatchStats.containsKey(columnBatchTableName)) {
             String reservoirRegionName = Misc.getReservoirRegionNameForSampleTable("APP", tableName);
             PartitionedRegion pr = Misc.getReservoirRegionForSampleTable(reservoirRegionName);
             if (managementService != null && pr != null) {
               RegionMXBean reservoirBean = managementService.getLocalRegionMBean(pr.getFullPath());
               if (reservoirBean != null) {
                 SnappyRegionStats rStats = collectDataFromBeanImpl(pr, reservoirBean, true);
-                SnappyRegionStats cStats = cachBatchStats.get(columnBatchTableName);
-                cachBatchStats.put(columnBatchTableName, cStats.getCombinedStats(rStats));
+                SnappyRegionStats cStats = cachedBatchStats.get(columnBatchTableName);
+                cachedBatchStats.put(columnBatchTableName, cStats.getCombinedStats(rStats));
               }
             }
           }
         }
       }
 
       // Create one entry per Column Table by combining the results of row buffer and column table
+
       for (SnappyRegionStats tableStats : otherStats) {
         StoreCallbacks callback = CallbackFactoryProvider.getStoreCallbacks();
         String columnBatchTableName = callback.columnBatchTableName(tableStats.getTableName());
-        if (cachBatchStats.containsKey(columnBatchTableName)) {
-          result.addRegionStat(tableStats.getCombinedStats(cachBatchStats.get(columnBatchTableName)));
+        if (cachedBatchStats.containsKey(columnBatchTableName)) {
+          result.addRegionStat(tableStats.getCombinedStats(cachedBatchStats.get(columnBatchTableName)));
         } else {
           result.addRegionStat(tableStats);
         }
       }
     } catch (CacheClosedException ignored) {
+    } catch (StandardException ignored) {
     } finally {
       context.getResultSender().lastResult(result);
     }
 
   }
 
+  private void getSnappyRegionStats(ArrayList<SnappyRegionStats> otherStats,
+                                    Map<String, SnappyRegionStats> cacheBatchStats,
+                                    GemFireContainer container,
+                                    SystemManagementService managementService) {
+    if (container.isApplicationTable()) {
+      LocalRegion r = container.getRegion();
+      if (managementService != null && r != null) {
+        RegionMXBean bean = managementService.getLocalRegionMBean(r.getFullPath());
+        if (bean != null && !(r.getFullPath().startsWith(
+            "/" + Misc.SNAPPY_HIVE_METASTORE + '/'))) {
+          SnappyRegionStats dataCollector = collectDataFromBean(r, bean);
+          if (dataCollector.isColumnTable()) {
+            cacheBatchStats.put(dataCollector.getTableName(), dataCollector);
+          } else {
+            otherStats.add(dataCollector);
+          }
+        }
+      }
+    }
+  }
+
   private SnappyRegionStats collectDataFromBean(LocalRegion lr, RegionMXBean bean) {
     return collectDataFromBeanImpl(lr, bean, false);
   }

gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/impl/sql/catalog/GfxdDataDictionary.java

@@ -2092,6 +2092,19 @@ private void createGfxdSystemProcedures(TransactionController tc,
           arg_names, arg_types, 0, 1, RoutineAliasInfo.READS_SQL_DATA, null,
           newlyCreatedRoutines, tc, GFXD_SYS_PROC_CLASSNAME, false);
     }
+    {
+      // GET_TABLE_SIZE
+      String[] arg_names = new String[]{"TABLE_NAME","IS_EXTERNAL_TABLE","SAMPLE_PERCENTAGE"};
+      TypeDescriptor[] arg_types = new TypeDescriptor[]{
+          DataTypeDescriptor.getCatalogType(Types.VARCHAR),
+          DataTypeDescriptor.getCatalogType(Types.BOOLEAN),
+          DataTypeDescriptor.getCatalogType(Types.INTEGER)
+      };
+      super.createSystemProcedureOrFunction("GET_TABLE_SIZE", sysUUID,
+          arg_names, arg_types, 0, 1, RoutineAliasInfo.NO_SQL, null,
+          newlyCreatedRoutines, tc, GFXD_SYS_PROC_CLASSNAME, false);
+    }
+
   }
 
   @SuppressWarnings({ "unchecked", "rawtypes" })