diff --git a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java index 415f377..553acf3 100644 --- a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java +++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java @@ -66,89 +66,68 @@ public void generateBinaryMatrix() { } file.createNewFile(); - - FileWriter fw = new FileWriter(file.getAbsoluteFile()); - BufferedWriter bw = new BufferedWriter(fw); - bw.write("Num" + ","); - - // step 1: write first row of csv - List logIndexList = es.getIndexListWithPrefix( - props.getProperty(MudrodConstants.LOG_INDEX)); - - String[] logIndices = logIndexList.toArray(new String[0]); - String[] statictypeArray = new String[] { this.sessionStats }; - int docCount = es.getDocCount(logIndices, statictypeArray); - - LOG.info("{}: {}", this.sessionStats, docCount); - - if (docCount==0) { - bw.close(); - file.delete(); - return; - } - - SearchResponse sr = es.getClient() - .prepareSearch(logIndices) - .setTypes(statictypeArray) - .setQuery(QueryBuilders.matchAllQuery()) - .setSize(0) - .addAggregation(AggregationBuilders.terms("IPs") - .field("IP") - .size(docCount)) - .execute() - .actionGet(); - Terms ips = sr.getAggregations().get("IPs"); - List ipList = new ArrayList<>(); - for (Terms.Bucket entry : ips.getBuckets()) { - // filter - if (entry.getDocCount() > Integer.parseInt(props.getProperty(MudrodConstants.QUERY_MIN))) { - // out less active users/ips - ipList.add(entry.getKey().toString()); + try (FileWriter fw = new FileWriter(file.getAbsoluteFile()); BufferedWriter bw = new BufferedWriter(fw);) { + bw.write("Num" + ","); + // step 1: write first row of csv + List logIndexList = es.getIndexListWithPrefix(props.getProperty(MudrodConstants.LOG_INDEX)); + + String[] logIndices = logIndexList.toArray(new String[0]); + String[] statictypeArray = new String[] { this.sessionStats }; + int docCount = es.getDocCount(logIndices, statictypeArray); + + LOG.info("{}: {}", this.sessionStats, docCount); + + if (docCount==0) + { + bw.close(); + file.delete(); + return; } - } - bw.write(String.join(",", ipList) + "\n"); - - // step 2: step the rest rows of csv - SearchRequestBuilder sr2Builder = es.getClient() - .prepareSearch(logIndices) - .setTypes(statictypeArray) - .setQuery(QueryBuilders.matchAllQuery()) - .setSize(0) - .addAggregation(AggregationBuilders.terms("KeywordAgg") - .field("keywords") - .size(docCount) - .subAggregation(AggregationBuilders.terms("IPAgg") - .field("IP") - .size(docCount))); - - SearchResponse sr2 = sr2Builder.execute().actionGet(); - Terms keywords = sr2.getAggregations().get("KeywordAgg"); - - for (Terms.Bucket keyword : keywords.getBuckets()) { - - Map ipMap = new HashMap<>(); - Terms ipAgg = keyword.getAggregations().get("IPAgg"); - - int distinctUser = ipAgg.getBuckets().size(); - if (distinctUser >= Integer.parseInt(props.getProperty(MudrodConstants.QUERY_MIN))) { - bw.write(keyword.getKey() + ","); - for (Terms.Bucket IP : ipAgg.getBuckets()) { - - ipMap.put(IP.getKey().toString(), 1); + + SearchResponse sr = es.getClient().prepareSearch(logIndices).setTypes(statictypeArray).setQuery(QueryBuilders.matchAllQuery()).setSize(0) + .addAggregation(AggregationBuilders.terms("IPs").field("IP").size(docCount)).execute().actionGet(); + Terms ips = sr.getAggregations().get("IPs"); + List ipList = new ArrayList<>(); + for (Terms.Bucket entry : ips.getBuckets()) { + if (entry.getDocCount() > Integer.parseInt(props.getProperty(MudrodConstants.QUERY_MIN))) { // filter + // out + // less + // active users/ips + ipList.add(entry.getKey().toString()); } - for (String anIpList : ipList) { - if (ipMap.containsKey(anIpList)) { - bw.write(ipMap.get(anIpList) + ","); - } else { - bw.write("0,"); + } + bw.write(String.join(",", ipList) + "\n"); + + // step 2: step the rest rows of csv + SearchRequestBuilder sr2Builder = es.getClient().prepareSearch(logIndices).setTypes(statictypeArray).setQuery(QueryBuilders.matchAllQuery()).setSize(0) + .addAggregation(AggregationBuilders.terms("KeywordAgg").field("keywords").size(docCount).subAggregation(AggregationBuilders.terms("IPAgg").field("IP").size(docCount))); + + SearchResponse sr2 = sr2Builder.execute().actionGet(); + Terms keywords = sr2.getAggregations().get("KeywordAgg"); + + for (Terms.Bucket keyword : keywords.getBuckets()) { + + Map ipMap = new HashMap<>(); + Terms ipAgg = keyword.getAggregations().get("IPAgg"); + + int distinctUser = ipAgg.getBuckets().size(); + if (distinctUser >= Integer.parseInt(props.getProperty(MudrodConstants.QUERY_MIN))) { + bw.write(keyword.getKey() + ","); + for (Terms.Bucket IP : ipAgg.getBuckets()) { + + ipMap.put(IP.getKey().toString(), 1); } + for (String anIpList : ipList) { + if (ipMap.containsKey(anIpList)) { + bw.write(ipMap.get(anIpList) + ","); + } else { + bw.write("0,"); + } + } + bw.write("\n"); } - bw.write("\n"); } } - - bw.close(); - fw.close(); } catch (IOException e) { e.printStackTrace(); } @@ -160,4 +139,4 @@ public Object execute(Object o) { return null; } -} +} \ No newline at end of file