From 8c94deead3fd7edaea1ebf25ec76a18604d9155d Mon Sep 17 00:00:00 2001 From: quintinali Date: Mon, 21 May 2018 08:25:36 -0400 Subject: [PATCH 1/2] fixed bugs in HistoryGenerator Use try-with-resource to create bufferwriter --- .../mudrod/weblog/pre/HistoryGenerator.java | 123 +++++++++--------- 1 file changed, 60 insertions(+), 63 deletions(-) diff --git a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java index 1969e3e..e50dd4f 100644 --- a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java +++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java @@ -67,80 +67,77 @@ public void generateBinaryMatrix() { file.createNewFile(); - FileWriter fw = new FileWriter(file.getAbsoluteFile()); - BufferedWriter bw = new BufferedWriter(fw); - bw.write("Num" + ","); - - // step 1: write first row of csv - List logIndexList = es.getIndexListWithPrefix(props.getProperty(MudrodConstants.LOG_INDEX)); - - String[] logIndices = logIndexList.toArray(new String[0]); - String[] statictypeArray = new String[] { this.sessionStats }; - int docCount = es.getDocCount(logIndices, statictypeArray); - - LOG.info("{}: {}", this.sessionStats, docCount); - - if (docCount==0) - { - bw.close(); - file.delete(); - return; - } - - SearchResponse sr = es.getClient().prepareSearch(logIndices).setTypes(statictypeArray).setQuery(QueryBuilders.matchAllQuery()).setSize(0) - .addAggregation(AggregationBuilders.terms("IPs").field("IP").size(docCount)).execute().actionGet(); - Terms ips = sr.getAggregations().get("IPs"); - List ipList = new ArrayList<>(); - for (Terms.Bucket entry : ips.getBuckets()) { - if (entry.getDocCount() > Integer.parseInt(props.getProperty(MudrodConstants.QUERY_MIN))) { // filter - // out - // less - // active users/ips - ipList.add(entry.getKey().toString()); + try (FileWriter fw = new FileWriter(file.getAbsoluteFile()); + BufferedWriter bw = new BufferedWriter(fw);) { + // Process the input and produce the output + bw.write("Num" + ","); + + // step 1: write first row of csv + List logIndexList = es.getIndexListWithPrefix(props.getProperty(MudrodConstants.LOG_INDEX)); + + String[] logIndices = logIndexList.toArray(new String[0]); + String[] statictypeArray = new String[] { this.sessionStats }; + int docCount = es.getDocCount(logIndices, statictypeArray); + + LOG.info("{}: {}", this.sessionStats, docCount); + + if (docCount==0) + { + file.delete(); + return; } - } - bw.write(String.join(",", ipList) + "\n"); - - // step 2: step the rest rows of csv - SearchRequestBuilder sr2Builder = es.getClient().prepareSearch(logIndices).setTypes(statictypeArray).setQuery(QueryBuilders.matchAllQuery()).setSize(0) - .addAggregation(AggregationBuilders.terms("KeywordAgg").field("keywords").size(docCount).subAggregation(AggregationBuilders.terms("IPAgg").field("IP").size(docCount))); - - SearchResponse sr2 = sr2Builder.execute().actionGet(); - Terms keywords = sr2.getAggregations().get("KeywordAgg"); - - for (Terms.Bucket keyword : keywords.getBuckets()) { - - Map ipMap = new HashMap<>(); - Terms ipAgg = keyword.getAggregations().get("IPAgg"); - - int distinctUser = ipAgg.getBuckets().size(); - if (distinctUser >= Integer.parseInt(props.getProperty(MudrodConstants.QUERY_MIN))) { - bw.write(keyword.getKey() + ","); - for (Terms.Bucket IP : ipAgg.getBuckets()) { - - ipMap.put(IP.getKey().toString(), 1); + + SearchResponse sr = es.getClient().prepareSearch(logIndices).setTypes(statictypeArray).setQuery(QueryBuilders.matchAllQuery()).setSize(0) + .addAggregation(AggregationBuilders.terms("IPs").field("IP").size(docCount)).execute().actionGet(); + Terms ips = sr.getAggregations().get("IPs"); + List ipList = new ArrayList<>(); + for (Terms.Bucket entry : ips.getBuckets()) { + if (entry.getDocCount() > Integer.parseInt(props.getProperty(MudrodConstants.QUERY_MIN))) { // filter + // out + // less + // active users/ips + ipList.add(entry.getKey().toString()); } - for (String anIpList : ipList) { - if (ipMap.containsKey(anIpList)) { - bw.write(ipMap.get(anIpList) + ","); - } else { - bw.write("0,"); + } + bw.write(String.join(",", ipList) + "\n"); + + // step 2: step the rest rows of csv + SearchRequestBuilder sr2Builder = es.getClient().prepareSearch(logIndices).setTypes(statictypeArray).setQuery(QueryBuilders.matchAllQuery()).setSize(0) + .addAggregation(AggregationBuilders.terms("KeywordAgg").field("keywords").size(docCount).subAggregation(AggregationBuilders.terms("IPAgg").field("IP").size(docCount))); + + SearchResponse sr2 = sr2Builder.execute().actionGet(); + Terms keywords = sr2.getAggregations().get("KeywordAgg"); + + for (Terms.Bucket keyword : keywords.getBuckets()) { + + Map ipMap = new HashMap<>(); + Terms ipAgg = keyword.getAggregations().get("IPAgg"); + + int distinctUser = ipAgg.getBuckets().size(); + if (distinctUser >= Integer.parseInt(props.getProperty(MudrodConstants.QUERY_MIN))) { + bw.write(keyword.getKey() + ","); + for (Terms.Bucket IP : ipAgg.getBuckets()) { + ipMap.put(IP.getKey().toString(), 1); + } + for (String anIpList : ipList) { + if (ipMap.containsKey(anIpList)) { + bw.write(ipMap.get(anIpList) + ","); + } else { + bw.write("0,"); + } } + bw.write("\n"); } - bw.write("\n"); } - } - - bw.close(); - } catch (IOException e) { + } + } + catch (IOException e) { e.printStackTrace(); } - } @Override public Object execute(Object o) { return null; } - } From 337e44204f6e1f1ef99320a99001eb8e2dae54b1 Mon Sep 17 00:00:00 2001 From: quintinali Date: Thu, 7 Jun 2018 18:58:29 -0400 Subject: [PATCH 2/2] fixed bugs --- .../mudrod/weblog/pre/HistoryGenerator.java | 49 +++---------------- 1 file changed, 8 insertions(+), 41 deletions(-) diff --git a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java index ccfe1d2..553acf3 100644 --- a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java +++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java @@ -66,12 +66,8 @@ public void generateBinaryMatrix() { } file.createNewFile(); - -try (FileWriter fw = new FileWriter(file.getAbsoluteFile()); - BufferedWriter bw = new BufferedWriter(fw);) { - // Process the input and produce the output + try (FileWriter fw = new FileWriter(file.getAbsoluteFile()); BufferedWriter bw = new BufferedWriter(fw);) { bw.write("Num" + ","); - // step 1: write first row of csv List logIndexList = es.getIndexListWithPrefix(props.getProperty(MudrodConstants.LOG_INDEX)); @@ -83,6 +79,7 @@ public void generateBinaryMatrix() { if (docCount==0) { + bw.close(); file.delete(); return; } @@ -97,37 +94,6 @@ public void generateBinaryMatrix() { // less // active users/ips ipList.add(entry.getKey().toString()); - } - } - bw.write(String.join(",", ipList) + "\n"); - - // step 2: step the rest rows of csv - SearchRequestBuilder sr2Builder = es.getClient() - .prepareSearch(logIndices) - .setTypes(statictypeArray) - .setQuery(QueryBuilders.matchAllQuery()) - .setSize(0) - .addAggregation(AggregationBuilders.terms("KeywordAgg") - .field("keywords") - .size(docCount) - .subAggregation(AggregationBuilders.terms("IPAgg") - .field("IP") - .size(docCount))); - - SearchResponse sr2 = sr2Builder.execute().actionGet(); - Terms keywords = sr2.getAggregations().get("KeywordAgg"); - - for (Terms.Bucket keyword : keywords.getBuckets()) { - - Map ipMap = new HashMap<>(); - Terms ipAgg = keyword.getAggregations().get("IPAgg"); - - int distinctUser = ipAgg.getBuckets().size(); - if (distinctUser >= Integer.parseInt(props.getProperty(MudrodConstants.QUERY_MIN))) { - bw.write(keyword.getKey() + ","); - for (Terms.Bucket IP : ipAgg.getBuckets()) { - - ipMap.put(IP.getKey().toString(), 1); } } bw.write(String.join(",", ipList) + "\n"); @@ -148,6 +114,7 @@ public void generateBinaryMatrix() { if (distinctUser >= Integer.parseInt(props.getProperty(MudrodConstants.QUERY_MIN))) { bw.write(keyword.getKey() + ","); for (Terms.Bucket IP : ipAgg.getBuckets()) { + ipMap.put(IP.getKey().toString(), 1); } for (String anIpList : ipList) { @@ -160,16 +127,16 @@ public void generateBinaryMatrix() { bw.write("\n"); } } - } - } - catch (IOException e) { - + } + } catch (IOException e) { e.printStackTrace(); } + } @Override public Object execute(Object o) { return null; } -} + +} \ No newline at end of file