From b537dbbdf39514392d2c825778a90cb9c70b3c07 Mon Sep 17 00:00:00 2001 From: quintinali Date: Mon, 21 May 2018 08:39:40 -0400 Subject: [PATCH 1/3] fixed bugs in TrainingImporter.java 1. Use try-with-resources to create "BufferedReader" 2. Store the value returned from "readLine" instead of throwing it away. --- .../ssearch/ranking/TrainingImporter.java | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java index ff55c85..95cdddf 100644 --- a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java +++ b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java @@ -63,29 +63,31 @@ public void addMapping() { /** * Method of importing training set in to Elasticsearch * - * @param dataFolder the path to the traing set - * @throws IOException IOException + * @param dataFolder + * the path to the traing set + * @throws IOException + * IOException */ public void importTrainingSet(String dataFolder) throws IOException { es.createBulkProcessor(); File[] files = new File(dataFolder).listFiles(); for (File file : files) { - BufferedReader br = new BufferedReader(new FileReader(file.getAbsolutePath())); - br.readLine(); - String line = br.readLine(); - while (line != null) { - String[] list = line.split(","); - String query = file.getName().replace(".csv", ""); - if (list.length > 0) { - IndexRequest ir = new IndexRequest(props.getProperty(MudrodConstants.ES_INDEX_NAME), "trainingranking") - .source(jsonBuilder().startObject().field("query", query).field("dataID", list[0]).field("label", list[list.length - 1]).endObject()); - es.getBulkProcessor().add(ir); + try (BufferedReader br = new BufferedReader(new FileReader(file.getAbsolutePath()))) { + String line = null; + line = br.readLine(); // cvs header + while ((line = br.readLine()) != null) { + String[] list = line.split(","); + String query = file.getName().replace(".csv", ""); + if (list.length > 0) { + IndexRequest ir = new IndexRequest(props.getProperty(MudrodConstants.ES_INDEX_NAME), "trainingranking") + .source(jsonBuilder().startObject().field("query", query).field("dataID", list[0]).field("label", list[list.length - 1]).endObject()); + es.getBulkProcessor().add(ir); + } } - line = br.readLine(); } - br.close(); + + es.destroyBulkProcessor(); } - es.destroyBulkProcessor(); } } From 013b078eaea7a21b461240773b61425c4fcd787a Mon Sep 17 00:00:00 2001 From: quintinali Date: Wed, 30 May 2018 11:16:08 -0400 Subject: [PATCH 2/3] Fixed bugs: delete bulkprocess after the while function --- .../apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java index 95cdddf..0c7b853 100644 --- a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java +++ b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java @@ -17,6 +17,7 @@ import org.apache.sdap.mudrod.driver.ESDriver; import org.apache.sdap.mudrod.driver.SparkDriver; import org.apache.sdap.mudrod.main.MudrodConstants; +import org.apache.sdap.mudrod.main.MudrodEngine; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -86,8 +87,8 @@ public void importTrainingSet(String dataFolder) throws IOException { } } } - - es.destroyBulkProcessor(); } + + es.destroyBulkProcessor(); } } From 66dff1a58b94e0fd71c8db93ab70428b44dfd88f Mon Sep 17 00:00:00 2001 From: quintinali Date: Thu, 7 Jun 2018 19:11:51 -0400 Subject: [PATCH 3/3] remove useless importer --- .../apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java index 0c7b853..3f87f20 100644 --- a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java +++ b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java @@ -17,11 +17,9 @@ import org.apache.sdap.mudrod.driver.ESDriver; import org.apache.sdap.mudrod.driver.SparkDriver; import org.apache.sdap.mudrod.main.MudrodConstants; -import org.apache.sdap.mudrod.main.MudrodEngine; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryBuilders; - import java.io.BufferedReader; import java.io.File; import java.io.FileReader;