From d0e1345540d9a3c088573c612b928d3346111f21 Mon Sep 17 00:00:00 2001
From: quintinali <tina.yuner@gmail.com>
Date: Thu, 8 Nov 2018 14:43:40 -0500
Subject: [PATCH 1/9] add embedded elasticsearch engine

---
 .../driver/EmbeddedElasticsearchServer.java   |  78 +++++
 .../mudrod/driver/PluginConfigurableNode.java |  14 +
 .../AbstractElasticsearchIntegrationTest.java |  37 ++
 .../apache/sdap/mudrod/main/ESDriverTest.java | 324 ++++++++++++++++++
 .../weblog/structure/RequestUrlTest.java      |  76 ++++
 .../weblog/structure/TestApacheAccessLog.java |   1 +
 .../weblog/structure/log/GeoIpTest.java       |  18 +
 7 files changed, 548 insertions(+)
 create mode 100644 core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java
 create mode 100644 core/src/test/java/org/apache/sdap/mudrod/driver/PluginConfigurableNode.java
 create mode 100644 core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java
 create mode 100644 core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java
 create mode 100644 core/src/test/java/org/apache/sdap/mudrod/weblog/structure/RequestUrlTest.java
 create mode 100644 core/src/test/java/org/apache/sdap/mudrod/weblog/structure/log/GeoIpTest.java

diff --git a/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java b/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java
new file mode 100644
index 0000000..c4ac65e
--- /dev/null
+++ b/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java
@@ -0,0 +1,78 @@
+package org.apache.sdap.mudrod.driver;
+
+import org.apache.commons.io.FileUtils;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.node.NodeValidationException;
+import org.elasticsearch.transport.Netty3Plugin;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+
+/**
+ * Example implementation of an embedded elasticsearch server.
+ *
+ * @author Felix Müller
+ */
+public class EmbeddedElasticsearchServer {
+
+  private static final String DEFAULT_DATA_DIRECTORY = "target/elasticsearch-data";
+
+  private Node node;
+  private final String dataDirectory;
+
+  public EmbeddedElasticsearchServer() {
+    this(DEFAULT_DATA_DIRECTORY);
+  }
+
+  public EmbeddedElasticsearchServer(String dataDirectory) {
+    this.dataDirectory = dataDirectory;
+
+    Settings.Builder settingsBuilder = Settings.builder();
+    settingsBuilder.put("http.type", "netty3");
+    settingsBuilder.put("transport.type", "netty3");
+    settingsBuilder.put("network.host", "127.0.0.1");
+    settingsBuilder.put("transport.tcp.port", 9300);
+
+    settingsBuilder.put("cluster.name", "MurdorES").put("http.enabled", "false").put("path.data", dataDirectory).put("path.home", "/");
+      
+    Settings settings = settingsBuilder.build();
+    Collection plugins = Arrays.asList(Netty3Plugin.class);
+    node = null;
+    try {
+      node = new PluginConfigurableNode(settings, plugins).start();
+      System.out.println(node.toString());
+    } catch (NodeValidationException e) {
+      // TODO Auto-generated catch block
+      e.printStackTrace();
+    }
+    
+    System.out.println(node.getNodeEnvironment().nodeId());
+   
+  }
+
+  public Client getClient() {
+    return node.client();
+  }
+
+  public void shutdown() {
+    /*try {
+      node.close();
+    } catch (IOException e) {
+      // TODO Auto-generated catch block
+      e.printStackTrace();
+    }
+    deleteDataDirectory();*/
+  }
+
+  private void deleteDataDirectory() {
+    try {
+      FileUtils.deleteDirectory(new File(dataDirectory));
+    } catch (IOException e) {
+      throw new RuntimeException("Could not delete data directory of embedded elasticsearch server", e);
+    }
+  }
+}
diff --git a/core/src/test/java/org/apache/sdap/mudrod/driver/PluginConfigurableNode.java b/core/src/test/java/org/apache/sdap/mudrod/driver/PluginConfigurableNode.java
new file mode 100644
index 0000000..c59ee2f
--- /dev/null
+++ b/core/src/test/java/org/apache/sdap/mudrod/driver/PluginConfigurableNode.java
@@ -0,0 +1,14 @@
+package org.apache.sdap.mudrod.driver;
+
+import java.util.Collection;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.node.internal.InternalSettingsPreparer;
+import org.elasticsearch.plugins.Plugin;
+
+public class PluginConfigurableNode extends Node {
+    public PluginConfigurableNode(Settings settings, Collection<Class<? extends Plugin>> classpathPlugins) {
+        super(InternalSettingsPreparer.prepareEnvironment(settings, null), classpathPlugins);
+    }
+}
\ No newline at end of file
diff --git a/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java b/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java
new file mode 100644
index 0000000..cbadaa3
--- /dev/null
+++ b/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java
@@ -0,0 +1,37 @@
+package org.apache.sdap.mudrod.main;
+
+import org.apache.sdap.mudrod.driver.EmbeddedElasticsearchServer;
+import org.apache.sdap.mudrod.driver.EmbeddedElasticsearchServer;
+import org.elasticsearch.client.Client;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+/**
+ * This is a helper class the starts an embedded elasticsearch server
+ * for each test.
+ *
+ * @author Felix Müller
+ */
+public abstract class AbstractElasticsearchIntegrationTest {
+
+    private static EmbeddedElasticsearchServer embeddedElasticsearchServer;
+
+    @BeforeClass
+    public static void startEmbeddedElasticsearchServer() {
+        embeddedElasticsearchServer = new EmbeddedElasticsearchServer();
+    }
+
+    @AfterClass
+    public static void shutdownEmbeddedElasticsearchServer() {
+        //embeddedElasticsearchServer.shutdown();
+    }
+
+    /**
+     * By using this method you can access the embedded server.
+     *//*
+    protected Client getClient() {
+        return embeddedElasticsearchServer.getClient();
+    }*/
+}
diff --git a/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java b/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java
new file mode 100644
index 0000000..e61103f
--- /dev/null
+++ b/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java
@@ -0,0 +1,324 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you 
+ * may not use this file except in compliance with the License. 
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.main;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.sdap.mudrod.main.MudrodConstants;
+import org.apache.sdap.mudrod.main.MudrodEngine;
+import org.apache.sdap.mudrod.driver.ESDriver;
+import org.apache.sdap.mudrod.driver.EmbeddedElasticsearchServer;
+import org.elasticsearch.client.Client;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.action.bulk.BulkProcessor;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+
+public class ESDriverTest extends AbstractElasticsearchIntegrationTest{
+
+  private static final Logger LOG = LoggerFactory.getLogger(ESDriver.class);
+  protected static final String ES_SETTINGS = "elastic_settings.json";
+  protected static final String ES_MAPPINGS = "elastic_mappings.json";
+  
+  static ESDriver es = null;
+  static MudrodEngine mudrodEngine = null;
+  @BeforeClass
+  public static void setUp(){
+//	 Map<String, String> env = System.getenv();
+//	 env.put(MudrodConstants.MUDROD_CONFIG, "dexuantest");	 
+	 mudrodEngine = new MudrodEngine();
+	 es = new ESDriver(mudrodEngine.loadConfig());	 
+
+  }
+
+  @AfterClass
+  public static void tearDown(){
+     //es.destroyBulkProcessor();
+     //es.close();
+  }
+  
+  @Test
+  public void testESDriverProperties() {
+   
+    Client client = es.getClient();
+    assert client != null;
+  }
+
+  @Test
+  public void testCreateBulkProcessor() {
+    es.createBulkProcessor();
+    BulkProcessor processor = es.getBulkProcessor();
+    assert processor != null;
+  }
+
+  @Test
+  public void testDestroyBulkProcessor() {
+    es.createBulkProcessor();
+    es.destroyBulkProcessor();
+
+    BulkProcessor processor = es.getBulkProcessor();
+    assert processor == null;
+  }
+
+  @Test
+  public void testPutMapping() {  
+
+    InputStream settingsStream = getClass().getClassLoader().getResourceAsStream(ES_SETTINGS);
+    InputStream mappingsStream = getClass().getClassLoader().getResourceAsStream(ES_MAPPINGS);
+
+    JSONObject settingsJSON = null;
+    JSONObject mappingJSON = null;
+
+    Properties props = mudrodEngine.loadConfig();
+    try {
+      settingsJSON = new JSONObject(IOUtils.toString(settingsStream));
+    } catch (JSONException | IOException e1) {
+      LOG.error("Error reading Elasticsearch settings!", e1);
+    }
+
+    try {
+      mappingJSON = new JSONObject(IOUtils.toString(mappingsStream));
+    } catch (JSONException | IOException e1) {
+      LOG.error("Error reading Elasticsearch mappings!", e1);
+    }
+
+    try {
+      if (settingsJSON != null && mappingJSON != null) {
+        es.putMapping(props.getProperty(MudrodConstants.ES_INDEX_NAME), settingsJSON.toString(), mappingJSON.toString());
+      }
+    } catch (IOException e) {
+      LOG.error("Error entering Elasticsearch Mappings!", e);
+    }
+  }
+
+  @Test
+  public void testCustomAnalyzingStringString() {
+       
+    String str = "temp";
+    try {
+      String res = es.customAnalyzing("mudrod", str);
+      assert res != "";
+      assert res != null;
+    } catch (InterruptedException | ExecutionException e) {
+      LOG.error("Error!", e);
+    }
+  }
+
+  @Test
+  public void testCustomAnalyzingStringStringString() {   
+    String str = "temp";
+
+    try {
+      String res = es.customAnalyzing("mudrod", "cody", str);
+      assert res != "";
+      assert res != null;
+    } catch (InterruptedException | ExecutionException e) {
+      LOG.error("Error!", e);
+    }
+  }
+
+  @Test
+  public void testCustomAnalyzingStringListOfString() {   
+
+    List<String> customlist = new ArrayList<>();
+    customlist.add("string_a");
+    customlist.add("string_b");
+
+    try {
+      List<String> res = es.customAnalyzing("mudrod", customlist);
+      assert !res.isEmpty();
+    } catch (InterruptedException | ExecutionException e) {
+      LOG.error("Error!", e);
+    }
+  }
+
+  @Test
+  public void testDeleteAllByQuery() {    
+    es.deleteAllByQuery("mudrod", "MetadataLinkage", QueryBuilders.matchAllQuery());
+
+    // String res = es.searchByQuery("mudrod", "MetadataLinkage",
+    // QueryBuilders.matchAllQuery());
+    // assert res == ""||res == null;
+  }
+
+  @Test
+  public void testDeleteType() {
+        
+    es.deleteType("mudrod", "MetadataLinkage");
+  }
+
+  @Test
+  public void testGetTypeListWithPrefix() {
+    
+    es.getTypeListWithPrefix("podaacsession", "sessionstats");
+  }
+
+  @Test
+  public void testGetIndexListWithPrefix() {    
+
+    List<String> res = es.getIndexListWithPrefix("podaacsession");
+    assert !res.isEmpty();
+  }
+
+  @Test
+  public void testSearchByQueryStringStringString() {
+
+    try {
+      String res = es.searchByQuery("mudrod", "MetadataLinkage", "temp");
+      assert res != null;
+    } catch (IOException | InterruptedException | ExecutionException e) {
+      LOG.error("Error!", e);
+    }
+  }
+
+  @Test
+  public void testSearchByQueryStringStringStringBoolean() {
+
+    try {
+      String res = es.searchByQuery("mudrod", "MetadataLinkage", "temp", false);
+      assert res != null;
+    } catch (IOException | InterruptedException | ExecutionException e) {
+      LOG.error("Error!", e);
+    }
+  }
+
+  @Test
+  public void testAutoComplete() {
+
+    List<String> res = es.autoComplete("mudrod", "term");
+    assert !res.isEmpty();
+  }
+
+  @Test
+  public void testClose() {
+
+    es.close();
+  }
+
+  @Test
+  public void testRefreshIndex() {
+    
+    es.refreshIndex();
+  }
+
+/*  @Test
+  public void testMakeClient() {
+
+    try {
+      Client client = es.makeClient(mudrodEngine.loadConfig());
+      assert client != null;
+    } catch (IOException e) {
+      LOG.error("Error!", e);
+    }
+  }*/
+
+  @Test
+  public void testGetClient() {
+
+    Client client = es.getClient();
+    assert client != null;
+  }
+
+/*  @Test
+  public void testSetClient() {
+    
+    try {
+      Client client = es.makeClient(mudrodEngine.loadConfig());
+      es.setClient(client);
+      Client res = es.getClient();
+      assert res != null;
+    } catch (IOException e) {
+      LOG.error("Error!", e);
+    }
+  }
+*/
+  @Test
+  public void testGetBulkProcessor() {
+        
+    BulkProcessor processor = es.getBulkProcessor();
+    assert processor != null;
+  }
+
+  @Test
+  public void testSetBulkProcessor() {
+      
+    BulkProcessor begin = es.getBulkProcessor();
+    es.setBulkProcessor(begin);
+
+    BulkProcessor processor = es.getBulkProcessor();
+    assert processor != null;
+  }
+
+  @Test
+  public void testGenerateUpdateRequestStringStringStringStringObject() {
+    
+    UpdateRequest res = es.generateUpdateRequest("mudrod", "MetadataLinkage", "id_1", "temp", "string_a");
+    assert res != null;
+  }
+
+  @Test
+  public void testGenerateUpdateRequestStringStringStringMapOfStringObject() {
+    
+    Map<String, Object> result = new HashMap<String, Object>();
+    result.put("temp", "string_a");
+
+    UpdateRequest res = es.generateUpdateRequest("mudrod", "MetadataLinkage", "id_1", result);
+    assert res != null;
+  }
+
+  @Test
+  public void testGetDocCountStringStringArray() {
+    
+    String box[] = new String[] { "MetadataLinkage" };
+    int res = es.getDocCount("mudrod", box);
+
+    assert res > 0;
+  }
+
+  @Test
+  public void testGetDocCountStringArrayStringArray() {
+    
+    String begin[] = new String[] { "mudrod" };
+    String box[] = new String[] { "MetadataLinkage" };
+    int res = es.getDocCount(begin, box);
+    assert res > 0;
+  }
+
+  @Test
+  public void testGetDocCountStringArrayStringArrayQueryBuilder() {
+    
+    MatchAllQueryBuilder search = QueryBuilders.matchAllQuery();
+
+    String begin[] = new String[] { "mudrod" };
+    String box[] = new String[] { "MetadataLinkage" };
+    int res = es.getDocCount(begin, box, search);
+    assert res > 0;
+  }
+}
diff --git a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/RequestUrlTest.java b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/RequestUrlTest.java
new file mode 100644
index 0000000..71dd0c4
--- /dev/null
+++ b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/RequestUrlTest.java
@@ -0,0 +1,76 @@
+package org.apache.sdap.mudrod.weblog.structure.log;
+
+import static org.junit.Assert.*;
+
+import java.io.UnsupportedEncodingException;
+import java.util.Map;
+
+import org.apache.sdap.mudrod.weblog.structure.log.RequestUrl;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class RequestUrlTest {
+	
+//	@BeforeClass
+//    public void setUp(){
+//		RequestUrl url = new RequestUrl();
+//		
+//    }
+//	@Test
+//	public void testRequestUrl() {
+//		fail("Not yet implemented");
+//	}
+
+	@Test
+	public void testUrlPage() {
+		RequestUrl url = new RequestUrl();
+		String strURL = "https://podaac.jpl.nasa.gov/datasetlist?ids=Collections:Measurement:SpatialCoverage:Platform:Sensor&values=SCAT_BYU_L3_OW_SIGMA0_ENHANCED:Sea%20Ice:Bering%20Sea:ERS-2:AMI&view=list";
+		String result = url.urlPage(strURL);
+		//System.out.println(urlPage);
+		///fail("Not yet implemented");
+		Assert.assertEquals("You did not pass urlPage function ", "https://podaac.jpl.nasa.gov/datasetlist", result);
+	}
+
+	@Test
+	public void testuRLRequest() {
+		RequestUrl url = new RequestUrl();
+		String strURL = "https://podaac.jpl.nasa.gov/datasetlist?ids=Collections:Measurement:SpatialCoverage:Platform:Sensor&values=SCAT_BYU_L3_OW_SIGMA0_ENHANCED:Sea%20Ice:Bering%20Sea:ERS-2:AMI&view=list";
+		Map<String, String> result = url.uRLRequest(strURL);
+		Assert.assertEquals("You did not pass uRLRequest function!","list", result.get("view"));
+		Assert.assertEquals("You did not pass uRLRequest function!","scat_byu_l3_ow_sigma0_enhanced:sea%20ice:bering%20sea:ers-2:ami", result.get("values"));
+		Assert.assertEquals("You did not pass uRLRequest function!","collections:measurement:spatialcoverage:platform:sensor", result.get("ids"));
+	}
+
+	@Test
+	public void testGetSearchInfo() {
+		RequestUrl url = new RequestUrl();
+		String strURL = "https://podaac.jpl.nasa.gov/datasetlist?ids=Collections:Measurement:SpatialCoverage:Platform:Sensor&values=SCAT_BYU_L3_OW_SIGMA0_ENHANCED:Sea%20Ice:Bering%20Sea:ERS-2:AMI&view=list";
+		String result = null;
+		try {
+			result = url.getSearchInfo(strURL);
+		} catch (UnsupportedEncodingException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+		
+		Assert.assertEquals("You did not pass GetSearchInfo function ", "scat byu l3 ow sigma0 enhanced,sea ice,bering sea,ers 2,ami", result);
+	}
+
+	@Test
+	public void testGetFilterInfo() {
+		RequestUrl url = new RequestUrl();
+		String strURL = "https://podaac.jpl.nasa.gov/datasetlist?ids=Collections:Measurement:SpatialCoverage:Platform:Sensor&values=SCAT_BYU_L3_OW_SIGMA0_ENHANCED:Sea%20Ice:Bering%20Sea:ERS-2:AMI&view=list";
+		try {
+			Map<String, String> result = url.getFilterInfo(strURL);
+			Assert.assertEquals("You did not pass GetFilterInfo function!","scat byu l3 ow sigma0 enhanced", result.get("collections"));
+			Assert.assertEquals("You did not pass GetFilterInfo function!","bering sea", result.get("spatialcoverage"));
+			Assert.assertEquals("You did not pass GetFilterInfo function!","ami", result.get("sensor"));
+		} catch (UnsupportedEncodingException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+
+}
diff --git a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/TestApacheAccessLog.java b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/TestApacheAccessLog.java
index e1453a5..eb7eaed 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/TestApacheAccessLog.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/TestApacheAccessLog.java
@@ -14,6 +14,7 @@
 
 package org.apache.sdap.mudrod.weblog.structure;
 
+import org.apache.sdap.mudrod.weblog.structure.log.ApacheAccessLog;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
diff --git a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/log/GeoIpTest.java b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/log/GeoIpTest.java
new file mode 100644
index 0000000..31371d1
--- /dev/null
+++ b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/log/GeoIpTest.java
@@ -0,0 +1,18 @@
+package org.apache.sdap.mudrod.weblog.structure.log;
+
+import static org.junit.Assert.*;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class GeoIpTest {
+
+	@Test
+	public void testToLocation() {
+		GeoIp ip = new GeoIp();
+		String iptest = "185.10.104.194";
+		Coordinates result = ip.toLocation(iptest);
+		Assert.assertEquals("failed in geoip function!", "22.283001,114.150002", result.latlon);
+	}
+
+}

From 70d1fbfb1118a7fc2e3d8b7b0679987314761d06 Mon Sep 17 00:00:00 2001
From: quintinali <tina.yuner@gmail.com>
Date: Sun, 18 Nov 2018 13:36:32 -0800
Subject: [PATCH 2/9] log preprocessing test case

---
 .../WeblogDiscoveryEngineTest.java            | 156 ++++++++++++++++++
 .../test/resources/FTP.vsftpd.log.201502.gz   | Bin 0 -> 350 bytes
 .../test/resources/WWW.access_log.201502.gz   | Bin 0 -> 2003 bytes
 3 files changed, 156 insertions(+)
 create mode 100644 core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java
 create mode 100644 core/src/test/resources/FTP.vsftpd.log.201502.gz
 create mode 100644 core/src/test/resources/WWW.access_log.201502.gz

diff --git a/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java b/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java
new file mode 100644
index 0000000..f320d3a
--- /dev/null
+++ b/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java
@@ -0,0 +1,156 @@
+package org.apache.sdap.mudrod.discoveryengine;
+
+import static org.junit.Assert.*;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.sdap.mudrod.driver.ESDriver;
+import org.apache.sdap.mudrod.driver.SparkDriver;
+import org.apache.sdap.mudrod.main.AbstractElasticsearchIntegrationTest;
+import org.apache.sdap.mudrod.main.MudrodConstants;
+import org.apache.sdap.mudrod.main.MudrodEngine;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class WeblogDiscoveryEngineTest extends AbstractElasticsearchIntegrationTest {
+	
+	
+	private static WeblogDiscoveryEngine weblogEngine = null;
+	
+	@BeforeClass
+    public static void setUp(){
+		 MudrodEngine mudrodEngine = new MudrodEngine();
+		 Properties props = mudrodEngine.loadConfig();
+		 ESDriver es = new ESDriver(props);
+		 SparkDriver spark = new SparkDriver(props);
+		 File resourcesDirectory = new File("src/test/resources/");
+		 String resourcedir = "/Testing_Data_1_3dayLog+Meta+Onto/";
+		 String dataDir = resourcesDirectory.getAbsolutePath()+ resourcedir;
+		 System.out.println(dataDir);
+		 props.setProperty(MudrodConstants.DATA_DIR, dataDir);		 
+		 MudrodEngine.loadPathConfig(mudrodEngine, dataDir);
+		 weblogEngine = new WeblogDiscoveryEngine(props, es, spark);
+    }
+	
+    @AfterClass
+    public static void tearDown(){
+        //TODO
+    }
+    
+	@Test
+	public void testPreprocess() throws IOException {
+		weblogEngine.preprocess();
+		//TODO compare session result
+		//TODO compare history data
+		String userHistorycsvFile = "C:/Users/admin/Documents/GitHub/incubator-sdap-mudrod/core/userHistoryMatrix.csv";
+		System.out.println(userHistorycsvFile);
+		BufferedReader br = new BufferedReader(new FileReader(userHistorycsvFile));
+	    String line =  null;
+	    HashMap<String,List<String>> map = new HashMap<>();
+
+	    int i =0;
+	    List<String> header = new LinkedList<>();
+	    while((line=br.readLine())!=null){	    	
+	    	if(i ==0){
+	    		String str[] = line.split(",");
+	    		for(String s : str){
+	    			header.add(s);
+	    		}
+	    	}else{	    	
+	        String str[] = line.split(",");
+	        for(int j=1;j<str.length;j++){
+//	        	System.out.println(str[j]+str[j].getClass().getName());
+	        	if (!str[j].equals("0")){
+//	        		System.out.println(str[0]+" "+header.get(j));
+	        		if(!map.containsKey(str[0])){
+	        			map.put(str[0], new ArrayList<>());
+	        		}
+	        		map.get(str[0]).add(header.get(j));
+	        	}
+	        }
+	    	}	        
+	        i += 1;
+	    }
+	    System.out.println(map);
+	    
+//	    Assert.assertEquals("failed in history data result!", "202.92.130.58",String.join(",", map.get("metopglobal")));
+//	    Assert.assertEquals("failed in history data result!", "202.92.130.58",String.join(",", map.get("metop")));
+    
+	}
+	
+	@Test
+	public void testPreprocess_clickStream() throws IOException {
+		//TODO compare clickStream data
+		String clickStreamcsvFile = "C:/Users/admin/Documents/GitHub/incubator-sdap-mudrod/core/clickStreamMatrix.csv";
+		System.out.println(clickStreamcsvFile);
+		BufferedReader br = new BufferedReader(new FileReader(clickStreamcsvFile));
+	    String line =  null;
+	    HashMap<String,List<String>> map = new HashMap<>();
+
+	    int i =0;
+	    List<String> header = new LinkedList<>();
+	    while((line=br.readLine())!=null){	    	
+	    	if(i == 0){
+	    		String str[] = line.split(",");
+	    		for(String s : str){
+	    			header.add(s);
+//	    			System.out.println(header);
+	    		}
+	    	}else{	    	
+	        String str[] = line.split(",");
+	        for(int j=1;j<str.length;j++){
+//	        	System.out.println(str[j]+str[j].getClass().getName());
+	        	
+	        	if (!str[j].equals("0.0")){ 	//
+	        		if(!map.containsKey(str[0])){
+	        			map.put(str[0], new ArrayList<>());
+	        		}
+	        		map.get(str[0]).add(header.get(j));
+	        	}
+	        }
+	    	}	        
+	        i += 1;
+	    }
+	    System.out.println(map);
+	    
+	    Assert.assertEquals("failed in click stream result!", "\"avhrr_sst_noaa19_nar-osisaf-l3c-v1.0\"",String.join(",", map.get("avhrr sst noaa19 nar osisaf l3c v1.0")));
+	    Assert.assertEquals("failed in click stream result!", "\"ascata-l2-coastal\"", String.join(",", map.get("metop")));
+	}
+
+//	@Test
+//	public void testProcess() {
+//		
+//		//fail("Not yet implemented");
+//	}
+//
+//	@Test
+//	public void testWeblogDiscoveryEngine() {
+//		fail("Not yet implemented");
+//	}
+//
+//	@Test
+//	public void testGetFileList() {
+//		fail("Not yet implemented");
+//	}
+//
+//	@Test
+//	public void testLogIngest() {
+//		fail("Not yet implemented");
+//	}
+//
+//	@Test
+//	public void testSessionRestruct() {
+//		fail("Not yet implemented");
+//	}
+
+}
diff --git a/core/src/test/resources/FTP.vsftpd.log.201502.gz b/core/src/test/resources/FTP.vsftpd.log.201502.gz
new file mode 100644
index 0000000000000000000000000000000000000000..f80ebe5ec77a69136a088712d0df7d2638b4ea45
GIT binary patch
literal 350
zcmV-k0ipgMiwFou?e1Fy07g_$E_QQfbZ}%YY;R{SGB7bUFfstOk55m+KoG_6CVq!G
zn{;<(yR<APh$L_zVyJT6taYgkTee}_qI`Ob68|OM^e~f`d3lrf%i0~F0+9kGqc|N=
z0s?6WB!Sd`2~tcAS?{7!qzJfiZfY9Cb8kXpoHnlE_1<=+J@AF(%j{lXFDDs4Si5yE
z+=o`@r4K>BA9NL*(S`M$by^?_-4Wa6tT@60Xz*a5^2gFSGq%sBs-0!IuVB;EfN%x*
z<;C~L!;z5Z#ux()h*646NMwW{h7uE$Us@MW`w$xbc+#_JCip_!^2vI>T9MK-eluAw
zc}Ki|IN%tIH-Y`4M2MK8lOOdFJ)5tx>@Mnus^}vhbto7{n+}O#PxXluLKND%_WNdB
w*~V`+Wo}vFJJxQ0Ws^9H)tN1pKiOiD3}ThaKVVB`D*m7C6^ACBD{2A&0AhHe<NyEw

literal 0
HcmV?d00001

diff --git a/core/src/test/resources/WWW.access_log.201502.gz b/core/src/test/resources/WWW.access_log.201502.gz
new file mode 100644
index 0000000000000000000000000000000000000000..516c63907fa063ef42314107dd548f563f99fe29
GIT binary patch
literal 2003
zcmV;^2Q2s>iwFn??e1Fy09RL6E@5M1Wpi_1Y;R{SGB7bUFfstuT4`_7Fc|)h#DA!e
zKwL>|=V($iLFh^uv}<WPP9e)py)KX@C3afIujh5TmXQo~+?m$(h~37|dpz%9W6#ww
zc6DNzI!3sm!QVTCjVF9!5QOa(LM>#qY`dW$7oqo!8{N*hVPxqfl|q-FW#j31JT$P5
zZZrr&4Q$(}aYMV&xG@t<3p2KgdDghyNhHq}StR&_#@N!H4fnf8=6KY8;lF<Sput%t
zPo_*VO|VCCB;c15CYU};m-j^$CDBrSg4^ZuIg8n&&ETCI*RNd)06gRG0jz|<Kg*1T
z(wJo|$z%RdoTQmNWOKgf2W@}sYdxY7`{Vr9*@R+<iI7POKWw5V_9!utgAko_nJySL
zZA2`}7lNjg*oYDWW9f3G$3#!UTa!upc~8SGy03ZYzIyf0t>ZMCGr5;=8BUMS3DZQR
zVK%K=%F_E@mMlrqJPG-nCsL@1riLsHdGyc^eA;7@C*dg#WIl!GJX`Y9lF>Mxi^x?)
zYH9@0;b7nQ+w>U|X`*5Jn1vsyOzB}dj}jIS7Cd7zN)u56-bA){HC0SBL^Nz~fWO2#
z*&k8=Q_iv|7Zv?FmR(1f*xRjNK<aE@4uS(3_)lm-5OftKQ$ZsIBFh(&Pb;uZ_sYVy
z;IV=IqJY~YkLgR3?!(dqmvqEq4sYm_G>+5N(V_yr>DDq5>;z8~cx-OKcLKkYEF%H5
z5C1%lBgw0Z_S}Dp)o}o(r#rTe9rH&>3HlzX5YpJJ97BXrGFq>?pMHtrm>IT?8n?rA
zz5uqG#IWjan$BVlvECp$e$cpgBITlG7^~GvpA|zo%r;<LKaunJC$vED4lUq93mazL
zGK*pwlTOcUFiBUGEm;(^NnBNmRZs8R37x>OM!XI5d;tl_lq>SNNsyu+mP~2K9|h-l
zh%T~pnuoF)VLWtYrC@5<DjR?n_-$>U1`8H)Kyxf<z>fJz8iPod@ZGW%xVBSI{O0%g
zlXS`rY>|roEaF{-^<ShvPb1q5q22BeRp{t^hICLsE=2cvsQkQ|E4cXT0{#{^4*809
zvuHZyQ`(NBImC{$*jaT_i?~-J4Xjp8M$O75B~F4pFrt4OkS{4vV(renfc3WA-&YdS
z{0oSlrnzJ@4l?btEIg^u2Ck_WSr^cr)vS;JBDG{~8|@|mSHs(a0Rp!2d8h(;1{t%A
zvSMlluI<#)TzA4%<_a^)K^kB$iH?dqtrCzu+q{xkF%*J@YQOROoM$tjcNWWN&SCwV
z#I)><Kzp_1cbn|QjFCo~8_eUUqt5<*P{c7BaA@UZRpN2#sovdCRjI#)`YO#nLTFpi
zb=)F!5OS8#AdEPCO8_&2Hz%`dx^31nUE2x2sCtQA?qs!92Wv(LIRu>JZK_>VAO$8E
zEIgqfi6W`9XrG-H0;@>IbZXfTxQt}L4TyrdEj+tVM?o;|k>MuA>&EGX#X&w1AESiA
zST&FT$1Q@(!nbz&g}0@EZxml}Ks9W8^>qGAfV4GS0XfXL5YHnCeTFoNA&6IkSxxiI
zO9DP$tgKCXxXv!Cy%ejK>7;G%c3ueD-gmSHtj6MKvTn&!Kn%^Jd9`N^HA*cZJ3Cbi
z58^7FZVlN8SPXZUr%%!>pI5CI*E8GR4Ha4EdPGMHSPxW8FY@T4Dwxj7?P`airWvSn
zQA@~1RcEYcXWef8oVw2Ubcqf-3(!5FL1#=42d}9A(m&|=k9(a9{IZ_fb1%@ITWxP)
zyPm%0?o=K|kOpm)*AGTohaMzCX1SU~pFkfz&$!AB*9Af^72<+jp<3?r?u5VI>2=Qq
z%ymTodqNs1Z;F+ZKhrg9=&=qwe~<bsy)}Z4|LWin3cbf~j%d%M;E%yUx9=nBACCP|
zdwkILK@vTGqy^*7s59t}{NdAsHhnxAyb3zB?+*j|<fzxvOD7t%`bhzFU+QuedVK}G
zHe(Y=jDRzBL1>#R^f}~&$tHI&U~jarfi7E=O&(*?tZ9;1!YxjP<a3R|>ZEv>W>cQ+
z9Y<W1d!dY${FZ=D@7|P)uzHLE*RbjZ`^FdBxBueE9~~S8u%`{RT2Ox@t-31PWOgN8
zxJ|E)8(u%ug}T{8!Br1?@2}8KVW1AF<6iKkgi^BAGBVrq>KR%}$TrE|1?F1A)+nss
z+C?3vQU?wC2R=3XgTwJtI2}{weu9bpuM1~ehYc_tP&U`tNYg-`g66ic(=x3VAwM>{
zXe##lkiz8ajfxgTxDa|E(B|v$C68OG-4&?gAuJwok;b{Ip<81<U!>XjOShIR&biow
zz^M)`<k>Oo*TcJUx=3MTaB*_lx@+BSA)VN*CS0GgQ*mn<@ztJM#`cuTnUPo~h6s9g
z_+*hz84L9fi+C%QzXGB45)4)yEcq2Itst5u(Qy>AM8c(4Y^lESlt;4@`3-0J=*?>k
zhir}}c_Q-%jb4=GpBtZDhdS1SMz(C3Fy~IA%fpYUQYS*ne4oIU(sBCPP=l9Kzv=va
z_5Y^q&Q1O%wjY~^ZDR7z&Er>))l>7vAQ<;mc!5_s=)yV9?l7o@jQ_3~{RbVm@H4;c
leU}r$kKM#W?k}_GxqII`yYszo_x;n7{0eF#_EnZJ000(^-_Za7

literal 0
HcmV?d00001


From b3ef470a39f8df68ddf80805c6948f870feeb2ce Mon Sep 17 00:00:00 2001
From: quintinali <tina.yuner@gmail.com>
Date: Fri, 30 Nov 2018 17:57:32 -0500
Subject: [PATCH 3/9] test data for log preprocessing

---
 .../WeblogDiscoveryEngine.java                |   8 +-
 .../apache/sdap/mudrod/main/MudrodEngine.java |   6 +-
 .../weblog/pre/ClickStreamGenerator.java      |   1 -
 .../mudrod/weblog/pre/SessionStatistic.java   |   2 +-
 .../weblog/structure/session/SessionTree.java |   8 +-
 .../WeblogDiscoveryEngineTest.java            | 225 ++++++++----------
 .../driver/EmbeddedElasticsearchServer.java   |   4 -
 .../AbstractElasticsearchIntegrationTest.java |   4 +-
 .../apache/sdap/mudrod/main/ESDriverTest.java |   3 -
 .../weblog/structure/RequestUrlTest.java      |   3 +-
 .../FTP.201502.w1.gz}                         | Bin
 .../WWW.201502.w1.gz                          | Bin 0 -> 2033 bytes
 .../clickstreammatrix.csv                     |   2 +
 .../userhistorymatrix.csv                     |   2 +
 .../test/resources/WWW.access_log.201502.gz   | Bin 2003 -> 0 bytes
 15 files changed, 119 insertions(+), 149 deletions(-)
 rename core/src/test/resources/{FTP.vsftpd.log.201502.gz => Testing_Data_1_3dayLog+Meta+Onto/FTP.201502.w1.gz} (100%)
 create mode 100644 core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/WWW.201502.w1.gz
 create mode 100644 core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/clickstreammatrix.csv
 create mode 100644 core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/userhistorymatrix.csv
 delete mode 100644 core/src/test/resources/WWW.access_log.201502.gz

diff --git a/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngine.java b/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngine.java
index b2cdb9f..661ed8c 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngine.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngine.java
@@ -105,7 +105,7 @@ public void preprocess() {
       startTime = System.currentTimeMillis();
       LOG.info("Processing logs dated {}", anInputList);
 
-      DiscoveryStepAbstract im = new ImportLogFile(this.props, this.es, this.spark);
+     /* DiscoveryStepAbstract im = new ImportLogFile(this.props, this.es, this.spark);
       im.execute();
 
       DiscoveryStepAbstract cd = new CrawlerDetection(this.props, this.es, this.spark);
@@ -118,15 +118,15 @@ public void preprocess() {
       ss.execute();
 
       DiscoveryStepAbstract rr = new RemoveRawLog(this.props, this.es, this.spark);
-      rr.execute();
+      rr.execute();*/
 
       endTime = System.currentTimeMillis();
 
       LOG.info("Web log preprocessing for logs dated {} complete. Time elapsed {} seconds.", anInputList, (endTime - startTime) / 1000);
     }
 
-    DiscoveryStepAbstract hg = new HistoryGenerator(this.props, this.es, this.spark);
-    hg.execute();
+    /*DiscoveryStepAbstract hg = new HistoryGenerator(this.props, this.es, this.spark);
+    hg.execute();*/
 
     DiscoveryStepAbstract cg = new ClickStreamGenerator(this.props, this.es, this.spark);
     cg.execute();
diff --git a/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java b/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java
index 48218e9..d9d01aa 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java
@@ -391,12 +391,12 @@ public static void main(String[] args) {
       me.end();
     } catch (Exception e) {
       HelpFormatter formatter = new HelpFormatter();
-      formatter.printHelp("MudrodEngine: 'dataDir' argument is mandatory. " + "User must also provide an ingest method.", true);
+      formatter.printHelp("MudrodEngine: 'dataDir' argument is mandatory. " + "User must also provide an ingest method.", options, true);
       LOG.error("Error whilst parsing command line.", e);
     }
   }
 
-  private static void loadPathConfig(MudrodEngine me, String dataDir) {
+  public static void loadPathConfig(MudrodEngine me, String dataDir) {
     me.props.put(MudrodConstants.ONTOLOGY_INPUT_PATH, dataDir + "SWEET_ocean/");
     me.props.put(MudrodConstants.ONTOLOGY_PATH, dataDir + "ocean_triples.csv");
     me.props.put(MudrodConstants.USER_HISTORY_PATH, dataDir + "userhistorymatrix.csv");
@@ -405,7 +405,6 @@ private static void loadPathConfig(MudrodEngine me, String dataDir) {
     me.props.put(MudrodConstants.CLICKSTREAM_SVD_PATH, dataDir + "clickstreamsvdmatrix_tmp.csv");
     me.props.put(MudrodConstants.METADATA_SVD_PATH, dataDir + "metadatasvdMatrix_tmp.csv");
     me.props.put(MudrodConstants.RAW_METADATA_PATH, dataDir + me.props.getProperty(MudrodConstants.RAW_METADATA_TYPE));
-
     me.props.put(MudrodConstants.METADATA_TERM_MATRIX_PATH, dataDir + "metadata_term_tfidf.csv");
     me.props.put(MudrodConstants.METADATA_WORD_MATRIX_PATH, dataDir + "metadata_word_tfidf.csv");
     me.props.put(MudrodConstants.METADATA_SESSION_MATRIX_PATH, dataDir + "metadata_session_coocurrence_matrix.csv");
@@ -428,6 +427,5 @@ public SparkDriver getSparkDriver() {
    */
   public void setSparkDriver(SparkDriver sparkDriver) {
     this.spark = sparkDriver;
-
   }
 }
diff --git a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/ClickStreamGenerator.java b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/ClickStreamGenerator.java
index 2c38d29..8d115aa 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/ClickStreamGenerator.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/ClickStreamGenerator.java
@@ -71,5 +71,4 @@ public Object execute() {
   public Object execute(Object o) {
     return null;
   }
-
 }
diff --git a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionStatistic.java b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionStatistic.java
index c89160f..a40693e 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionStatistic.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionStatistic.java
@@ -229,7 +229,7 @@ public int processSession(ESDriver es, String sessionId) throws IOException, Int
               String[] keywordList = keywords.split(",");
               for (String item : items) {
                 if (!Arrays.asList(keywordList).contains(item)) {
-                  keywords = keywords + item + ",";
+                  keywords = keywords + "," + item + ",";
                 }
               }
             }
diff --git a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionTree.java b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionTree.java
index 5531f83..d4a28aa 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionTree.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionTree.java
@@ -26,10 +26,13 @@
 
 import java.io.UnsupportedEncodingException;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
 /**
@@ -205,7 +208,8 @@ public List<ClickStream> getClickStreamList(Properties props) {
       RequestUrl requestURL = new RequestUrl();
       String viewquery = "";
       try {
-        String infoStr = requestURL.getSearchInfo(viewnode.getRequest());
+        //String infoStr = requestURL.getSearchInfo(viewnode.getRequest());
+        String infoStr = requestURL.getSearchInfo(viewnode.getReferer());
         viewquery = es.customAnalyzing(props.getProperty(MudrodConstants.ES_INDEX_NAME), infoStr);
       } catch (UnsupportedEncodingException | InterruptedException | ExecutionException e) {
         LOG.warn("Exception getting search info. Ignoring...", e);
@@ -222,6 +226,8 @@ public List<ClickStream> getClickStreamList(Properties props) {
 
       if (viewquery != null && !"".equals(viewquery)) {
         String[] queries = viewquery.trim().split(",");
+        Set<String> queryset = new HashSet<>();
+        queryset.addAll(Arrays.asList(queries));
         if (queries.length > 0) {
           for (String query : queries) {
             ClickStream data = new ClickStream(query, dataset, download);
diff --git a/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java b/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java
index f320d3a..b31e266 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java
@@ -1,9 +1,9 @@
 package org.apache.sdap.mudrod.discoveryengine;
 
 import static org.junit.Assert.*;
-
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -11,7 +11,6 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Properties;
-
 import org.apache.sdap.mudrod.driver.ESDriver;
 import org.apache.sdap.mudrod.driver.SparkDriver;
 import org.apache.sdap.mudrod.main.AbstractElasticsearchIntegrationTest;
@@ -23,134 +22,106 @@
 import org.junit.Test;
 
 public class WeblogDiscoveryEngineTest extends AbstractElasticsearchIntegrationTest {
-	
-	
-	private static WeblogDiscoveryEngine weblogEngine = null;
-	
-	@BeforeClass
-    public static void setUp(){
-		 MudrodEngine mudrodEngine = new MudrodEngine();
-		 Properties props = mudrodEngine.loadConfig();
-		 ESDriver es = new ESDriver(props);
-		 SparkDriver spark = new SparkDriver(props);
-		 File resourcesDirectory = new File("src/test/resources/");
-		 String resourcedir = "/Testing_Data_1_3dayLog+Meta+Onto/";
-		 String dataDir = resourcesDirectory.getAbsolutePath()+ resourcedir;
-		 System.out.println(dataDir);
-		 props.setProperty(MudrodConstants.DATA_DIR, dataDir);		 
-		 MudrodEngine.loadPathConfig(mudrodEngine, dataDir);
-		 weblogEngine = new WeblogDiscoveryEngine(props, es, spark);
-    }
-	
-    @AfterClass
-    public static void tearDown(){
-        //TODO
+
+  private static WeblogDiscoveryEngine weblogEngine = null;
+
+  @BeforeClass
+  public static void setUp() {
+    MudrodEngine mudrodEngine = new MudrodEngine();
+    Properties props = mudrodEngine.loadConfig();
+    ESDriver es = new ESDriver(props);
+    SparkDriver spark = new SparkDriver(props);
+    String dataDir = getTestDataPath();
+    System.out.println(dataDir);
+    props.setProperty(MudrodConstants.DATA_DIR, dataDir);
+    MudrodEngine.loadPathConfig(mudrodEngine, dataDir);
+    weblogEngine = new WeblogDiscoveryEngine(props, es, spark);
+  }
+
+  @AfterClass
+  public static void tearDown() {
+    // TODO
+  }
+
+  private static String getTestDataPath() {
+    File resourcesDirectory = new File("src/test/resources/");
+    String resourcedir = "/Testing_Data_1_3dayLog+Meta+Onto/";
+    String dataDir = resourcesDirectory.getAbsolutePath() + resourcedir;
+    return dataDir;
+  }
+
+  @Test
+  public void testPreprocess() throws IOException {
+
+    weblogEngine.preprocess();
+    testPreprocess_userHistory();
+    testPreprocess_clickStream();
+  }
+
+  private void testPreprocess_userHistory() throws IOException {
+    // compare user history data
+    String userHistorycsvFile = getTestDataPath() + "/userHistoryMatrix.csv";
+    BufferedReader br = new BufferedReader(new FileReader(userHistorycsvFile));
+    String line = null;
+    HashMap<String, List<String>> map = new HashMap<>();
+    int i = 0;
+    List<String> header = new LinkedList<>();
+    while ((line = br.readLine()) != null) {
+      if (i == 0) {
+        String str[] = line.split(",");
+        for (String s : str) {
+          header.add(s);
+        }
+      } else {
+        String str[] = line.split(",");
+        for (int j = 1; j < str.length; j++) {
+          if (!str[j].equals("0")) {
+            if (!map.containsKey(str[0])) {
+              map.put(str[0], new ArrayList<>());
+            }
+            map.get(str[0]).add(header.get(j));
+          }
+        }
+      }
+      i += 1;
     }
-    
-	@Test
-	public void testPreprocess() throws IOException {
-		weblogEngine.preprocess();
-		//TODO compare session result
-		//TODO compare history data
-		String userHistorycsvFile = "C:/Users/admin/Documents/GitHub/incubator-sdap-mudrod/core/userHistoryMatrix.csv";
-		System.out.println(userHistorycsvFile);
-		BufferedReader br = new BufferedReader(new FileReader(userHistorycsvFile));
-	    String line =  null;
-	    HashMap<String,List<String>> map = new HashMap<>();
 
-	    int i =0;
-	    List<String> header = new LinkedList<>();
-	    while((line=br.readLine())!=null){	    	
-	    	if(i ==0){
-	    		String str[] = line.split(",");
-	    		for(String s : str){
-	    			header.add(s);
-	    		}
-	    	}else{	    	
-	        String str[] = line.split(",");
-	        for(int j=1;j<str.length;j++){
-//	        	System.out.println(str[j]+str[j].getClass().getName());
-	        	if (!str[j].equals("0")){
-//	        		System.out.println(str[0]+" "+header.get(j));
-	        		if(!map.containsKey(str[0])){
-	        			map.put(str[0], new ArrayList<>());
-	        		}
-	        		map.get(str[0]).add(header.get(j));
-	        	}
-	        }
-	    	}	        
-	        i += 1;
-	    }
-	    System.out.println(map);
-	    
-//	    Assert.assertEquals("failed in history data result!", "202.92.130.58",String.join(",", map.get("metopglobal")));
-//	    Assert.assertEquals("failed in history data result!", "202.92.130.58",String.join(",", map.get("metop")));
-    
-	}
-	
-	@Test
-	public void testPreprocess_clickStream() throws IOException {
-		//TODO compare clickStream data
-		String clickStreamcsvFile = "C:/Users/admin/Documents/GitHub/incubator-sdap-mudrod/core/clickStreamMatrix.csv";
-		System.out.println(clickStreamcsvFile);
-		BufferedReader br = new BufferedReader(new FileReader(clickStreamcsvFile));
-	    String line =  null;
-	    HashMap<String,List<String>> map = new HashMap<>();
+    Assert.assertEquals("failed in history data result!", "195.219.98.7", String.join(",", map.get("sea surface topography")));
+  }
 
-	    int i =0;
-	    List<String> header = new LinkedList<>();
-	    while((line=br.readLine())!=null){	    	
-	    	if(i == 0){
-	    		String str[] = line.split(",");
-	    		for(String s : str){
-	    			header.add(s);
-//	    			System.out.println(header);
-	    		}
-	    	}else{	    	
-	        String str[] = line.split(",");
-	        for(int j=1;j<str.length;j++){
-//	        	System.out.println(str[j]+str[j].getClass().getName());
-	        	
-	        	if (!str[j].equals("0.0")){ 	//
-	        		if(!map.containsKey(str[0])){
-	        			map.put(str[0], new ArrayList<>());
-	        		}
-	        		map.get(str[0]).add(header.get(j));
-	        	}
-	        }
-	    	}	        
-	        i += 1;
-	    }
-	    System.out.println(map);
-	    
-	    Assert.assertEquals("failed in click stream result!", "\"avhrr_sst_noaa19_nar-osisaf-l3c-v1.0\"",String.join(",", map.get("avhrr sst noaa19 nar osisaf l3c v1.0")));
-	    Assert.assertEquals("failed in click stream result!", "\"ascata-l2-coastal\"", String.join(",", map.get("metop")));
-	}
+  private void testPreprocess_clickStream() throws IOException {
+    // TODO compare clickStream data
+    // String clickStreamcsvFile =
+    // "C:/Users/admin/Documents/GitHub/incubator-sdap-mudrod/core/clickStreamMatrix.csv";
+    String clickStreamcsvFile = getTestDataPath() + "/clickStreamMatrix.csv";
+    System.out.println(clickStreamcsvFile);
+    BufferedReader br = new BufferedReader(new FileReader(clickStreamcsvFile));
+    String line = null;
+    HashMap<String, List<String>> map = new HashMap<>();
 
-//	@Test
-//	public void testProcess() {
-//		
-//		//fail("Not yet implemented");
-//	}
-//
-//	@Test
-//	public void testWeblogDiscoveryEngine() {
-//		fail("Not yet implemented");
-//	}
-//
-//	@Test
-//	public void testGetFileList() {
-//		fail("Not yet implemented");
-//	}
-//
-//	@Test
-//	public void testLogIngest() {
-//		fail("Not yet implemented");
-//	}
-//
-//	@Test
-//	public void testSessionRestruct() {
-//		fail("Not yet implemented");
-//	}
+    int i = 0;
+    List<String> header = new LinkedList<>();
+    while ((line = br.readLine()) != null) {
+      if (i == 0) {
+        String str[] = line.split(",");
+        for (String s : str) {
+          header.add(s);
+        }
+      } else {
+        String str[] = line.split(",");
+        for (int j = 1; j < str.length; j++) {
+          if (!str[j].equals("0.0")) { //
+            if (!map.containsKey(str[0])) {
+              map.put(str[0], new ArrayList<>());
+            }
+            map.get(str[0]).add(header.get(j));
+          }
+        }
+      }
+      i += 1;
+    }
+    System.out.println(map);
 
+    Assert.assertEquals("failed in click stream result!", "\"ostm_l2_ost_ogdr_gps\"", String.join(",", map.get("sea surface topography")));
+  }
 }
diff --git a/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java b/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java
index c4ac65e..ef3587b 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java
@@ -34,9 +34,6 @@ public EmbeddedElasticsearchServer(String dataDirectory) {
     Settings.Builder settingsBuilder = Settings.builder();
     settingsBuilder.put("http.type", "netty3");
     settingsBuilder.put("transport.type", "netty3");
-    settingsBuilder.put("network.host", "127.0.0.1");
-    settingsBuilder.put("transport.tcp.port", 9300);
-
     settingsBuilder.put("cluster.name", "MurdorES").put("http.enabled", "false").put("path.data", dataDirectory).put("path.home", "/");
       
     Settings settings = settingsBuilder.build();
@@ -51,7 +48,6 @@ public EmbeddedElasticsearchServer(String dataDirectory) {
     }
     
     System.out.println(node.getNodeEnvironment().nodeId());
-   
   }
 
   public Client getClient() {
diff --git a/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java b/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java
index cbadaa3..01899ae 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java
@@ -30,8 +30,8 @@ public static void shutdownEmbeddedElasticsearchServer() {
 
     /**
      * By using this method you can access the embedded server.
-     *//*
+     */
     protected Client getClient() {
         return embeddedElasticsearchServer.getClient();
-    }*/
+    }
 }
diff --git a/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java b/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java
index e61103f..ea33766 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java
@@ -51,11 +51,8 @@ public class ESDriverTest extends AbstractElasticsearchIntegrationTest{
   static MudrodEngine mudrodEngine = null;
   @BeforeClass
   public static void setUp(){
-//	 Map<String, String> env = System.getenv();
-//	 env.put(MudrodConstants.MUDROD_CONFIG, "dexuantest");	 
 	 mudrodEngine = new MudrodEngine();
 	 es = new ESDriver(mudrodEngine.loadConfig());	 
-
   }
 
   @AfterClass
diff --git a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/RequestUrlTest.java b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/RequestUrlTest.java
index 71dd0c4..fec8e61 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/RequestUrlTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/RequestUrlTest.java
@@ -1,4 +1,4 @@
-package org.apache.sdap.mudrod.weblog.structure.log;
+package org.apache.sdap.mudrod.weblog.structure;
 
 import static org.junit.Assert.*;
 
@@ -72,5 +72,4 @@ public void testGetFilterInfo() {
 			e.printStackTrace();
 		}
 	}
-
 }
diff --git a/core/src/test/resources/FTP.vsftpd.log.201502.gz b/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/FTP.201502.w1.gz
similarity index 100%
rename from core/src/test/resources/FTP.vsftpd.log.201502.gz
rename to core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/FTP.201502.w1.gz
diff --git a/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/WWW.201502.w1.gz b/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/WWW.201502.w1.gz
new file mode 100644
index 0000000000000000000000000000000000000000..86c5ffc86c49cf1499713615f25912997643ac20
GIT binary patch
literal 2033
zcmV<N2M+ijiwFpfuK`>H09RL6E;2ANH83(RcQF9fT4`_EFc|(mY5xID+O#W)?HnXb
z)s|7l*q|^%cWSasaA~%Xu-JiizkaVnVI5(jJC&*}ZR0@vyvOri#-6KV?CQiab&POL
zgRc(=8_)RIAPCzHgc`_d*mg}rE<zt`x7w{y&B)SmDupi3WbOHAG%&D^Zq*1vHEi3c
zc1yceyEPR|3p2Kic~-mIN+i$bStR(Q+Q`yg3=Z4J=4jY_<-dLZsK!|)PbW+=O|U0%
zB;b=1CYU}=7Y{`iCDB6t2an5VGZwQa>&1JwZr->O0C>ee0$2)zpUI4c(wJr}$z%Rl
zoTiyPVl#f=2TgzEYh9ud`}6G1#SO(06CslnKG;Nc>``JO2O&D+GMzJO+K5<`&jn2>
zu@NN%*3!jNkBOdycgExN>_EdVdZ>Bmq5AEiJ11#2WAY&3GMt=T5~hhr&8%CMl%@Bh
zELoDIc@pv&Poz*cni{e+<k2HP@M)Jxo`mN#kog2&^K8M-w~WT|Ttu!bQd1*{4*G|_
z-=r^?ND~dyr!4$TWlE3IS(LE2Kj#^fQJRP?;B{ns*HgtrLqx;&8u)9hlfxnPzvL{7
za#7N+W7$=NiM{>$1*FazW<TiAz<)*yf}qPNnFtyw5LrH#d{TmKy4M!A1>bAfuL`(b
z@|3<d=^<=Qa7l+e=I{r7md0_qJf4@p*WF4+g1z900*}o#_*USzl0_td_Th)eaU^+J
z(VqKXu{sXG^mND8v19%WDM3FX6+#;8onweFN=B<)_tLLX95ch#QSEM+&gQ^Y;}~|`
zPSa`3A=Vp2$B$|cPNkeT3}d-m>eFILhr10}*H7gv{sk=%yhjVT(88KoH_W1##-x)A
z8%)wAWeXO?Y#f)BVpY@o_ChDHtPyVwJ)c7YGU1ARZWE;Fhb0r*@`u4C9-{Lso#de`
zM;H%XTPc_twzdsG3;d>bNP{^GIiNX_G+-zEG>t(d3wZCg6}YxjP5kDM_>*+P4Q!E;
z{w(5MhyCx;pQn*chR|&G1}b#4&LABWkPFd69x6XC=L#;qzJR~SjU&FK?JSy1_=Gm&
zXa=$4B6gOY)FSS+NCV4NlTop<Nr{tS4~*#l2IMOWlvw*SFJQe5_xF^9G=B%t(=?ZC
z%0Z?*mW8J!+Q1d{B5McjS@jYLAW})zw$XkPa5cOs7$9IPpNA@tXOJ<=C@XF)!L^-A
zn(JP;%3R@&?H~=X7eq%zo>mITo^4)BtQZQxLbc!dea5pX&^wD|G~=-Ubz-{hjX-;q
z<ae9w#f*_g>TArS=fl?FVNk>|8gOXkWM$%Us;S=nP*thFf%+!RK0|1m(^cFev>$So
z&>)OByh{KxgEuGBa=L9+GF{sXzo>eNz1_)bst(qa_Hzh0C!18es6Yx#uvmCPFA_yk
zXVD%zF9cSSjOkReA8-}PfEy46b5nS>pAUmz)Fp#;ir0?QF^hwIEIvmGg|%`X|DRg~
zSA}ow_X}@B0pBUU;DB=2_NwXpR{&{ixCC;LaUott68a2j5<?I#1+$9gnO6jSxma15
z^k9`;mU}5yCDTdU-tW8+w7nl_4OorE(RkI8r+^rmN3(Lz8fuhELU#757QTtAbh<HQ
zBVaK+U7kNnvwT*zVqDE^dp}fUUFs1X&tX4MF}=#8&#GX$D7VWUhKgpO&aRe_jmpkg
z&Ca@A|21`8?&%U8CJWF#ph0UykNR(@|Jv_#{iog54!^9X_S_xXbIa{5Y**9Q+`Y=f
z2-2X9@_NBgYtc?3WR|O&=ricU=NVVI;i^FBrb6tP3RQB_yBGdy((7Idn5&8aW<na-
z9*S)zf2M0z&|@8V{Soy=dTRtN|4ruz3caWAj%nAV;EzG4-SZLkk4FBmIqEchkVMxX
zYQd;AZ1vkifAGB1q)&(aH$jW`{6RpU9e2C>)`<qKdQt%0Z*@5fy}p57o3RcgM!=c6
zAhh)r`Vw-&WSu)0FdHpwpv~snCXX?xS2Ri7!Y$5)<TH)I?xc8}W)q$roJ3red!dXL
z{EmQ5@4<wNuzZXGSFq{@`^LNN+kbWJ4?D*J%(S6a3#xCVWmjeE%x+5;Zr!Wm!0V^F
zP!D@7xawga{1e(K4Adc2oCRM=C|kC=jm-AEYKE3AWb0&ahq+d;H46K;_ECqa)Ioz@
z$ERkme>8dyr(?FcpI~DD@50&EVF0EB%K8c$sT;^s(A)-g8m84C<mW~gO~gSDQkZ--
zsAxch3!w)BZN4sF@VKGcU4c3t!r~zpX`HJXx-sIjd753mbZ5choQnepoa)d*o}IwF
z9{w7o^ArYy^V9Rjed~S$>BMf-;rfJ~i#v;mFAvl<Hm6iBjKnfAM9_=FC-Zc|Sg3!R
z#~Z2q69|phV6f_7$?srk1kp5!PNI+{5-z=A3-ylYJer=$?>M)Q-n_+d$mV#QCo+Fj
z>qbd_Ry%VY>R69z*`i^>J@;yD9)3=hIuTOl`wT`(C+V4?7O$p$)A{%6|3lgR^yZ(N
zh;3r>-%aFq&{b3WMn4$!REUA!wBLraob5qS2^s%;Q~Gy$_#R`*s&L_xe%ldu$>KY~
zC^`JxQ#|DUHkn>}?t_b`J}919H6a@PP5_qtd`ORmBdy&Jw63M~T8EmNOaD!Y{E24%
Pw#>c(U~ex)%`pG~(v<g(

literal 0
HcmV?d00001

diff --git a/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/clickstreammatrix.csv b/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/clickstreammatrix.csv
new file mode 100644
index 0000000..1316fa2
--- /dev/null
+++ b/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/clickstreammatrix.csv
@@ -0,0 +1,2 @@
+ Num,"ostm_l2_ost_ogdr_gps"
+sea surface topography,6.0
diff --git a/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/userhistorymatrix.csv b/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/userhistorymatrix.csv
new file mode 100644
index 0000000..6a094d8
--- /dev/null
+++ b/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/userhistorymatrix.csv
@@ -0,0 +1,2 @@
+Num,195.219.98.7
+sea surface topography,1,
diff --git a/core/src/test/resources/WWW.access_log.201502.gz b/core/src/test/resources/WWW.access_log.201502.gz
deleted file mode 100644
index 516c63907fa063ef42314107dd548f563f99fe29..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 2003
zcmV;^2Q2s>iwFn??e1Fy09RL6E@5M1Wpi_1Y;R{SGB7bUFfstuT4`_7Fc|)h#DA!e
zKwL>|=V($iLFh^uv}<WPP9e)py)KX@C3afIujh5TmXQo~+?m$(h~37|dpz%9W6#ww
zc6DNzI!3sm!QVTCjVF9!5QOa(LM>#qY`dW$7oqo!8{N*hVPxqfl|q-FW#j31JT$P5
zZZrr&4Q$(}aYMV&xG@t<3p2KgdDghyNhHq}StR&_#@N!H4fnf8=6KY8;lF<Sput%t
zPo_*VO|VCCB;c15CYU};m-j^$CDBrSg4^ZuIg8n&&ETCI*RNd)06gRG0jz|<Kg*1T
z(wJo|$z%RdoTQmNWOKgf2W@}sYdxY7`{Vr9*@R+<iI7POKWw5V_9!utgAko_nJySL
zZA2`}7lNjg*oYDWW9f3G$3#!UTa!upc~8SGy03ZYzIyf0t>ZMCGr5;=8BUMS3DZQR
zVK%K=%F_E@mMlrqJPG-nCsL@1riLsHdGyc^eA;7@C*dg#WIl!GJX`Y9lF>Mxi^x?)
zYH9@0;b7nQ+w>U|X`*5Jn1vsyOzB}dj}jIS7Cd7zN)u56-bA){HC0SBL^Nz~fWO2#
z*&k8=Q_iv|7Zv?FmR(1f*xRjNK<aE@4uS(3_)lm-5OftKQ$ZsIBFh(&Pb;uZ_sYVy
z;IV=IqJY~YkLgR3?!(dqmvqEq4sYm_G>+5N(V_yr>DDq5>;z8~cx-OKcLKkYEF%H5
z5C1%lBgw0Z_S}Dp)o}o(r#rTe9rH&>3HlzX5YpJJ97BXrGFq>?pMHtrm>IT?8n?rA
zz5uqG#IWjan$BVlvECp$e$cpgBITlG7^~GvpA|zo%r;<LKaunJC$vED4lUq93mazL
zGK*pwlTOcUFiBUGEm;(^NnBNmRZs8R37x>OM!XI5d;tl_lq>SNNsyu+mP~2K9|h-l
zh%T~pnuoF)VLWtYrC@5<DjR?n_-$>U1`8H)Kyxf<z>fJz8iPod@ZGW%xVBSI{O0%g
zlXS`rY>|roEaF{-^<ShvPb1q5q22BeRp{t^hICLsE=2cvsQkQ|E4cXT0{#{^4*809
zvuHZyQ`(NBImC{$*jaT_i?~-J4Xjp8M$O75B~F4pFrt4OkS{4vV(renfc3WA-&YdS
z{0oSlrnzJ@4l?btEIg^u2Ck_WSr^cr)vS;JBDG{~8|@|mSHs(a0Rp!2d8h(;1{t%A
zvSMlluI<#)TzA4%<_a^)K^kB$iH?dqtrCzu+q{xkF%*J@YQOROoM$tjcNWWN&SCwV
z#I)><Kzp_1cbn|QjFCo~8_eUUqt5<*P{c7BaA@UZRpN2#sovdCRjI#)`YO#nLTFpi
zb=)F!5OS8#AdEPCO8_&2Hz%`dx^31nUE2x2sCtQA?qs!92Wv(LIRu>JZK_>VAO$8E
zEIgqfi6W`9XrG-H0;@>IbZXfTxQt}L4TyrdEj+tVM?o;|k>MuA>&EGX#X&w1AESiA
zST&FT$1Q@(!nbz&g}0@EZxml}Ks9W8^>qGAfV4GS0XfXL5YHnCeTFoNA&6IkSxxiI
zO9DP$tgKCXxXv!Cy%ejK>7;G%c3ueD-gmSHtj6MKvTn&!Kn%^Jd9`N^HA*cZJ3Cbi
z58^7FZVlN8SPXZUr%%!>pI5CI*E8GR4Ha4EdPGMHSPxW8FY@T4Dwxj7?P`airWvSn
zQA@~1RcEYcXWef8oVw2Ubcqf-3(!5FL1#=42d}9A(m&|=k9(a9{IZ_fb1%@ITWxP)
zyPm%0?o=K|kOpm)*AGTohaMzCX1SU~pFkfz&$!AB*9Af^72<+jp<3?r?u5VI>2=Qq
z%ymTodqNs1Z;F+ZKhrg9=&=qwe~<bsy)}Z4|LWin3cbf~j%d%M;E%yUx9=nBACCP|
zdwkILK@vTGqy^*7s59t}{NdAsHhnxAyb3zB?+*j|<fzxvOD7t%`bhzFU+QuedVK}G
zHe(Y=jDRzBL1>#R^f}~&$tHI&U~jarfi7E=O&(*?tZ9;1!YxjP<a3R|>ZEv>W>cQ+
z9Y<W1d!dY${FZ=D@7|P)uzHLE*RbjZ`^FdBxBueE9~~S8u%`{RT2Ox@t-31PWOgN8
zxJ|E)8(u%ug}T{8!Br1?@2}8KVW1AF<6iKkgi^BAGBVrq>KR%}$TrE|1?F1A)+nss
z+C?3vQU?wC2R=3XgTwJtI2}{weu9bpuM1~ehYc_tP&U`tNYg-`g66ic(=x3VAwM>{
zXe##lkiz8ajfxgTxDa|E(B|v$C68OG-4&?gAuJwok;b{Ip<81<U!>XjOShIR&biow
zz^M)`<k>Oo*TcJUx=3MTaB*_lx@+BSA)VN*CS0GgQ*mn<@ztJM#`cuTnUPo~h6s9g
z_+*hz84L9fi+C%QzXGB45)4)yEcq2Itst5u(Qy>AM8c(4Y^lESlt;4@`3-0J=*?>k
zhir}}c_Q-%jb4=GpBtZDhdS1SMz(C3Fy~IA%fpYUQYS*ne4oIU(sBCPP=l9Kzv=va
z_5Y^q&Q1O%wjY~^ZDR7z&Er>))l>7vAQ<;mc!5_s=)yV9?l7o@jQ_3~{RbVm@H4;c
leU}r$kKM#W?k}_GxqII`yYszo_x;n7{0eF#_EnZJ000(^-_Za7


From 08337c1af07e2d1e97f58aa748d9840759eb3e0d Mon Sep 17 00:00:00 2001
From: quintinali <tina.yuner@gmail.com>
Date: Mon, 3 Dec 2018 10:41:29 -0500
Subject: [PATCH 4/9] modify code as request

1. formatter
2. remove useless import
---
 .../WeblogDiscoveryEngine.java                |  11 +-
 .../WeblogDiscoveryEngineTest.java            |  14 ++-
 .../driver/EmbeddedElasticsearchServer.java   |  32 +++--
 .../mudrod/driver/PluginConfigurableNode.java |  19 ++-
 .../AbstractElasticsearchIntegrationTest.java |  47 +++++---
 .../apache/sdap/mudrod/main/ESDriverTest.java |  89 +++++++-------
 .../sdap/mudrod/main/TestMudrodEngine.java    |  48 +++++---
 .../weblog/structure/RequestUrlTest.java      | 111 +++++++++---------
 .../weblog/structure/TestApacheAccessLog.java |  37 +++---
 .../weblog/structure/log/GeoIpTest.java       |  29 +++--
 10 files changed, 246 insertions(+), 191 deletions(-)

diff --git a/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngine.java b/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngine.java
index 661ed8c..d6fe035 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngine.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngine.java
@@ -54,7 +54,8 @@ public WeblogDiscoveryEngine(Properties props, ESDriver es, SparkDriver spark) {
   /**
    * Get log file list from a directory
    *
-   * @param logDir path to directory containing logs either local or in HDFS.
+   * @param logDir
+   *          path to directory containing logs either local or in HDFS.
    * @return a list of log files
    */
   public List<String> getFileList(String logDir) {
@@ -105,7 +106,7 @@ public void preprocess() {
       startTime = System.currentTimeMillis();
       LOG.info("Processing logs dated {}", anInputList);
 
-     /* DiscoveryStepAbstract im = new ImportLogFile(this.props, this.es, this.spark);
+      DiscoveryStepAbstract im = new ImportLogFile(this.props, this.es, this.spark);
       im.execute();
 
       DiscoveryStepAbstract cd = new CrawlerDetection(this.props, this.es, this.spark);
@@ -118,15 +119,15 @@ public void preprocess() {
       ss.execute();
 
       DiscoveryStepAbstract rr = new RemoveRawLog(this.props, this.es, this.spark);
-      rr.execute();*/
+      rr.execute();
 
       endTime = System.currentTimeMillis();
 
       LOG.info("Web log preprocessing for logs dated {} complete. Time elapsed {} seconds.", anInputList, (endTime - startTime) / 1000);
     }
 
-    /*DiscoveryStepAbstract hg = new HistoryGenerator(this.props, this.es, this.spark);
-    hg.execute();*/
+    DiscoveryStepAbstract hg = new HistoryGenerator(this.props, this.es, this.spark);
+    hg.execute();
 
     DiscoveryStepAbstract cg = new ClickStreamGenerator(this.props, this.es, this.spark);
     cg.execute();
diff --git a/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java b/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java
index b31e266..ba3782d 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java
@@ -1,6 +1,18 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you 
+ * may not use this file except in compliance with the License. 
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.sdap.mudrod.discoveryengine;
 
-import static org.junit.Assert.*;
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileNotFoundException;
diff --git a/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java b/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java
index ef3587b..6566f29 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java
@@ -1,3 +1,16 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you 
+ * may not use this file except in compliance with the License. 
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.sdap.mudrod.driver;
 
 import org.apache.commons.io.FileUtils;
@@ -13,9 +26,7 @@
 import java.util.Collection;
 
 /**
- * Example implementation of an embedded elasticsearch server.
- *
- * @author Felix Müller
+ * embedded elasticsearch server.
  */
 public class EmbeddedElasticsearchServer {
 
@@ -35,7 +46,7 @@ public EmbeddedElasticsearchServer(String dataDirectory) {
     settingsBuilder.put("http.type", "netty3");
     settingsBuilder.put("transport.type", "netty3");
     settingsBuilder.put("cluster.name", "MurdorES").put("http.enabled", "false").put("path.data", dataDirectory).put("path.home", "/");
-      
+
     Settings settings = settingsBuilder.build();
     Collection plugins = Arrays.asList(Netty3Plugin.class);
     node = null;
@@ -46,7 +57,7 @@ public EmbeddedElasticsearchServer(String dataDirectory) {
       // TODO Auto-generated catch block
       e.printStackTrace();
     }
-    
+
     System.out.println(node.getNodeEnvironment().nodeId());
   }
 
@@ -55,13 +66,10 @@ public Client getClient() {
   }
 
   public void shutdown() {
-    /*try {
-      node.close();
-    } catch (IOException e) {
-      // TODO Auto-generated catch block
-      e.printStackTrace();
-    }
-    deleteDataDirectory();*/
+    /*
+     * try { node.close(); } catch (IOException e) { // TODO Auto-generated
+     * catch block e.printStackTrace(); } deleteDataDirectory();
+     */
   }
 
   private void deleteDataDirectory() {
diff --git a/core/src/test/java/org/apache/sdap/mudrod/driver/PluginConfigurableNode.java b/core/src/test/java/org/apache/sdap/mudrod/driver/PluginConfigurableNode.java
index c59ee2f..97ba265 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/driver/PluginConfigurableNode.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/driver/PluginConfigurableNode.java
@@ -1,3 +1,16 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you 
+ * may not use this file except in compliance with the License. 
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.sdap.mudrod.driver;
 
 import java.util.Collection;
@@ -8,7 +21,7 @@
 import org.elasticsearch.plugins.Plugin;
 
 public class PluginConfigurableNode extends Node {
-    public PluginConfigurableNode(Settings settings, Collection<Class<? extends Plugin>> classpathPlugins) {
-        super(InternalSettingsPreparer.prepareEnvironment(settings, null), classpathPlugins);
-    }
+  public PluginConfigurableNode(Settings settings, Collection<Class<? extends Plugin>> classpathPlugins) {
+    super(InternalSettingsPreparer.prepareEnvironment(settings, null), classpathPlugins);
+  }
 }
\ No newline at end of file
diff --git a/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java b/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java
index 01899ae..9a0deea 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java
@@ -1,3 +1,16 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you 
+ * may not use this file except in compliance with the License. 
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.sdap.mudrod.main;
 
 import org.apache.sdap.mudrod.driver.EmbeddedElasticsearchServer;
@@ -9,29 +22,29 @@
 import org.junit.BeforeClass;
 
 /**
- * This is a helper class the starts an embedded elasticsearch server
- * for each test.
+ * This is a helper class the starts an embedded elasticsearch server for each
+ * test.
  *
  * @author Felix Müller
  */
 public abstract class AbstractElasticsearchIntegrationTest {
 
-    private static EmbeddedElasticsearchServer embeddedElasticsearchServer;
+  private static EmbeddedElasticsearchServer embeddedElasticsearchServer;
 
-    @BeforeClass
-    public static void startEmbeddedElasticsearchServer() {
-        embeddedElasticsearchServer = new EmbeddedElasticsearchServer();
-    }
+  @BeforeClass
+  public static void startEmbeddedElasticsearchServer() {
+    embeddedElasticsearchServer = new EmbeddedElasticsearchServer();
+  }
 
-    @AfterClass
-    public static void shutdownEmbeddedElasticsearchServer() {
-        //embeddedElasticsearchServer.shutdown();
-    }
+  @AfterClass
+  public static void shutdownEmbeddedElasticsearchServer() {
+    // embeddedElasticsearchServer.shutdown();
+  }
 
-    /**
-     * By using this method you can access the embedded server.
-     */
-    protected Client getClient() {
-        return embeddedElasticsearchServer.getClient();
-    }
+  /**
+   * By using this method you can access the embedded server.
+   */
+  protected Client getClient() {
+    return embeddedElasticsearchServer.getClient();
+  }
 }
diff --git a/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java b/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java
index ea33766..45abbdb 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java
@@ -41,29 +41,30 @@
 import java.util.Properties;
 import java.util.concurrent.ExecutionException;
 
-public class ESDriverTest extends AbstractElasticsearchIntegrationTest{
+public class ESDriverTest extends AbstractElasticsearchIntegrationTest {
 
   private static final Logger LOG = LoggerFactory.getLogger(ESDriver.class);
   protected static final String ES_SETTINGS = "elastic_settings.json";
   protected static final String ES_MAPPINGS = "elastic_mappings.json";
-  
+
   static ESDriver es = null;
   static MudrodEngine mudrodEngine = null;
+
   @BeforeClass
-  public static void setUp(){
-	 mudrodEngine = new MudrodEngine();
-	 es = new ESDriver(mudrodEngine.loadConfig());	 
+  public static void setUp() {
+    mudrodEngine = new MudrodEngine();
+    es = new ESDriver(mudrodEngine.loadConfig());
   }
 
   @AfterClass
-  public static void tearDown(){
-     //es.destroyBulkProcessor();
-     //es.close();
+  public static void tearDown() {
+    // es.destroyBulkProcessor();
+    // es.close();
   }
-  
+
   @Test
   public void testESDriverProperties() {
-   
+
     Client client = es.getClient();
     assert client != null;
   }
@@ -85,7 +86,7 @@ public void testDestroyBulkProcessor() {
   }
 
   @Test
-  public void testPutMapping() {  
+  public void testPutMapping() {
 
     InputStream settingsStream = getClass().getClassLoader().getResourceAsStream(ES_SETTINGS);
     InputStream mappingsStream = getClass().getClassLoader().getResourceAsStream(ES_MAPPINGS);
@@ -117,7 +118,7 @@ public void testPutMapping() {
 
   @Test
   public void testCustomAnalyzingStringString() {
-       
+
     String str = "temp";
     try {
       String res = es.customAnalyzing("mudrod", str);
@@ -129,7 +130,7 @@ public void testCustomAnalyzingStringString() {
   }
 
   @Test
-  public void testCustomAnalyzingStringStringString() {   
+  public void testCustomAnalyzingStringStringString() {
     String str = "temp";
 
     try {
@@ -142,7 +143,7 @@ public void testCustomAnalyzingStringStringString() {
   }
 
   @Test
-  public void testCustomAnalyzingStringListOfString() {   
+  public void testCustomAnalyzingStringListOfString() {
 
     List<String> customlist = new ArrayList<>();
     customlist.add("string_a");
@@ -157,7 +158,7 @@ public void testCustomAnalyzingStringListOfString() {
   }
 
   @Test
-  public void testDeleteAllByQuery() {    
+  public void testDeleteAllByQuery() {
     es.deleteAllByQuery("mudrod", "MetadataLinkage", QueryBuilders.matchAllQuery());
 
     // String res = es.searchByQuery("mudrod", "MetadataLinkage",
@@ -167,18 +168,18 @@ public void testDeleteAllByQuery() {
 
   @Test
   public void testDeleteType() {
-        
+
     es.deleteType("mudrod", "MetadataLinkage");
   }
 
   @Test
   public void testGetTypeListWithPrefix() {
-    
+
     es.getTypeListWithPrefix("podaacsession", "sessionstats");
   }
 
   @Test
-  public void testGetIndexListWithPrefix() {    
+  public void testGetIndexListWithPrefix() {
 
     List<String> res = es.getIndexListWithPrefix("podaacsession");
     assert !res.isEmpty();
@@ -221,20 +222,16 @@ public void testClose() {
 
   @Test
   public void testRefreshIndex() {
-    
+
     es.refreshIndex();
   }
 
-/*  @Test
-  public void testMakeClient() {
-
-    try {
-      Client client = es.makeClient(mudrodEngine.loadConfig());
-      assert client != null;
-    } catch (IOException e) {
-      LOG.error("Error!", e);
-    }
-  }*/
+  /*
+   * @Test public void testMakeClient() {
+   * 
+   * try { Client client = es.makeClient(mudrodEngine.loadConfig()); assert
+   * client != null; } catch (IOException e) { LOG.error("Error!", e); } }
+   */
 
   @Test
   public void testGetClient() {
@@ -243,29 +240,23 @@ public void testGetClient() {
     assert client != null;
   }
 
-/*  @Test
-  public void testSetClient() {
-    
-    try {
-      Client client = es.makeClient(mudrodEngine.loadConfig());
-      es.setClient(client);
-      Client res = es.getClient();
-      assert res != null;
-    } catch (IOException e) {
-      LOG.error("Error!", e);
-    }
-  }
-*/
+  /*
+   * @Test public void testSetClient() {
+   * 
+   * try { Client client = es.makeClient(mudrodEngine.loadConfig());
+   * es.setClient(client); Client res = es.getClient(); assert res != null; }
+   * catch (IOException e) { LOG.error("Error!", e); } }
+   */
   @Test
   public void testGetBulkProcessor() {
-        
+
     BulkProcessor processor = es.getBulkProcessor();
     assert processor != null;
   }
 
   @Test
   public void testSetBulkProcessor() {
-      
+
     BulkProcessor begin = es.getBulkProcessor();
     es.setBulkProcessor(begin);
 
@@ -275,14 +266,14 @@ public void testSetBulkProcessor() {
 
   @Test
   public void testGenerateUpdateRequestStringStringStringStringObject() {
-    
+
     UpdateRequest res = es.generateUpdateRequest("mudrod", "MetadataLinkage", "id_1", "temp", "string_a");
     assert res != null;
   }
 
   @Test
   public void testGenerateUpdateRequestStringStringStringMapOfStringObject() {
-    
+
     Map<String, Object> result = new HashMap<String, Object>();
     result.put("temp", "string_a");
 
@@ -292,7 +283,7 @@ public void testGenerateUpdateRequestStringStringStringMapOfStringObject() {
 
   @Test
   public void testGetDocCountStringStringArray() {
-    
+
     String box[] = new String[] { "MetadataLinkage" };
     int res = es.getDocCount("mudrod", box);
 
@@ -301,7 +292,7 @@ public void testGetDocCountStringStringArray() {
 
   @Test
   public void testGetDocCountStringArrayStringArray() {
-    
+
     String begin[] = new String[] { "mudrod" };
     String box[] = new String[] { "MetadataLinkage" };
     int res = es.getDocCount(begin, box);
@@ -310,7 +301,7 @@ public void testGetDocCountStringArrayStringArray() {
 
   @Test
   public void testGetDocCountStringArrayStringArrayQueryBuilder() {
-    
+
     MatchAllQueryBuilder search = QueryBuilders.matchAllQuery();
 
     String begin[] = new String[] { "mudrod" };
diff --git a/core/src/test/java/org/apache/sdap/mudrod/main/TestMudrodEngine.java b/core/src/test/java/org/apache/sdap/mudrod/main/TestMudrodEngine.java
index 80bd52b..61bed12 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/main/TestMudrodEngine.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/main/TestMudrodEngine.java
@@ -23,9 +23,9 @@
 import org.junit.Test;
 
 /**
- * Initial test case for {@link org.apache.sdap.mudrod.main.MudrodEngine}, currently no 
- * tests are implemented, this is merely in place to get the JaCoCo test reporting to
- * work.
+ * Initial test case for {@link org.apache.sdap.mudrod.main.MudrodEngine},
+ * currently no tests are implemented, this is merely in place to get the JaCoCo
+ * test reporting to work.
  */
 public class TestMudrodEngine {
 
@@ -58,7 +58,8 @@ public void tearDown() throws Exception {
   }
 
   /**
-   * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#MudrodEngine()}.
+   * Test method for
+   * {@link org.apache.sdap.mudrod.main.MudrodEngine#MudrodEngine()}.
    */
   @Ignore
   @Test
@@ -67,7 +68,8 @@ public final void testMudrodEngine() {
   }
 
   /**
-   * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#startESDriver()}.
+   * Test method for
+   * {@link org.apache.sdap.mudrod.main.MudrodEngine#startESDriver()}.
    */
   @Ignore
   @Test
@@ -76,7 +78,8 @@ public final void testStartESDriver() {
   }
 
   /**
-   * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#startSparkDriver()}.
+   * Test method for
+   * {@link org.apache.sdap.mudrod.main.MudrodEngine#startSparkDriver()}.
    */
   @Ignore
   @Test
@@ -85,7 +88,8 @@ public final void testStartSparkDriver() {
   }
 
   /**
-   * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#getConfig()}.
+   * Test method for
+   * {@link org.apache.sdap.mudrod.main.MudrodEngine#getConfig()}.
    */
   @Ignore
   @Test
@@ -94,7 +98,8 @@ public final void testGetConfig() {
   }
 
   /**
-   * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#getESDriver()}.
+   * Test method for
+   * {@link org.apache.sdap.mudrod.main.MudrodEngine#getESDriver()}.
    */
   @Ignore
   @Test
@@ -103,7 +108,8 @@ public final void testGetESDriver() {
   }
 
   /**
-   * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#setESDriver(org.apache.sdap.mudrod.driver.ESDriver)}.
+   * Test method for
+   * {@link org.apache.sdap.mudrod.main.MudrodEngine#setESDriver(org.apache.sdap.mudrod.driver.ESDriver)}.
    */
   @Ignore
   @Test
@@ -112,7 +118,8 @@ public final void testSetESDriver() {
   }
 
   /**
-   * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#loadConfig()}.
+   * Test method for
+   * {@link org.apache.sdap.mudrod.main.MudrodEngine#loadConfig()}.
    */
   @Ignore
   @Test
@@ -121,7 +128,8 @@ public final void testLoadConfig() {
   }
 
   /**
-   * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#startLogIngest()}.
+   * Test method for
+   * {@link org.apache.sdap.mudrod.main.MudrodEngine#startLogIngest()}.
    */
   @Ignore
   @Test
@@ -130,7 +138,8 @@ public final void testStartLogIngest() {
   }
 
   /**
-   * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#startMetaIngest()}.
+   * Test method for
+   * {@link org.apache.sdap.mudrod.main.MudrodEngine#startMetaIngest()}.
    */
   @Ignore
   @Test
@@ -139,7 +148,8 @@ public final void testStartMetaIngest() {
   }
 
   /**
-   * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#startFullIngest()}.
+   * Test method for
+   * {@link org.apache.sdap.mudrod.main.MudrodEngine#startFullIngest()}.
    */
   @Ignore
   @Test
@@ -148,7 +158,8 @@ public final void testStartFullIngest() {
   }
 
   /**
-   * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#startProcessing()}.
+   * Test method for
+   * {@link org.apache.sdap.mudrod.main.MudrodEngine#startProcessing()}.
    */
   @Ignore
   @Test
@@ -166,7 +177,8 @@ public final void testEnd() {
   }
 
   /**
-   * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#main(java.lang.String[])}.
+   * Test method for
+   * {@link org.apache.sdap.mudrod.main.MudrodEngine#main(java.lang.String[])}.
    */
   @Ignore
   @Test
@@ -175,7 +187,8 @@ public final void testMain() {
   }
 
   /**
-   * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#getSparkDriver()}.
+   * Test method for
+   * {@link org.apache.sdap.mudrod.main.MudrodEngine#getSparkDriver()}.
    */
   @Ignore
   @Test
@@ -184,7 +197,8 @@ public final void testGetSparkDriver() {
   }
 
   /**
-   * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#setSparkDriver(org.apache.sdap.mudrod.driver.SparkDriver)}.
+   * Test method for
+   * {@link org.apache.sdap.mudrod.main.MudrodEngine#setSparkDriver(org.apache.sdap.mudrod.driver.SparkDriver)}.
    */
   @Ignore
   @Test
diff --git a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/RequestUrlTest.java b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/RequestUrlTest.java
index fec8e61..3f2ba5d 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/RequestUrlTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/RequestUrlTest.java
@@ -1,7 +1,18 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you 
+ * may not use this file except in compliance with the License. 
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.sdap.mudrod.weblog.structure;
 
-import static org.junit.Assert.*;
-
 import java.io.UnsupportedEncodingException;
 import java.util.Map;
 
@@ -12,64 +23,48 @@
 import org.junit.Test;
 
 public class RequestUrlTest {
-	
-//	@BeforeClass
-//    public void setUp(){
-//		RequestUrl url = new RequestUrl();
-//		
-//    }
-//	@Test
-//	public void testRequestUrl() {
-//		fail("Not yet implemented");
-//	}
+  
+  String strURL = "https://podaac.jpl.nasa.gov/datasetlist?ids=Collections:Measurement:SpatialCoverage:Platform:Sensor&values=SCAT_BYU_L3_OW_SIGMA0_ENHANCED:Sea%20Ice:Bering%20Sea:ERS-2:AMI&view=list";
+
+  @Test
+  public void testUrlPage() {
+    RequestUrl url = new RequestUrl();
+    String result = url.urlPage(strURL);
+    Assert.assertEquals("You did not pass urlPage function ", "https://podaac.jpl.nasa.gov/datasetlist", result);
+  }
 
-	@Test
-	public void testUrlPage() {
-		RequestUrl url = new RequestUrl();
-		String strURL = "https://podaac.jpl.nasa.gov/datasetlist?ids=Collections:Measurement:SpatialCoverage:Platform:Sensor&values=SCAT_BYU_L3_OW_SIGMA0_ENHANCED:Sea%20Ice:Bering%20Sea:ERS-2:AMI&view=list";
-		String result = url.urlPage(strURL);
-		//System.out.println(urlPage);
-		///fail("Not yet implemented");
-		Assert.assertEquals("You did not pass urlPage function ", "https://podaac.jpl.nasa.gov/datasetlist", result);
-	}
+  @Test
+  public void testuRLRequest() {
+    RequestUrl url = new RequestUrl();
+    Map<String, String> result = url.uRLRequest(strURL);
+    Assert.assertEquals("You did not pass uRLRequest function!", "list", result.get("view"));
+    Assert.assertEquals("You did not pass uRLRequest function!", "scat_byu_l3_ow_sigma0_enhanced:sea%20ice:bering%20sea:ers-2:ami", result.get("values"));
+    Assert.assertEquals("You did not pass uRLRequest function!", "collections:measurement:spatialcoverage:platform:sensor", result.get("ids"));
+  }
 
-	@Test
-	public void testuRLRequest() {
-		RequestUrl url = new RequestUrl();
-		String strURL = "https://podaac.jpl.nasa.gov/datasetlist?ids=Collections:Measurement:SpatialCoverage:Platform:Sensor&values=SCAT_BYU_L3_OW_SIGMA0_ENHANCED:Sea%20Ice:Bering%20Sea:ERS-2:AMI&view=list";
-		Map<String, String> result = url.uRLRequest(strURL);
-		Assert.assertEquals("You did not pass uRLRequest function!","list", result.get("view"));
-		Assert.assertEquals("You did not pass uRLRequest function!","scat_byu_l3_ow_sigma0_enhanced:sea%20ice:bering%20sea:ers-2:ami", result.get("values"));
-		Assert.assertEquals("You did not pass uRLRequest function!","collections:measurement:spatialcoverage:platform:sensor", result.get("ids"));
-	}
+  @Test
+  public void testGetSearchInfo() {
+    RequestUrl url = new RequestUrl();
+    String result = null;
+    try {
+      result = url.getSearchInfo(strURL);
+    } catch (UnsupportedEncodingException e) {
+      e.printStackTrace();
+    }
 
-	@Test
-	public void testGetSearchInfo() {
-		RequestUrl url = new RequestUrl();
-		String strURL = "https://podaac.jpl.nasa.gov/datasetlist?ids=Collections:Measurement:SpatialCoverage:Platform:Sensor&values=SCAT_BYU_L3_OW_SIGMA0_ENHANCED:Sea%20Ice:Bering%20Sea:ERS-2:AMI&view=list";
-		String result = null;
-		try {
-			result = url.getSearchInfo(strURL);
-		} catch (UnsupportedEncodingException e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
-		}
-		
-		Assert.assertEquals("You did not pass GetSearchInfo function ", "scat byu l3 ow sigma0 enhanced,sea ice,bering sea,ers 2,ami", result);
-	}
+    Assert.assertEquals("You did not pass GetSearchInfo function ", "scat byu l3 ow sigma0 enhanced,sea ice,bering sea,ers 2,ami", result);
+  }
 
-	@Test
-	public void testGetFilterInfo() {
-		RequestUrl url = new RequestUrl();
-		String strURL = "https://podaac.jpl.nasa.gov/datasetlist?ids=Collections:Measurement:SpatialCoverage:Platform:Sensor&values=SCAT_BYU_L3_OW_SIGMA0_ENHANCED:Sea%20Ice:Bering%20Sea:ERS-2:AMI&view=list";
-		try {
-			Map<String, String> result = url.getFilterInfo(strURL);
-			Assert.assertEquals("You did not pass GetFilterInfo function!","scat byu l3 ow sigma0 enhanced", result.get("collections"));
-			Assert.assertEquals("You did not pass GetFilterInfo function!","bering sea", result.get("spatialcoverage"));
-			Assert.assertEquals("You did not pass GetFilterInfo function!","ami", result.get("sensor"));
-		} catch (UnsupportedEncodingException e) {
-			// TODO Auto-generated catch block
-			e.printStackTrace();
-		}
-	}
+  @Test
+  public void testGetFilterInfo() {
+    RequestUrl url = new RequestUrl();
+    try {
+      Map<String, String> result = url.getFilterInfo(strURL);
+      Assert.assertEquals("You did not pass GetFilterInfo function!", "scat byu l3 ow sigma0 enhanced", result.get("collections"));
+      Assert.assertEquals("You did not pass GetFilterInfo function!", "bering sea", result.get("spatialcoverage"));
+      Assert.assertEquals("You did not pass GetFilterInfo function!", "ami", result.get("sensor"));
+    } catch (UnsupportedEncodingException e) {
+      e.printStackTrace();
+    }
+  }
 }
diff --git a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/TestApacheAccessLog.java b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/TestApacheAccessLog.java
index 1456bf6..7054cff 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/TestApacheAccessLog.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/TestApacheAccessLog.java
@@ -1,8 +1,8 @@
 /*
- * Licensed under the Apache License, Version 2.0 (the "License"); you
- * may not use this file except in compliance with the License.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you 
+ * may not use this file except in compliance with the License. 
  * You may obtain a copy of the License at
- *
+ * 
  * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
@@ -11,7 +11,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.sdap.mudrod.weblog.structure;
 
 import org.apache.sdap.mudrod.weblog.structure.log.ApacheAccessLog;
@@ -28,30 +27,28 @@
 
 import static org.junit.Assert.assertNotEquals;
 
-
 public class TestApacheAccessLog {
 
-    private static Properties testProperties = new Properties();
+  private static Properties testProperties = new Properties();
 
-    @BeforeClass
-    public static void loadProperties() throws IOException {
+  @BeforeClass
+  public static void loadProperties() throws IOException {
 
-        URL configURL = ClassLoader.getSystemClassLoader().getResource("config.properties");
+    URL configURL = ClassLoader.getSystemClassLoader().getResource("config.properties");
 
-        assert configURL != null : "Could not load config.properties";
-        try (InputStream instream = new FileInputStream(configURL.getFile())) {
-            testProperties.load(instream);
-        }
+    assert configURL != null : "Could not load config.properties";
+    try (InputStream instream = new FileInputStream(configURL.getFile())) {
+      testProperties.load(instream);
     }
+  }
 
-    @Test
-    public void testLogMatch() throws IOException, ParseException {
+  @Test
+  public void testLogMatch() throws IOException, ParseException {
 
+    String testLogLine = "198.118.243.84 - - [31/Dec/2017:23:59:20 +0000] \"GET /events?page=12&amp%25252525252525252525252525252525252525253Bsort=asc&order=field_location&sort=desc HTTP/1.1\" 200 86173";
 
-        String testLogLine = "198.118.243.84 - - [31/Dec/2017:23:59:20 +0000] \"GET /events?page=12&amp%25252525252525252525252525252525252525253Bsort=asc&order=field_location&sort=desc HTTP/1.1\" 200 86173";
+    String result = ApacheAccessLog.parseFromLogLine(testLogLine, testProperties);
 
-        String result = ApacheAccessLog.parseFromLogLine(testLogLine, testProperties);
-
-        assertNotEquals("Log line does not match", "{}", result);
-    }
+    assertNotEquals("Log line does not match", "{}", result);
+  }
 }
diff --git a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/log/GeoIpTest.java b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/log/GeoIpTest.java
index 31371d1..31a6a30 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/log/GeoIpTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/log/GeoIpTest.java
@@ -1,18 +1,29 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you 
+ * may not use this file except in compliance with the License. 
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.sdap.mudrod.weblog.structure.log;
 
-import static org.junit.Assert.*;
-
 import org.junit.Assert;
 import org.junit.Test;
 
 public class GeoIpTest {
 
-	@Test
-	public void testToLocation() {
-		GeoIp ip = new GeoIp();
-		String iptest = "185.10.104.194";
-		Coordinates result = ip.toLocation(iptest);
-		Assert.assertEquals("failed in geoip function!", "22.283001,114.150002", result.latlon);
-	}
+  @Test
+  public void testToLocation() {
+    GeoIp ip = new GeoIp();
+    String iptest = "185.10.104.194";
+    Coordinates result = ip.toLocation(iptest);
+    Assert.assertEquals("failed in geoip function!", "22.283001,114.150002", result.latlon);
+  }
 
 }

From 8b28adcfa5f23f80b720bbaac3851e0f9a802258 Mon Sep 17 00:00:00 2001
From: quintinali <tina.yuner@gmail.com>
Date: Fri, 7 Dec 2018 15:45:51 -0500
Subject: [PATCH 5/9] Solved most comments provided by the reviewer

---
 .../weblog/structure/session/SessionTree.java |   1 -
 .../WeblogDiscoveryEngineTest.java            |  93 +++--
 .../driver/EmbeddedElasticsearchServer.java   |   5 -
 .../AbstractElasticsearchIntegrationTest.java |   4 +-
 .../apache/sdap/mudrod/main/ESDriverTest.java |  51 +--
 .../sdap/mudrod/main/TestMudrodEngine.java    | 317 ------------------
 6 files changed, 43 insertions(+), 428 deletions(-)
 delete mode 100644 core/src/test/java/org/apache/sdap/mudrod/main/TestMudrodEngine.java

diff --git a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionTree.java b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionTree.java
index d4a28aa..24179cf 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionTree.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionTree.java
@@ -208,7 +208,6 @@ public List<ClickStream> getClickStreamList(Properties props) {
       RequestUrl requestURL = new RequestUrl();
       String viewquery = "";
       try {
-        //String infoStr = requestURL.getSearchInfo(viewnode.getRequest());
         String infoStr = requestURL.getSearchInfo(viewnode.getReferer());
         viewquery = es.customAnalyzing(props.getProperty(MudrodConstants.ES_INDEX_NAME), infoStr);
       } catch (UnsupportedEncodingException | InterruptedException | ExecutionException e) {
diff --git a/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java b/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java
index ba3782d..b61d8aa 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java
@@ -22,6 +22,7 @@
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 import org.apache.sdap.mudrod.driver.ESDriver;
 import org.apache.sdap.mudrod.driver.SparkDriver;
@@ -52,7 +53,6 @@ public static void setUp() {
 
   @AfterClass
   public static void tearDown() {
-    // TODO
   }
 
   private static String getTestDataPath() {
@@ -64,76 +64,63 @@ private static String getTestDataPath() {
 
   @Test
   public void testPreprocess() throws IOException {
-
     weblogEngine.preprocess();
-    testPreprocess_userHistory();
-    testPreprocess_clickStream();
+    testPreprocessUserHistory();
+    testPreprocessClickStream();
   }
 
-  private void testPreprocess_userHistory() throws IOException {
+  private void testPreprocessUserHistory() throws IOException {
     // compare user history data
     String userHistorycsvFile = getTestDataPath() + "/userHistoryMatrix.csv";
-    BufferedReader br = new BufferedReader(new FileReader(userHistorycsvFile));
-    String line = null;
-    HashMap<String, List<String>> map = new HashMap<>();
-    int i = 0;
-    List<String> header = new LinkedList<>();
-    while ((line = br.readLine()) != null) {
-      if (i == 0) {
-        String str[] = line.split(",");
-        for (String s : str) {
-          header.add(s);
-        }
-      } else {
-        String str[] = line.split(",");
-        for (int j = 1; j < str.length; j++) {
-          if (!str[j].equals("0")) {
-            if (!map.containsKey(str[0])) {
-              map.put(str[0], new ArrayList<>());
-            }
-            map.get(str[0]).add(header.get(j));
-          }
-        }
-      }
-      i += 1;
-    }
-
+    HashMap<String, List<String>> map = extractPairFromCSV(userHistorycsvFile);
     Assert.assertEquals("failed in history data result!", "195.219.98.7", String.join(",", map.get("sea surface topography")));
   }
 
-  private void testPreprocess_clickStream() throws IOException {
-    // TODO compare clickStream data
-    // String clickStreamcsvFile =
-    // "C:/Users/admin/Documents/GitHub/incubator-sdap-mudrod/core/clickStreamMatrix.csv";
+  private void testPreprocessClickStream() throws IOException {
     String clickStreamcsvFile = getTestDataPath() + "/clickStreamMatrix.csv";
     System.out.println(clickStreamcsvFile);
-    BufferedReader br = new BufferedReader(new FileReader(clickStreamcsvFile));
+    HashMap<String, List<String>> map = extractPairFromCSV(clickStreamcsvFile);
+    System.out.println(map);
+    Assert.assertEquals("failed in click stream result!", "\"ostm_l2_ost_ogdr_gps\"", String.join(",", map.get("sea surface topography")));
+  }
+  
+  private HashMap extractPairFromCSV(String csvfile){
+    
+    BufferedReader br = null;
+    try {
+      br = new BufferedReader(new FileReader(csvfile));
+    } catch (FileNotFoundException e) {
+      e.printStackTrace();
+    }
     String line = null;
     HashMap<String, List<String>> map = new HashMap<>();
-
     int i = 0;
     List<String> header = new LinkedList<>();
-    while ((line = br.readLine()) != null) {
-      if (i == 0) {
-        String str[] = line.split(",");
-        for (String s : str) {
-          header.add(s);
-        }
-      } else {
-        String str[] = line.split(",");
-        for (int j = 1; j < str.length; j++) {
-          if (!str[j].equals("0.0")) { //
-            if (!map.containsKey(str[0])) {
-              map.put(str[0], new ArrayList<>());
+    try {
+      while ((line = br.readLine()) != null) {
+        if (i == 0) {
+          String str[] = line.split(",");
+          for (String s : str) {
+            header.add(s);
+          }
+        } else {
+          String str[] = line.split(",");
+          for (int j = 1; j < str.length; j++) {
+            int value = Integer.parseInt(str[j]);
+            if (value > 0) {
+              if (!map.containsKey(str[0])) {
+                map.put(str[0], new ArrayList<>());
+              }
+              map.get(str[0]).add(header.get(j));
             }
-            map.get(str[0]).add(header.get(j));
           }
         }
+        i += 1;
       }
-      i += 1;
+    } catch (IOException e) {
+      e.printStackTrace();
     }
-    System.out.println(map);
-
-    Assert.assertEquals("failed in click stream result!", "\"ostm_l2_ost_ogdr_gps\"", String.join(",", map.get("sea surface topography")));
+    
+    return map;
   }
 }
diff --git a/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java b/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java
index 6566f29..c1fd7a3 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java
@@ -54,7 +54,6 @@ public EmbeddedElasticsearchServer(String dataDirectory) {
       node = new PluginConfigurableNode(settings, plugins).start();
       System.out.println(node.toString());
     } catch (NodeValidationException e) {
-      // TODO Auto-generated catch block
       e.printStackTrace();
     }
 
@@ -66,10 +65,6 @@ public Client getClient() {
   }
 
   public void shutdown() {
-    /*
-     * try { node.close(); } catch (IOException e) { // TODO Auto-generated
-     * catch block e.printStackTrace(); } deleteDataDirectory();
-     */
   }
 
   private void deleteDataDirectory() {
diff --git a/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java b/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java
index 9a0deea..3f9a137 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java
@@ -24,8 +24,6 @@
 /**
  * This is a helper class the starts an embedded elasticsearch server for each
  * test.
- *
- * @author Felix Müller
  */
 public abstract class AbstractElasticsearchIntegrationTest {
 
@@ -38,7 +36,7 @@ public static void startEmbeddedElasticsearchServer() {
 
   @AfterClass
   public static void shutdownEmbeddedElasticsearchServer() {
-    // embeddedElasticsearchServer.shutdown();
+    embeddedElasticsearchServer.shutdown();
   }
 
   /**
diff --git a/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java b/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java
index 45abbdb..6757584 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java
@@ -58,8 +58,8 @@ public static void setUp() {
 
   @AfterClass
   public static void tearDown() {
-    // es.destroyBulkProcessor();
-    // es.close();
+    es.destroyBulkProcessor();
+    es.close();
   }
 
   @Test
@@ -157,27 +157,6 @@ public void testCustomAnalyzingStringListOfString() {
     }
   }
 
-  @Test
-  public void testDeleteAllByQuery() {
-    es.deleteAllByQuery("mudrod", "MetadataLinkage", QueryBuilders.matchAllQuery());
-
-    // String res = es.searchByQuery("mudrod", "MetadataLinkage",
-    // QueryBuilders.matchAllQuery());
-    // assert res == ""||res == null;
-  }
-
-  @Test
-  public void testDeleteType() {
-
-    es.deleteType("mudrod", "MetadataLinkage");
-  }
-
-  @Test
-  public void testGetTypeListWithPrefix() {
-
-    es.getTypeListWithPrefix("podaacsession", "sessionstats");
-  }
-
   @Test
   public void testGetIndexListWithPrefix() {
 
@@ -214,25 +193,6 @@ public void testAutoComplete() {
     assert !res.isEmpty();
   }
 
-  @Test
-  public void testClose() {
-
-    es.close();
-  }
-
-  @Test
-  public void testRefreshIndex() {
-
-    es.refreshIndex();
-  }
-
-  /*
-   * @Test public void testMakeClient() {
-   * 
-   * try { Client client = es.makeClient(mudrodEngine.loadConfig()); assert
-   * client != null; } catch (IOException e) { LOG.error("Error!", e); } }
-   */
-
   @Test
   public void testGetClient() {
 
@@ -240,13 +200,6 @@ public void testGetClient() {
     assert client != null;
   }
 
-  /*
-   * @Test public void testSetClient() {
-   * 
-   * try { Client client = es.makeClient(mudrodEngine.loadConfig());
-   * es.setClient(client); Client res = es.getClient(); assert res != null; }
-   * catch (IOException e) { LOG.error("Error!", e); } }
-   */
   @Test
   public void testGetBulkProcessor() {
 
diff --git a/core/src/test/java/org/apache/sdap/mudrod/main/TestMudrodEngine.java b/core/src/test/java/org/apache/sdap/mudrod/main/TestMudrodEngine.java
deleted file mode 100644
index 61bed12..0000000
--- a/core/src/test/java/org/apache/sdap/mudrod/main/TestMudrodEngine.java
+++ /dev/null
@@ -1,317 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License"); you 
- * may not use this file except in compliance with the License. 
- * You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sdap.mudrod.main;
-
-import static org.junit.Assert.fail;
-
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * Initial test case for {@link org.apache.sdap.mudrod.main.MudrodEngine},
- * currently no tests are implemented, this is merely in place to get the JaCoCo
- * test reporting to work.
- */
-public class TestMudrodEngine {
-
-  /**
-   * @throws java.lang.Exception
-   */
-  @BeforeClass
-  public static void setUpBeforeClass() throws Exception {
-  }
-
-  /**
-   * @throws java.lang.Exception
-   */
-  @AfterClass
-  public static void tearDownAfterClass() throws Exception {
-  }
-
-  /**
-   * @throws java.lang.Exception
-   */
-  @Before
-  public void setUp() throws Exception {
-  }
-
-  /**
-   * @throws java.lang.Exception
-   */
-  @After
-  public void tearDown() throws Exception {
-  }
-
-  /**
-   * Test method for
-   * {@link org.apache.sdap.mudrod.main.MudrodEngine#MudrodEngine()}.
-   */
-  @Ignore
-  @Test
-  public final void testMudrodEngine() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for
-   * {@link org.apache.sdap.mudrod.main.MudrodEngine#startESDriver()}.
-   */
-  @Ignore
-  @Test
-  public final void testStartESDriver() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for
-   * {@link org.apache.sdap.mudrod.main.MudrodEngine#startSparkDriver()}.
-   */
-  @Ignore
-  @Test
-  public final void testStartSparkDriver() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for
-   * {@link org.apache.sdap.mudrod.main.MudrodEngine#getConfig()}.
-   */
-  @Ignore
-  @Test
-  public final void testGetConfig() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for
-   * {@link org.apache.sdap.mudrod.main.MudrodEngine#getESDriver()}.
-   */
-  @Ignore
-  @Test
-  public final void testGetESDriver() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for
-   * {@link org.apache.sdap.mudrod.main.MudrodEngine#setESDriver(org.apache.sdap.mudrod.driver.ESDriver)}.
-   */
-  @Ignore
-  @Test
-  public final void testSetESDriver() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for
-   * {@link org.apache.sdap.mudrod.main.MudrodEngine#loadConfig()}.
-   */
-  @Ignore
-  @Test
-  public final void testLoadConfig() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for
-   * {@link org.apache.sdap.mudrod.main.MudrodEngine#startLogIngest()}.
-   */
-  @Ignore
-  @Test
-  public final void testStartLogIngest() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for
-   * {@link org.apache.sdap.mudrod.main.MudrodEngine#startMetaIngest()}.
-   */
-  @Ignore
-  @Test
-  public final void testStartMetaIngest() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for
-   * {@link org.apache.sdap.mudrod.main.MudrodEngine#startFullIngest()}.
-   */
-  @Ignore
-  @Test
-  public final void testStartFullIngest() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for
-   * {@link org.apache.sdap.mudrod.main.MudrodEngine#startProcessing()}.
-   */
-  @Ignore
-  @Test
-  public final void testStartProcessing() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#end()}.
-   */
-  @Ignore
-  @Test
-  public final void testEnd() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for
-   * {@link org.apache.sdap.mudrod.main.MudrodEngine#main(java.lang.String[])}.
-   */
-  @Ignore
-  @Test
-  public final void testMain() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for
-   * {@link org.apache.sdap.mudrod.main.MudrodEngine#getSparkDriver()}.
-   */
-  @Ignore
-  @Test
-  public final void testGetSparkDriver() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for
-   * {@link org.apache.sdap.mudrod.main.MudrodEngine#setSparkDriver(org.apache.sdap.mudrod.driver.SparkDriver)}.
-   */
-  @Ignore
-  @Test
-  public final void testSetSparkDriver() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for {@link java.lang.Object#Object()}.
-   */
-  @Ignore
-  @Test
-  public final void testObject() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for {@link java.lang.Object#getClass()}.
-   */
-  @Ignore
-  @Test
-  public final void testGetClass() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for {@link java.lang.Object#hashCode()}.
-   */
-  @Ignore
-  @Test
-  public final void testHashCode() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for {@link java.lang.Object#equals(java.lang.Object)}.
-   */
-  @Ignore
-  @Test
-  public final void testEquals() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for {@link java.lang.Object#clone()}.
-   */
-  @Ignore
-  @Test
-  public final void testClone() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for {@link java.lang.Object#toString()}.
-   */
-  @Ignore
-  @Test
-  public final void testToString() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for {@link java.lang.Object#notify()}.
-   */
-  @Ignore
-  @Test
-  public final void testNotify() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for {@link java.lang.Object#notifyAll()}.
-   */
-  @Ignore
-  @Test
-  public final void testNotifyAll() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for {@link java.lang.Object#wait(long)}.
-   */
-  @Ignore
-  @Test
-  public final void testWaitLong() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for {@link java.lang.Object#wait(long, int)}.
-   */
-  @Ignore
-  @Test
-  public final void testWaitLongInt() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for {@link java.lang.Object#wait()}.
-   */
-  @Ignore
-  @Test
-  public final void testWait() {
-    fail("Not yet implemented"); // TODO
-  }
-
-  /**
-   * Test method for {@link java.lang.Object#finalize()}.
-   */
-  @Ignore
-  @Test
-  public final void testFinalize() {
-    fail("Not yet implemented"); // TODO
-  }
-
-}

From 122b228f7d51f745b86d6f7ab8c211988511166f Mon Sep 17 00:00:00 2001
From: quintinali <tina.yuner@gmail.com>
Date: Mon, 10 Dec 2018 12:00:09 -0500
Subject: [PATCH 6/9] Revised code based on Lewis's comments

---
 .../apache/sdap/mudrod/main/MudrodEngine.java |  3 +--
 .../WeblogDiscoveryEngineTest.java            | 25 +++++++++++--------
 .../driver/EmbeddedElasticsearchServer.java   |  5 ++++
 .../apache/sdap/mudrod/main/ESDriverTest.java | 21 ----------------
 .../weblog/structure/log/GeoIpTest.java       |  1 -
 5 files changed, 20 insertions(+), 35 deletions(-)

diff --git a/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java b/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java
index d9d01aa..4f024e1 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java
@@ -390,9 +390,8 @@ public static void main(String[] args) {
       }
       me.end();
     } catch (Exception e) {
-      HelpFormatter formatter = new HelpFormatter();
-      formatter.printHelp("MudrodEngine: 'dataDir' argument is mandatory. " + "User must also provide an ingest method.", options, true);
       LOG.error("Error whilst parsing command line.", e);
+      throw new RuntimeException(e);
     }
   }
 
diff --git a/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java b/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java
index b61d8aa..96e6510 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java
@@ -37,6 +37,8 @@
 public class WeblogDiscoveryEngineTest extends AbstractElasticsearchIntegrationTest {
 
   private static WeblogDiscoveryEngine weblogEngine = null;
+  
+  static String DIR_TESTDATA_ONE = "Testing_Data_1_3dayLog+Meta+Onto";
 
   @BeforeClass
   public static void setUp() {
@@ -44,7 +46,7 @@ public static void setUp() {
     Properties props = mudrodEngine.loadConfig();
     ESDriver es = new ESDriver(props);
     SparkDriver spark = new SparkDriver(props);
-    String dataDir = getTestDataPath();
+    String dataDir = getTestDataPath(DIR_TESTDATA_ONE);
     System.out.println(dataDir);
     props.setProperty(MudrodConstants.DATA_DIR, dataDir);
     MudrodEngine.loadPathConfig(mudrodEngine, dataDir);
@@ -55,11 +57,12 @@ public static void setUp() {
   public static void tearDown() {
   }
 
-  private static String getTestDataPath() {
-    File resourcesDirectory = new File("src/test/resources/");
-    String resourcedir = "/Testing_Data_1_3dayLog+Meta+Onto/";
-    String dataDir = resourcesDirectory.getAbsolutePath() + resourcedir;
-    return dataDir;
+  private static String getTestDataPath(String testDataDir) {
+    String path = WeblogDiscoveryEngineTest.class.getClassLoader().getResource(testDataDir).toString();
+    if(path.startsWith("file:/")){
+      path = path.replaceAll("file:/", "");
+    }
+    return path;
   }
 
   @Test
@@ -71,13 +74,14 @@ public void testPreprocess() throws IOException {
 
   private void testPreprocessUserHistory() throws IOException {
     // compare user history data
-    String userHistorycsvFile = getTestDataPath() + "/userHistoryMatrix.csv";
+    String userHistorycsvFile = getTestDataPath(DIR_TESTDATA_ONE) + "/userHistoryMatrix.csv";
+    System.out.println(userHistorycsvFile);
     HashMap<String, List<String>> map = extractPairFromCSV(userHistorycsvFile);
     Assert.assertEquals("failed in history data result!", "195.219.98.7", String.join(",", map.get("sea surface topography")));
   }
 
   private void testPreprocessClickStream() throws IOException {
-    String clickStreamcsvFile = getTestDataPath() + "/clickStreamMatrix.csv";
+    String clickStreamcsvFile = getTestDataPath(DIR_TESTDATA_ONE) + "/clickStreamMatrix.csv";
     System.out.println(clickStreamcsvFile);
     HashMap<String, List<String>> map = extractPairFromCSV(clickStreamcsvFile);
     System.out.println(map);
@@ -85,7 +89,6 @@ private void testPreprocessClickStream() throws IOException {
   }
   
   private HashMap extractPairFromCSV(String csvfile){
-    
     BufferedReader br = null;
     try {
       br = new BufferedReader(new FileReader(csvfile));
@@ -106,8 +109,8 @@ private HashMap extractPairFromCSV(String csvfile){
         } else {
           String str[] = line.split(",");
           for (int j = 1; j < str.length; j++) {
-            int value = Integer.parseInt(str[j]);
-            if (value > 0) {
+            double value = Double.parseDouble(str[j]);
+            if (value > 0.0) {
               if (!map.containsKey(str[0])) {
                 map.put(str[0], new ArrayList<>());
               }
diff --git a/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java b/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java
index c1fd7a3..6c34dee 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java
@@ -65,6 +65,11 @@ public Client getClient() {
   }
 
   public void shutdown() {
+    try {
+      node.close();
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
   }
 
   private void deleteDataDirectory() {
diff --git a/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java b/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java
index 6757584..c2c3a57 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java
@@ -64,7 +64,6 @@ public static void tearDown() {
 
   @Test
   public void testESDriverProperties() {
-
     Client client = es.getClient();
     assert client != null;
   }
@@ -87,7 +86,6 @@ public void testDestroyBulkProcessor() {
 
   @Test
   public void testPutMapping() {
-
     InputStream settingsStream = getClass().getClassLoader().getResourceAsStream(ES_SETTINGS);
     InputStream mappingsStream = getClass().getClassLoader().getResourceAsStream(ES_MAPPINGS);
 
@@ -118,7 +116,6 @@ public void testPutMapping() {
 
   @Test
   public void testCustomAnalyzingStringString() {
-
     String str = "temp";
     try {
       String res = es.customAnalyzing("mudrod", str);
@@ -132,7 +129,6 @@ public void testCustomAnalyzingStringString() {
   @Test
   public void testCustomAnalyzingStringStringString() {
     String str = "temp";
-
     try {
       String res = es.customAnalyzing("mudrod", "cody", str);
       assert res != "";
@@ -144,7 +140,6 @@ public void testCustomAnalyzingStringStringString() {
 
   @Test
   public void testCustomAnalyzingStringListOfString() {
-
     List<String> customlist = new ArrayList<>();
     customlist.add("string_a");
     customlist.add("string_b");
@@ -159,14 +154,12 @@ public void testCustomAnalyzingStringListOfString() {
 
   @Test
   public void testGetIndexListWithPrefix() {
-
     List<String> res = es.getIndexListWithPrefix("podaacsession");
     assert !res.isEmpty();
   }
 
   @Test
   public void testSearchByQueryStringStringString() {
-
     try {
       String res = es.searchByQuery("mudrod", "MetadataLinkage", "temp");
       assert res != null;
@@ -177,7 +170,6 @@ public void testSearchByQueryStringStringString() {
 
   @Test
   public void testSearchByQueryStringStringStringBoolean() {
-
     try {
       String res = es.searchByQuery("mudrod", "MetadataLinkage", "temp", false);
       assert res != null;
@@ -188,64 +180,53 @@ public void testSearchByQueryStringStringStringBoolean() {
 
   @Test
   public void testAutoComplete() {
-
     List<String> res = es.autoComplete("mudrod", "term");
     assert !res.isEmpty();
   }
 
   @Test
   public void testGetClient() {
-
     Client client = es.getClient();
     assert client != null;
   }
 
   @Test
   public void testGetBulkProcessor() {
-
     BulkProcessor processor = es.getBulkProcessor();
     assert processor != null;
   }
 
   @Test
   public void testSetBulkProcessor() {
-
     BulkProcessor begin = es.getBulkProcessor();
     es.setBulkProcessor(begin);
-
     BulkProcessor processor = es.getBulkProcessor();
     assert processor != null;
   }
 
   @Test
   public void testGenerateUpdateRequestStringStringStringStringObject() {
-
     UpdateRequest res = es.generateUpdateRequest("mudrod", "MetadataLinkage", "id_1", "temp", "string_a");
     assert res != null;
   }
 
   @Test
   public void testGenerateUpdateRequestStringStringStringMapOfStringObject() {
-
     Map<String, Object> result = new HashMap<String, Object>();
     result.put("temp", "string_a");
-
     UpdateRequest res = es.generateUpdateRequest("mudrod", "MetadataLinkage", "id_1", result);
     assert res != null;
   }
 
   @Test
   public void testGetDocCountStringStringArray() {
-
     String box[] = new String[] { "MetadataLinkage" };
     int res = es.getDocCount("mudrod", box);
-
     assert res > 0;
   }
 
   @Test
   public void testGetDocCountStringArrayStringArray() {
-
     String begin[] = new String[] { "mudrod" };
     String box[] = new String[] { "MetadataLinkage" };
     int res = es.getDocCount(begin, box);
@@ -254,9 +235,7 @@ public void testGetDocCountStringArrayStringArray() {
 
   @Test
   public void testGetDocCountStringArrayStringArrayQueryBuilder() {
-
     MatchAllQueryBuilder search = QueryBuilders.matchAllQuery();
-
     String begin[] = new String[] { "mudrod" };
     String box[] = new String[] { "MetadataLinkage" };
     int res = es.getDocCount(begin, box, search);
diff --git a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/log/GeoIpTest.java b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/log/GeoIpTest.java
index 31a6a30..1773ec0 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/log/GeoIpTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/log/GeoIpTest.java
@@ -25,5 +25,4 @@ public void testToLocation() {
     Coordinates result = ip.toLocation(iptest);
     Assert.assertEquals("failed in geoip function!", "22.283001,114.150002", result.latlon);
   }
-
 }

From 04c9cf38d02ab60f5b60793f1b647264863f5308 Mon Sep 17 00:00:00 2001
From: quintinali <tina.yuner@gmail.com>
Date: Tue, 15 Jan 2019 10:29:08 -0500
Subject: [PATCH 7/9] solved the embedded elasticsearch engine issue

---
 .../driver/EmbeddedElasticsearchServer.java   | 33 ++++++++++++++++++-
 .../AbstractElasticsearchIntegrationTest.java |  4 ++-
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java b/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java
index 6c34dee..2fe328d 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java
@@ -40,12 +40,13 @@ public EmbeddedElasticsearchServer() {
   }
 
   public EmbeddedElasticsearchServer(String dataDirectory) {
+    
     this.dataDirectory = dataDirectory;
 
     Settings.Builder settingsBuilder = Settings.builder();
     settingsBuilder.put("http.type", "netty3");
     settingsBuilder.put("transport.type", "netty3");
-    settingsBuilder.put("cluster.name", "MurdorES").put("http.enabled", "false").put("path.data", dataDirectory).put("path.home", "/");
+    settingsBuilder.put("cluster.name", "MudrodES").put("http.enabled", "true").put("path.data", dataDirectory).put("path.home", "/");
 
     Settings settings = settingsBuilder.build();
     Collection plugins = Arrays.asList(Netty3Plugin.class);
@@ -58,6 +59,36 @@ public EmbeddedElasticsearchServer(String dataDirectory) {
     }
 
     System.out.println(node.getNodeEnvironment().nodeId());
+    
+    
+   /* System.out.println("======= INTEGRATION TEST: START Embedded Elasticsearch Server ========");
+    Settings settings = Settings.builder().put("path.home", this.dataDirectory)
+            .put("transport.type", "local")
+            .put("network.host", "127.0.0.1")
+            .put("cluster.name", "MudrodES")
+            //.put("http.port", "9200-9300")
+            .put("http.type", "netty3")
+            //.put("transport.tcp.port", "9300-9400")
+            .put("http.enabled", false)
+           // .put("client.transport.sniff", true)
+            .build();
+    node = new Node(settings);
+   
+    
+    Settings set = node.settings();
+    try {
+      node.start();
+    } catch (NodeValidationException e) {
+      // TODO Auto-generated catch block
+      e.printStackTrace();
+    }
+    
+    final String clusterName = node.settings().get("cluster.name");
+    final String clusterport = node.settings().get("transport.tcp.port");
+    System.out.println("starting server with cluster-name: "+ clusterport);
+    
+    System.out.println(node.client());*/
+
   }
 
   public Client getClient() {
diff --git a/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java b/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java
index 3f9a137..6540189 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java
@@ -31,7 +31,9 @@ public abstract class AbstractElasticsearchIntegrationTest {
 
   @BeforeClass
   public static void startEmbeddedElasticsearchServer() {
-    embeddedElasticsearchServer = new EmbeddedElasticsearchServer();
+    if(embeddedElasticsearchServer == null){
+      embeddedElasticsearchServer = new EmbeddedElasticsearchServer();
+    }
   }
 
   @AfterClass

From 024d545818bb5e161f0c1f8a6fb965661193ff9d Mon Sep 17 00:00:00 2001
From: Hengning Cao <503662753@qq.com>
Date: Fri, 17 May 2019 12:54:54 -0400
Subject: [PATCH 8/9] Fitting Bugs

Fitting bugs
---
 .../sdap/mudrod/weblog/structure/log/ApacheAccessLog.java | 1 +
 .../mudrod/main/AbstractElasticsearchIntegrationTest.java | 3 ---
 .../java/org/apache/sdap/mudrod/main/ESDriverTest.java    | 2 --
 .../sdap/mudrod/weblog/structure/RequestUrlTest.java      | 8 ++++----
 .../sdap/mudrod/weblog/structure/TestApacheAccessLog.java | 8 ++++----
 .../sdap/mudrod/weblog/structure/log/GeoIpTest.java       | 1 +
 6 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/ApacheAccessLog.java b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/ApacheAccessLog.java
index 8224b33..abb03b4 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/ApacheAccessLog.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/ApacheAccessLog.java
@@ -67,6 +67,7 @@ public static String parseFromLogLine(String log, Properties props) throws IOExc
     String lineJson = "{}";
     matcher = p.matcher(log);
     if (!matcher.matches() || numFields != matcher.groupCount()) {
+      System.out.println("Number of fields does not match.");
       return lineJson;
     }
 
diff --git a/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java b/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java
index 6540189..fd76018 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java
@@ -13,12 +13,9 @@
  */
 package org.apache.sdap.mudrod.main;
 
-import org.apache.sdap.mudrod.driver.EmbeddedElasticsearchServer;
 import org.apache.sdap.mudrod.driver.EmbeddedElasticsearchServer;
 import org.elasticsearch.client.Client;
-import org.junit.After;
 import org.junit.AfterClass;
-import org.junit.Before;
 import org.junit.BeforeClass;
 
 /**
diff --git a/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java b/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java
index c2c3a57..77492b0 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java
@@ -25,9 +25,7 @@
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.action.bulk.BulkProcessor;
 import org.elasticsearch.action.update.UpdateRequest;
-import org.junit.After;
 import org.junit.AfterClass;
-import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.slf4j.Logger;
diff --git a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/RequestUrlTest.java b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/RequestUrlTest.java
index 3f2ba5d..f94ed5e 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/RequestUrlTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/RequestUrlTest.java
@@ -28,15 +28,15 @@ public class RequestUrlTest {
 
   @Test
   public void testUrlPage() {
-    RequestUrl url = new RequestUrl();
-    String result = url.urlPage(strURL);
+//    RequestUrl url = new RequestUrl();
+    String result = RequestUrl.urlPage(strURL);
     Assert.assertEquals("You did not pass urlPage function ", "https://podaac.jpl.nasa.gov/datasetlist", result);
   }
 
   @Test
   public void testuRLRequest() {
-    RequestUrl url = new RequestUrl();
-    Map<String, String> result = url.uRLRequest(strURL);
+//    RequestUrl url = new RequestUrl();
+    Map<String, String> result = RequestUrl.uRLRequest(strURL);
     Assert.assertEquals("You did not pass uRLRequest function!", "list", result.get("view"));
     Assert.assertEquals("You did not pass uRLRequest function!", "scat_byu_l3_ow_sigma0_enhanced:sea%20ice:bering%20sea:ers-2:ami", result.get("values"));
     Assert.assertEquals("You did not pass uRLRequest function!", "collections:measurement:spatialcoverage:platform:sensor", result.get("ids"));
diff --git a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/TestApacheAccessLog.java b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/TestApacheAccessLog.java
index 7054cff..748358e 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/TestApacheAccessLog.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/TestApacheAccessLog.java
@@ -23,7 +23,6 @@
 import java.net.URL;
 import java.text.ParseException;
 import java.util.Properties;
-import org.apache.sdap.mudrod.weblog.structure.log.ApacheAccessLog;
 
 import static org.junit.Assert.assertNotEquals;
 
@@ -44,9 +43,10 @@ public static void loadProperties() throws IOException {
 
   @Test
   public void testLogMatch() throws IOException, ParseException {
-
-    String testLogLine = "198.118.243.84 - - [31/Dec/2017:23:59:20 +0000] \"GET /events?page=12&amp%25252525252525252525252525252525252525253Bsort=asc&order=field_location&sort=desc HTTP/1.1\" 200 86173";
-
+    
+    // only have 7 fields? Should be 9? why not equal?
+//    String testLogLine = "198.118.243.84 - - [31/Dec/2017:23:59:20 +0000] \"GET /events?page=12&amp%25252525252525252525252525252525252525253Bsort=asc&order=field_location&sort=desc HTTP/1.1\" 200 86173";
+    String testLogLine =  "140.90.107.54 - - [01/Feb/2015:17:08:09 -0800] \"GET /datasetlist?search=VIIRS HTTP/1.1\" 200 82024 \"http://podaac.jpl.nasa.gov/\" \"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0\"";
     String result = ApacheAccessLog.parseFromLogLine(testLogLine, testProperties);
 
     assertNotEquals("Log line does not match", "{}", result);
diff --git a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/log/GeoIpTest.java b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/log/GeoIpTest.java
index 1773ec0..70773a1 100644
--- a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/log/GeoIpTest.java
+++ b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/log/GeoIpTest.java
@@ -22,6 +22,7 @@ public class GeoIpTest {
   public void testToLocation() {
     GeoIp ip = new GeoIp();
     String iptest = "185.10.104.194";
+    
     Coordinates result = ip.toLocation(iptest);
     Assert.assertEquals("failed in geoip function!", "22.283001,114.150002", result.latlon);
   }

From da22f9cf00b0417439e82214b10fe8f5c3cf00a3 Mon Sep 17 00:00:00 2001
From: quintinali <tina.yuner@gmail.com>
Date: Mon, 3 Jun 2019 09:38:16 -0700
Subject: [PATCH 9/9] Fix bugs

---
 .../java/org/apache/sdap/mudrod/driver/ESDriver.java | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/core/src/main/java/org/apache/sdap/mudrod/driver/ESDriver.java b/core/src/main/java/org/apache/sdap/mudrod/driver/ESDriver.java
index dc1b4c8..666629f 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/driver/ESDriver.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/driver/ESDriver.java
@@ -490,13 +490,11 @@ protected Client makeClient(Properties props) throws IOException {
 
     Client client = null;
 
-    if (hosts != null && port > 1) {
-      try (TransportClient transportClient = new ESTransportClient(settings)) {
-        for (String host : hosts) {
-          transportClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), port));
-        }
-        return transportClient;
-      }
+    if (hosts != null && port > 1) {  
+      TransportClient transportClient = new ESTransportClient(settings);
+      for (String host : hosts)
+        transportClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), port));
+      return transportClient;
     } else if (clusterName != null) {
       node = new Node(settings);
       client = node.client();