diff --git a/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngine.java b/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngine.java index b2cdb9f..d6fe035 100644 --- a/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngine.java +++ b/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngine.java @@ -54,7 +54,8 @@ public WeblogDiscoveryEngine(Properties props, ESDriver es, SparkDriver spark) { /** * Get log file list from a directory * - * @param logDir path to directory containing logs either local or in HDFS. + * @param logDir + * path to directory containing logs either local or in HDFS. * @return a list of log files */ public List getFileList(String logDir) { diff --git a/core/src/main/java/org/apache/sdap/mudrod/driver/ESDriver.java b/core/src/main/java/org/apache/sdap/mudrod/driver/ESDriver.java index dc1b4c8..666629f 100644 --- a/core/src/main/java/org/apache/sdap/mudrod/driver/ESDriver.java +++ b/core/src/main/java/org/apache/sdap/mudrod/driver/ESDriver.java @@ -490,13 +490,11 @@ protected Client makeClient(Properties props) throws IOException { Client client = null; - if (hosts != null && port > 1) { - try (TransportClient transportClient = new ESTransportClient(settings)) { - for (String host : hosts) { - transportClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), port)); - } - return transportClient; - } + if (hosts != null && port > 1) { + TransportClient transportClient = new ESTransportClient(settings); + for (String host : hosts) + transportClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), port)); + return transportClient; } else if (clusterName != null) { node = new Node(settings); client = node.client(); diff --git a/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java b/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java index 02cf1e6..bb92ceb 100644 --- a/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java +++ b/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java @@ -396,7 +396,7 @@ public static void main(String[] args) { } } - private static void loadPathConfig(MudrodEngine me, String dataDir) { + public static void loadPathConfig(MudrodEngine me, String dataDir) { me.props.put(MudrodConstants.ONTOLOGY_INPUT_PATH, dataDir + "SWEET_ocean/"); me.props.put(MudrodConstants.ONTOLOGY_PATH, dataDir + "ocean_triples.csv"); me.props.put(MudrodConstants.USER_HISTORY_PATH, dataDir + "userhistorymatrix.csv"); diff --git a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/ClickStreamGenerator.java b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/ClickStreamGenerator.java index 2c38d29..8d115aa 100644 --- a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/ClickStreamGenerator.java +++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/ClickStreamGenerator.java @@ -71,5 +71,4 @@ public Object execute() { public Object execute(Object o) { return null; } - } diff --git a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionStatistic.java b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionStatistic.java index c89160f..a40693e 100644 --- a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionStatistic.java +++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionStatistic.java @@ -229,7 +229,7 @@ public int processSession(ESDriver es, String sessionId) throws IOException, Int String[] keywordList = keywords.split(","); for (String item : items) { if (!Arrays.asList(keywordList).contains(item)) { - keywords = keywords + item + ","; + keywords = keywords + "," + item + ","; } } } diff --git a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/ApacheAccessLog.java b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/ApacheAccessLog.java index 6ffd3be..80fc3a7 100644 --- a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/ApacheAccessLog.java +++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/ApacheAccessLog.java @@ -67,6 +67,7 @@ public static String parseFromLogLine(String log, Properties props) throws Parse String lineJson = "{}"; matcher = p.matcher(log); if (!matcher.matches() || numFields != matcher.groupCount()) { + System.out.println("Number of fields does not match."); return lineJson; } diff --git a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionTree.java b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionTree.java index 5531f83..24179cf 100644 --- a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionTree.java +++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionTree.java @@ -26,10 +26,13 @@ import java.io.UnsupportedEncodingException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.concurrent.ExecutionException; /** @@ -205,7 +208,7 @@ public List getClickStreamList(Properties props) { RequestUrl requestURL = new RequestUrl(); String viewquery = ""; try { - String infoStr = requestURL.getSearchInfo(viewnode.getRequest()); + String infoStr = requestURL.getSearchInfo(viewnode.getReferer()); viewquery = es.customAnalyzing(props.getProperty(MudrodConstants.ES_INDEX_NAME), infoStr); } catch (UnsupportedEncodingException | InterruptedException | ExecutionException e) { LOG.warn("Exception getting search info. Ignoring...", e); @@ -222,6 +225,8 @@ public List getClickStreamList(Properties props) { if (viewquery != null && !"".equals(viewquery)) { String[] queries = viewquery.trim().split(","); + Set queryset = new HashSet<>(); + queryset.addAll(Arrays.asList(queries)); if (queries.length > 0) { for (String query : queries) { ClickStream data = new ClickStream(query, dataset, download); diff --git a/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java b/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java new file mode 100644 index 0000000..96e6510 --- /dev/null +++ b/core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java @@ -0,0 +1,129 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sdap.mudrod.discoveryengine; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import org.apache.sdap.mudrod.driver.ESDriver; +import org.apache.sdap.mudrod.driver.SparkDriver; +import org.apache.sdap.mudrod.main.AbstractElasticsearchIntegrationTest; +import org.apache.sdap.mudrod.main.MudrodConstants; +import org.apache.sdap.mudrod.main.MudrodEngine; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class WeblogDiscoveryEngineTest extends AbstractElasticsearchIntegrationTest { + + private static WeblogDiscoveryEngine weblogEngine = null; + + static String DIR_TESTDATA_ONE = "Testing_Data_1_3dayLog+Meta+Onto"; + + @BeforeClass + public static void setUp() { + MudrodEngine mudrodEngine = new MudrodEngine(); + Properties props = mudrodEngine.loadConfig(); + ESDriver es = new ESDriver(props); + SparkDriver spark = new SparkDriver(props); + String dataDir = getTestDataPath(DIR_TESTDATA_ONE); + System.out.println(dataDir); + props.setProperty(MudrodConstants.DATA_DIR, dataDir); + MudrodEngine.loadPathConfig(mudrodEngine, dataDir); + weblogEngine = new WeblogDiscoveryEngine(props, es, spark); + } + + @AfterClass + public static void tearDown() { + } + + private static String getTestDataPath(String testDataDir) { + String path = WeblogDiscoveryEngineTest.class.getClassLoader().getResource(testDataDir).toString(); + if(path.startsWith("file:/")){ + path = path.replaceAll("file:/", ""); + } + return path; + } + + @Test + public void testPreprocess() throws IOException { + weblogEngine.preprocess(); + testPreprocessUserHistory(); + testPreprocessClickStream(); + } + + private void testPreprocessUserHistory() throws IOException { + // compare user history data + String userHistorycsvFile = getTestDataPath(DIR_TESTDATA_ONE) + "/userHistoryMatrix.csv"; + System.out.println(userHistorycsvFile); + HashMap> map = extractPairFromCSV(userHistorycsvFile); + Assert.assertEquals("failed in history data result!", "195.219.98.7", String.join(",", map.get("sea surface topography"))); + } + + private void testPreprocessClickStream() throws IOException { + String clickStreamcsvFile = getTestDataPath(DIR_TESTDATA_ONE) + "/clickStreamMatrix.csv"; + System.out.println(clickStreamcsvFile); + HashMap> map = extractPairFromCSV(clickStreamcsvFile); + System.out.println(map); + Assert.assertEquals("failed in click stream result!", "\"ostm_l2_ost_ogdr_gps\"", String.join(",", map.get("sea surface topography"))); + } + + private HashMap extractPairFromCSV(String csvfile){ + BufferedReader br = null; + try { + br = new BufferedReader(new FileReader(csvfile)); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + String line = null; + HashMap> map = new HashMap<>(); + int i = 0; + List header = new LinkedList<>(); + try { + while ((line = br.readLine()) != null) { + if (i == 0) { + String str[] = line.split(","); + for (String s : str) { + header.add(s); + } + } else { + String str[] = line.split(","); + for (int j = 1; j < str.length; j++) { + double value = Double.parseDouble(str[j]); + if (value > 0.0) { + if (!map.containsKey(str[0])) { + map.put(str[0], new ArrayList<>()); + } + map.get(str[0]).add(header.get(j)); + } + } + } + i += 1; + } + } catch (IOException e) { + e.printStackTrace(); + } + + return map; + } +} diff --git a/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java b/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java new file mode 100644 index 0000000..2fe328d --- /dev/null +++ b/core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java @@ -0,0 +1,113 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sdap.mudrod.driver; + +import org.apache.commons.io.FileUtils; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.NodeValidationException; +import org.elasticsearch.transport.Netty3Plugin; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; + +/** + * embedded elasticsearch server. + */ +public class EmbeddedElasticsearchServer { + + private static final String DEFAULT_DATA_DIRECTORY = "target/elasticsearch-data"; + + private Node node; + private final String dataDirectory; + + public EmbeddedElasticsearchServer() { + this(DEFAULT_DATA_DIRECTORY); + } + + public EmbeddedElasticsearchServer(String dataDirectory) { + + this.dataDirectory = dataDirectory; + + Settings.Builder settingsBuilder = Settings.builder(); + settingsBuilder.put("http.type", "netty3"); + settingsBuilder.put("transport.type", "netty3"); + settingsBuilder.put("cluster.name", "MudrodES").put("http.enabled", "true").put("path.data", dataDirectory).put("path.home", "/"); + + Settings settings = settingsBuilder.build(); + Collection plugins = Arrays.asList(Netty3Plugin.class); + node = null; + try { + node = new PluginConfigurableNode(settings, plugins).start(); + System.out.println(node.toString()); + } catch (NodeValidationException e) { + e.printStackTrace(); + } + + System.out.println(node.getNodeEnvironment().nodeId()); + + + /* System.out.println("======= INTEGRATION TEST: START Embedded Elasticsearch Server ========"); + Settings settings = Settings.builder().put("path.home", this.dataDirectory) + .put("transport.type", "local") + .put("network.host", "127.0.0.1") + .put("cluster.name", "MudrodES") + //.put("http.port", "9200-9300") + .put("http.type", "netty3") + //.put("transport.tcp.port", "9300-9400") + .put("http.enabled", false) + // .put("client.transport.sniff", true) + .build(); + node = new Node(settings); + + + Settings set = node.settings(); + try { + node.start(); + } catch (NodeValidationException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + final String clusterName = node.settings().get("cluster.name"); + final String clusterport = node.settings().get("transport.tcp.port"); + System.out.println("starting server with cluster-name: "+ clusterport); + + System.out.println(node.client());*/ + + } + + public Client getClient() { + return node.client(); + } + + public void shutdown() { + try { + node.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + private void deleteDataDirectory() { + try { + FileUtils.deleteDirectory(new File(dataDirectory)); + } catch (IOException e) { + throw new RuntimeException("Could not delete data directory of embedded elasticsearch server", e); + } + } +} diff --git a/core/src/test/java/org/apache/sdap/mudrod/driver/PluginConfigurableNode.java b/core/src/test/java/org/apache/sdap/mudrod/driver/PluginConfigurableNode.java new file mode 100644 index 0000000..97ba265 --- /dev/null +++ b/core/src/test/java/org/apache/sdap/mudrod/driver/PluginConfigurableNode.java @@ -0,0 +1,27 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sdap.mudrod.driver; + +import java.util.Collection; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.internal.InternalSettingsPreparer; +import org.elasticsearch.plugins.Plugin; + +public class PluginConfigurableNode extends Node { + public PluginConfigurableNode(Settings settings, Collection> classpathPlugins) { + super(InternalSettingsPreparer.prepareEnvironment(settings, null), classpathPlugins); + } +} \ No newline at end of file diff --git a/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java b/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java new file mode 100644 index 0000000..fd76018 --- /dev/null +++ b/core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java @@ -0,0 +1,47 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sdap.mudrod.main; + +import org.apache.sdap.mudrod.driver.EmbeddedElasticsearchServer; +import org.elasticsearch.client.Client; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +/** + * This is a helper class the starts an embedded elasticsearch server for each + * test. + */ +public abstract class AbstractElasticsearchIntegrationTest { + + private static EmbeddedElasticsearchServer embeddedElasticsearchServer; + + @BeforeClass + public static void startEmbeddedElasticsearchServer() { + if(embeddedElasticsearchServer == null){ + embeddedElasticsearchServer = new EmbeddedElasticsearchServer(); + } + } + + @AfterClass + public static void shutdownEmbeddedElasticsearchServer() { + embeddedElasticsearchServer.shutdown(); + } + + /** + * By using this method you can access the embedded server. + */ + protected Client getClient() { + return embeddedElasticsearchServer.getClient(); + } +} diff --git a/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java b/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java new file mode 100644 index 0000000..77492b0 --- /dev/null +++ b/core/src/test/java/org/apache/sdap/mudrod/main/ESDriverTest.java @@ -0,0 +1,242 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sdap.mudrod.main; + +import org.apache.commons.io.IOUtils; +import org.apache.sdap.mudrod.main.MudrodConstants; +import org.apache.sdap.mudrod.main.MudrodEngine; +import org.apache.sdap.mudrod.driver.ESDriver; +import org.apache.sdap.mudrod.driver.EmbeddedElasticsearchServer; +import org.elasticsearch.client.Client; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.action.bulk.BulkProcessor; +import org.elasticsearch.action.update.UpdateRequest; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ExecutionException; + +public class ESDriverTest extends AbstractElasticsearchIntegrationTest { + + private static final Logger LOG = LoggerFactory.getLogger(ESDriver.class); + protected static final String ES_SETTINGS = "elastic_settings.json"; + protected static final String ES_MAPPINGS = "elastic_mappings.json"; + + static ESDriver es = null; + static MudrodEngine mudrodEngine = null; + + @BeforeClass + public static void setUp() { + mudrodEngine = new MudrodEngine(); + es = new ESDriver(mudrodEngine.loadConfig()); + } + + @AfterClass + public static void tearDown() { + es.destroyBulkProcessor(); + es.close(); + } + + @Test + public void testESDriverProperties() { + Client client = es.getClient(); + assert client != null; + } + + @Test + public void testCreateBulkProcessor() { + es.createBulkProcessor(); + BulkProcessor processor = es.getBulkProcessor(); + assert processor != null; + } + + @Test + public void testDestroyBulkProcessor() { + es.createBulkProcessor(); + es.destroyBulkProcessor(); + + BulkProcessor processor = es.getBulkProcessor(); + assert processor == null; + } + + @Test + public void testPutMapping() { + InputStream settingsStream = getClass().getClassLoader().getResourceAsStream(ES_SETTINGS); + InputStream mappingsStream = getClass().getClassLoader().getResourceAsStream(ES_MAPPINGS); + + JSONObject settingsJSON = null; + JSONObject mappingJSON = null; + + Properties props = mudrodEngine.loadConfig(); + try { + settingsJSON = new JSONObject(IOUtils.toString(settingsStream)); + } catch (JSONException | IOException e1) { + LOG.error("Error reading Elasticsearch settings!", e1); + } + + try { + mappingJSON = new JSONObject(IOUtils.toString(mappingsStream)); + } catch (JSONException | IOException e1) { + LOG.error("Error reading Elasticsearch mappings!", e1); + } + + try { + if (settingsJSON != null && mappingJSON != null) { + es.putMapping(props.getProperty(MudrodConstants.ES_INDEX_NAME), settingsJSON.toString(), mappingJSON.toString()); + } + } catch (IOException e) { + LOG.error("Error entering Elasticsearch Mappings!", e); + } + } + + @Test + public void testCustomAnalyzingStringString() { + String str = "temp"; + try { + String res = es.customAnalyzing("mudrod", str); + assert res != ""; + assert res != null; + } catch (InterruptedException | ExecutionException e) { + LOG.error("Error!", e); + } + } + + @Test + public void testCustomAnalyzingStringStringString() { + String str = "temp"; + try { + String res = es.customAnalyzing("mudrod", "cody", str); + assert res != ""; + assert res != null; + } catch (InterruptedException | ExecutionException e) { + LOG.error("Error!", e); + } + } + + @Test + public void testCustomAnalyzingStringListOfString() { + List customlist = new ArrayList<>(); + customlist.add("string_a"); + customlist.add("string_b"); + + try { + List res = es.customAnalyzing("mudrod", customlist); + assert !res.isEmpty(); + } catch (InterruptedException | ExecutionException e) { + LOG.error("Error!", e); + } + } + + @Test + public void testGetIndexListWithPrefix() { + List res = es.getIndexListWithPrefix("podaacsession"); + assert !res.isEmpty(); + } + + @Test + public void testSearchByQueryStringStringString() { + try { + String res = es.searchByQuery("mudrod", "MetadataLinkage", "temp"); + assert res != null; + } catch (IOException | InterruptedException | ExecutionException e) { + LOG.error("Error!", e); + } + } + + @Test + public void testSearchByQueryStringStringStringBoolean() { + try { + String res = es.searchByQuery("mudrod", "MetadataLinkage", "temp", false); + assert res != null; + } catch (IOException | InterruptedException | ExecutionException e) { + LOG.error("Error!", e); + } + } + + @Test + public void testAutoComplete() { + List res = es.autoComplete("mudrod", "term"); + assert !res.isEmpty(); + } + + @Test + public void testGetClient() { + Client client = es.getClient(); + assert client != null; + } + + @Test + public void testGetBulkProcessor() { + BulkProcessor processor = es.getBulkProcessor(); + assert processor != null; + } + + @Test + public void testSetBulkProcessor() { + BulkProcessor begin = es.getBulkProcessor(); + es.setBulkProcessor(begin); + BulkProcessor processor = es.getBulkProcessor(); + assert processor != null; + } + + @Test + public void testGenerateUpdateRequestStringStringStringStringObject() { + UpdateRequest res = es.generateUpdateRequest("mudrod", "MetadataLinkage", "id_1", "temp", "string_a"); + assert res != null; + } + + @Test + public void testGenerateUpdateRequestStringStringStringMapOfStringObject() { + Map result = new HashMap(); + result.put("temp", "string_a"); + UpdateRequest res = es.generateUpdateRequest("mudrod", "MetadataLinkage", "id_1", result); + assert res != null; + } + + @Test + public void testGetDocCountStringStringArray() { + String box[] = new String[] { "MetadataLinkage" }; + int res = es.getDocCount("mudrod", box); + assert res > 0; + } + + @Test + public void testGetDocCountStringArrayStringArray() { + String begin[] = new String[] { "mudrod" }; + String box[] = new String[] { "MetadataLinkage" }; + int res = es.getDocCount(begin, box); + assert res > 0; + } + + @Test + public void testGetDocCountStringArrayStringArrayQueryBuilder() { + MatchAllQueryBuilder search = QueryBuilders.matchAllQuery(); + String begin[] = new String[] { "mudrod" }; + String box[] = new String[] { "MetadataLinkage" }; + int res = es.getDocCount(begin, box, search); + assert res > 0; + } +} diff --git a/core/src/test/java/org/apache/sdap/mudrod/main/TestMudrodEngine.java b/core/src/test/java/org/apache/sdap/mudrod/main/TestMudrodEngine.java deleted file mode 100644 index 80bd52b..0000000 --- a/core/src/test/java/org/apache/sdap/mudrod/main/TestMudrodEngine.java +++ /dev/null @@ -1,303 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sdap.mudrod.main; - -import static org.junit.Assert.fail; - -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; - -/** - * Initial test case for {@link org.apache.sdap.mudrod.main.MudrodEngine}, currently no - * tests are implemented, this is merely in place to get the JaCoCo test reporting to - * work. - */ -public class TestMudrodEngine { - - /** - * @throws java.lang.Exception - */ - @BeforeClass - public static void setUpBeforeClass() throws Exception { - } - - /** - * @throws java.lang.Exception - */ - @AfterClass - public static void tearDownAfterClass() throws Exception { - } - - /** - * @throws java.lang.Exception - */ - @Before - public void setUp() throws Exception { - } - - /** - * @throws java.lang.Exception - */ - @After - public void tearDown() throws Exception { - } - - /** - * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#MudrodEngine()}. - */ - @Ignore - @Test - public final void testMudrodEngine() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#startESDriver()}. - */ - @Ignore - @Test - public final void testStartESDriver() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#startSparkDriver()}. - */ - @Ignore - @Test - public final void testStartSparkDriver() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#getConfig()}. - */ - @Ignore - @Test - public final void testGetConfig() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#getESDriver()}. - */ - @Ignore - @Test - public final void testGetESDriver() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#setESDriver(org.apache.sdap.mudrod.driver.ESDriver)}. - */ - @Ignore - @Test - public final void testSetESDriver() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#loadConfig()}. - */ - @Ignore - @Test - public final void testLoadConfig() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#startLogIngest()}. - */ - @Ignore - @Test - public final void testStartLogIngest() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#startMetaIngest()}. - */ - @Ignore - @Test - public final void testStartMetaIngest() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#startFullIngest()}. - */ - @Ignore - @Test - public final void testStartFullIngest() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#startProcessing()}. - */ - @Ignore - @Test - public final void testStartProcessing() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#end()}. - */ - @Ignore - @Test - public final void testEnd() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#main(java.lang.String[])}. - */ - @Ignore - @Test - public final void testMain() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#getSparkDriver()}. - */ - @Ignore - @Test - public final void testGetSparkDriver() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link org.apache.sdap.mudrod.main.MudrodEngine#setSparkDriver(org.apache.sdap.mudrod.driver.SparkDriver)}. - */ - @Ignore - @Test - public final void testSetSparkDriver() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link java.lang.Object#Object()}. - */ - @Ignore - @Test - public final void testObject() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link java.lang.Object#getClass()}. - */ - @Ignore - @Test - public final void testGetClass() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link java.lang.Object#hashCode()}. - */ - @Ignore - @Test - public final void testHashCode() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link java.lang.Object#equals(java.lang.Object)}. - */ - @Ignore - @Test - public final void testEquals() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link java.lang.Object#clone()}. - */ - @Ignore - @Test - public final void testClone() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link java.lang.Object#toString()}. - */ - @Ignore - @Test - public final void testToString() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link java.lang.Object#notify()}. - */ - @Ignore - @Test - public final void testNotify() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link java.lang.Object#notifyAll()}. - */ - @Ignore - @Test - public final void testNotifyAll() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link java.lang.Object#wait(long)}. - */ - @Ignore - @Test - public final void testWaitLong() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link java.lang.Object#wait(long, int)}. - */ - @Ignore - @Test - public final void testWaitLongInt() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link java.lang.Object#wait()}. - */ - @Ignore - @Test - public final void testWait() { - fail("Not yet implemented"); // TODO - } - - /** - * Test method for {@link java.lang.Object#finalize()}. - */ - @Ignore - @Test - public final void testFinalize() { - fail("Not yet implemented"); // TODO - } - -} diff --git a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/RequestUrlTest.java b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/RequestUrlTest.java new file mode 100644 index 0000000..f94ed5e --- /dev/null +++ b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/RequestUrlTest.java @@ -0,0 +1,70 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sdap.mudrod.weblog.structure; + +import java.io.UnsupportedEncodingException; +import java.util.Map; + +import org.apache.sdap.mudrod.weblog.structure.log.RequestUrl; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class RequestUrlTest { + + String strURL = "https://podaac.jpl.nasa.gov/datasetlist?ids=Collections:Measurement:SpatialCoverage:Platform:Sensor&values=SCAT_BYU_L3_OW_SIGMA0_ENHANCED:Sea%20Ice:Bering%20Sea:ERS-2:AMI&view=list"; + + @Test + public void testUrlPage() { +// RequestUrl url = new RequestUrl(); + String result = RequestUrl.urlPage(strURL); + Assert.assertEquals("You did not pass urlPage function ", "https://podaac.jpl.nasa.gov/datasetlist", result); + } + + @Test + public void testuRLRequest() { +// RequestUrl url = new RequestUrl(); + Map result = RequestUrl.uRLRequest(strURL); + Assert.assertEquals("You did not pass uRLRequest function!", "list", result.get("view")); + Assert.assertEquals("You did not pass uRLRequest function!", "scat_byu_l3_ow_sigma0_enhanced:sea%20ice:bering%20sea:ers-2:ami", result.get("values")); + Assert.assertEquals("You did not pass uRLRequest function!", "collections:measurement:spatialcoverage:platform:sensor", result.get("ids")); + } + + @Test + public void testGetSearchInfo() { + RequestUrl url = new RequestUrl(); + String result = null; + try { + result = url.getSearchInfo(strURL); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + + Assert.assertEquals("You did not pass GetSearchInfo function ", "scat byu l3 ow sigma0 enhanced,sea ice,bering sea,ers 2,ami", result); + } + + @Test + public void testGetFilterInfo() { + RequestUrl url = new RequestUrl(); + try { + Map result = url.getFilterInfo(strURL); + Assert.assertEquals("You did not pass GetFilterInfo function!", "scat byu l3 ow sigma0 enhanced", result.get("collections")); + Assert.assertEquals("You did not pass GetFilterInfo function!", "bering sea", result.get("spatialcoverage")); + Assert.assertEquals("You did not pass GetFilterInfo function!", "ami", result.get("sensor")); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + } +} diff --git a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/TestApacheAccessLog.java b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/TestApacheAccessLog.java index a0e51d8..2822c5e 100644 --- a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/TestApacheAccessLog.java +++ b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/TestApacheAccessLog.java @@ -14,6 +14,7 @@ package org.apache.sdap.mudrod.weblog.structure; +import org.apache.sdap.mudrod.weblog.structure.log.ApacheAccessLog; import org.junit.BeforeClass; import org.junit.Test; @@ -27,14 +28,14 @@ import static org.junit.Assert.assertNotEquals; - public class TestApacheAccessLog { - private static Properties testProperties = new Properties(); + private static Properties testProperties = new Properties(); + + @BeforeClass + public static void loadProperties() throws IOException { + URL configURL = ClassLoader.getSystemClassLoader().getResource("config.properties"); - @BeforeClass - public static void loadProperties() throws IOException { - URL configURL = ClassLoader.getSystemClassLoader().getResource("config.properties"); assert configURL != null : "Could not load config.properties"; try (InputStream instream = new FileInputStream(configURL.getFile())) { @@ -43,9 +44,12 @@ public static void loadProperties() throws IOException { } @Test - public void testLogMatch() throws ParseException { - String testLogLine = "198.118.243.84 - - [31/Dec/2017:23:59:20 +0000] \"GET /events?page=12&%25252525252525252525252525252525252525253Bsort=asc&order=field_location&sort=desc HTTP/1.1\" 200 86173 \"http://www.example.com/start.html\" \"Mozilla/4.08 [en] (Win98; I ;Nav)\""; - String result = ApacheAccessLog.parseFromLogLine(testLogLine, testProperties); - assertNotEquals("Log line does not match", "{}", result); - } + public void testLogMatch() throws IOException, ParseException { + + // only have 7 fields? Should be 9? why not equal? +// String testLogLine = "198.118.243.84 - - [31/Dec/2017:23:59:20 +0000] \"GET /events?page=12&%25252525252525252525252525252525252525253Bsort=asc&order=field_location&sort=desc HTTP/1.1\" 200 86173"; + String testLogLine = "140.90.107.54 - - [01/Feb/2015:17:08:09 -0800] \"GET /datasetlist?search=VIIRS HTTP/1.1\" 200 82024 \"http://podaac.jpl.nasa.gov/\" \"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0\""; + String result = ApacheAccessLog.parseFromLogLine(testLogLine, testProperties); + assertNotEquals("Log line does not match", "{}", result); + } } diff --git a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/log/GeoIpTest.java b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/log/GeoIpTest.java new file mode 100644 index 0000000..70773a1 --- /dev/null +++ b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/log/GeoIpTest.java @@ -0,0 +1,29 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sdap.mudrod.weblog.structure.log; + +import org.junit.Assert; +import org.junit.Test; + +public class GeoIpTest { + + @Test + public void testToLocation() { + GeoIp ip = new GeoIp(); + String iptest = "185.10.104.194"; + + Coordinates result = ip.toLocation(iptest); + Assert.assertEquals("failed in geoip function!", "22.283001,114.150002", result.latlon); + } +} diff --git a/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/FTP.201502.w1.gz b/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/FTP.201502.w1.gz new file mode 100644 index 0000000..f80ebe5 Binary files /dev/null and b/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/FTP.201502.w1.gz differ diff --git a/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/WWW.201502.w1.gz b/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/WWW.201502.w1.gz new file mode 100644 index 0000000..86c5ffc Binary files /dev/null and b/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/WWW.201502.w1.gz differ diff --git a/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/clickstreammatrix.csv b/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/clickstreammatrix.csv new file mode 100644 index 0000000..1316fa2 --- /dev/null +++ b/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/clickstreammatrix.csv @@ -0,0 +1,2 @@ + Num,"ostm_l2_ost_ogdr_gps" +sea surface topography,6.0 diff --git a/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/userhistorymatrix.csv b/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/userhistorymatrix.csv new file mode 100644 index 0000000..6a094d8 --- /dev/null +++ b/core/src/test/resources/Testing_Data_1_3dayLog+Meta+Onto/userhistorymatrix.csv @@ -0,0 +1,2 @@ +Num,195.219.98.7 +sea surface topography,1,