Online Grammar Extraction

cmake/Findnanomsg.cmake

@@ -0,0 +1,69 @@
+# Locate nanomsg.
+# (This file is based on the original FindGTest.cmake file,
+#  feel free to use it as it is or modify it for your own needs.)
+#
+#
+# Defines the following variables:
+#
+#   NANOMSG_FOUND - Found the nanomsg library
+#   NANOMSG_INCLUDE_DIR - Include directory
+#
+# Also defines the library variables below as normal
+# variables.
+#
+#   NANOMSG_LIBRARIES - libnanomsg.so and libnanomsg.a
+#
+#-----------------------
+# Example Usage:
+#
+#    find_package(nanomsg REQUIRED)
+#    include_directories(${NANOMSG_INCLUDE_DIR})
+#
+#    add_executable(foo foo.cc)
+#    target_link_libraries(foo ${NANOMSG_LIBRARIES})
+#
+#=============================================================================
+# This file is released under the MIT licence:
+#
+# Copyright (c) 2011 Matej Svec
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#=============================================================================
+
+find_path(NANOMSG_INCLUDE_DIR NAMES nanomsg/bus.h nanomsg/inproc.h nanomsg/ipc.h nanomsg/nn.h nanomsg/pair.h nanomsg/pipeline.h nanomsg/pubsub.h nanomsg/reqrep.h nanomsg/survey.h nanomsg/tcp.h)
+mark_as_advanced(NANOMSG_INCLUDE_DIR)
+
+find_library(NANOMSG_SHARED libnanomsg.so)
+mark_as_advanced(NANOMSG_SHARED)
+
+find_library(NANOMSG_STATIC libnanomsg.a)
+mark_as_advanced(NANOMSG_STATIC)
+
+if(NANOMSG_INCLUDE_DIR AND NANOMSG_SHARED AND NANOMSG_STATIC)
+  set(NANOMSG_FOUND TRUE)
+else()
+  set(NANOMSG_FOUND FALSE)
+endif()
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(nanomsg DEFAULT_MSG NANOMSG_SHARED NANOMSG_STATIC NANOMSG_INCLUDE_DIR)
+
+if(NANOMSG_FOUND)
+  set(NANOMSG_LIBRARIES ${NANOMSG_STATIC} ${NANOMSG_SHARED})
+endif()

extractor/CMakeLists.txt

@@ -43,12 +43,12 @@ if(GTEST_FOUND)
     target_link_libraries(${testName} extractor ${GMOCK_BOTH_LIBRARIES} ${GTEST_BOTH_LIBRARIES} ${Boost_LIBRARIES} ${ZLIB_LIBRARIES})
 
     #I like to move testing binaries into a testBin directory
-    set_target_properties(${testName} PROPERTIES 
+    set_target_properties(${testName} PROPERTIES
       RUNTIME_OUTPUT_DIRECTORY  ${CMAKE_CURRENT_SOURCE_DIR})
 
-    #Finally add it to test execution - 
+    #Finally add it to test execution -
     #Notice the WORKING_DIRECTORY and COMMAND
-    add_test(NAME ${testName} COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/${testName} 
+    add_test(NAME ${testName} COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/${testName}
        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
   endforeach(testSrc)
  endif(GMOCK_FOUND)
@@ -66,10 +66,30 @@ set(extract_SRCS extract.cc)
 add_executable(extract ${extract_SRCS})
 target_link_libraries(extract extractor utils ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES})
 
+# nanomsg, use for the extractor daemon
+# set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/extractor/cmake/Modules/")
+find_package(nanomsg)
+if(NANOMSG_FOUND)
+  include_directories(${NANOMSG_INCLUDE_DIR})
+
+  set(extract_request_STAT_SRCS extract_request.cc)
+  add_library(extract_request STATIC ${extract_request_STAT_SRCS})
+  target_link_libraries(extract_request ${NANOMSG_STATIC})
+
+  #set(extract_request_test_SRCS extract_request_test.cc)
+  #add_executable(run_extractor_request_test ${extract_request_test_SRCS})
+  #target_link_libraries(run_extractor_request_test extractor ${NANOMSG_LIBRARIES})
+
+  set(extract_daemon_SRCS extract_daemon.cc)
+  add_executable(extract_daemon ${extract_daemon_SRCS})
+  target_link_libraries(extract_daemon extractor utils ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES} ${NANOMSG_LIBRARIES})
+endif(NANOMSG_FOUND)
+
 set(extractor_STAT_SRCS
     alignment.cc
     backoff_sampler.cc
     data_array.cc
+    extract_request.cc
     fast_intersector.cc
     features/count_source_target.cc
     features/feature.cc
@@ -111,6 +131,7 @@ set(extractor_STAT_SRCS
     alignment.h
     backoff_sampler.h
     data_array.h
+    extract_request.h
     fast_intersector.h
     grammar.h
     grammar_extractor.h
@@ -136,4 +157,3 @@ set(extractor_STAT_SRCS
     vocabulary.h)
 
 add_library(extractor STATIC ${extractor_STAT_SRCS})
-

extractor/README.md

@@ -18,3 +18,28 @@ Then, you simply need to:
 
     cd cdec/extractor
     make check
+
+To run the extractor as a daemon for online grammar extraction you need to compile with [nanomsg](http://nanomsg.org/download.html) in your system's PATH and then:
+
+    cdec/extractor/extract_daemon -c <compile_config_file> -g <grammar_output_path> -a <address>
+
+Online here means that sentences are received in an online fashion and rather than loading the files from <compile_config_file> anew when a new sentence is received, they are stored in RAM and a new grammar can be requested via the daemon. The daemon can be killed using the SID supplied in the log file.
+
+To then query the daemon you need to implement the Requester class supplied in
+extractor/libextract_request.a. It's constructor takes the same address string as supplied for the daemon. E.g.:
+
+    #include <iostream>
+
+    #include "extract_request.h"
+
+    using namespace std;
+
+    int main(int argc, char** argv) {
+      extractor::Requester requester("ipc:///tmp/extract_daemon.ipc");
+      cout << requester.request_for_sentence("<input sentence>") << endl;
+      return 0;
+    }
+
+which can be compiled with:
+
+    g++ extract_request_test.cc -o run_test -lextract_request -L/workspace/osm/cdec/extractor/ -I/workspace/osm/cdec/extractor/ -lnanomsg