fast_align: added option to dump expected counts to file after EM traini...

.gitignore

@@ -0,0 +1,221 @@
+example_extff/ff_example.lo
+example_extff/libff_example.la
+mteval/meteor_jar.cc
+training/utils/grammar_convert
+*.a
+*.trs
+*.aux
+*.bbl
+*.blg
+*.dvi
+*.idx
+*.log
+*.o
+*.pdf
+*.ps
+*.pyc
+*.so
+*.toc
+*swp
+*~
+.*
+./cdec/
+Makefile
+Makefile.in
+aclocal.m4
+autom4te.cache/
+config.guess
+config.h
+config.h.in
+config.h.in~
+config.log
+config.status
+config.sub
+configure
+decoder/Makefile
+decoder/Makefile.in
+decoder/bin/
+decoder/cdec
+decoder/dict_test
+decoder/sv_test
+decoder/ff_test
+decoder/grammar_test
+decoder/hg_test
+decoder/logval_test
+decoder/parser_test
+decoder/rule_lexer.cc
+decoder/small_vector_test
+decoder/trule_test
+decoder/weights_test
+depcomp
+dist
+dpmert/Makefile
+dpmert/Makefile.in
+dpmert/fast_score
+dpmert/lo_test
+dpmert/mr_dpmert_generate_mapper_input
+dpmert/mr_dpmert_map
+dpmert/mr_dpmert_reduce
+dpmert/scorer_test
+dpmert/sentclient
+dpmert/sentserver
+dpmert/union_forests
+dtrain/dtrain
+extools/build_lexical_translation
+extools/extractor
+extools/extractor_monolingual
+extools/featurize_grammar
+extools/filter_grammar
+extools/filter_score_grammar
+extools/mr_stripe_rule_reduce
+extools/score_grammar
+extools/sg_lexer.cc
+extractor/*_test
+extractor/compile
+extractor/extract
+extractor/run_extractor
+gi/clda/src/clda
+gi/markov_al/ml
+gi/pf/align-lexonly
+gi/pf/align-lexonly-pyp
+gi/pf/align-tl
+gi/pf/bayes_lattice_score
+gi/pf/brat
+gi/pf/cbgi
+gi/pf/condnaive
+gi/pf/dpnaive
+gi/pf/itg
+gi/pf/learn_cfg
+gi/pf/nuisance_test
+gi/pf/pf_test
+gi/pf/pfbrat
+gi/pf/pfdist
+gi/pf/pfnaive
+gi/pf/pyp_lm
+gi/posterior-regularisation/prjava/build/
+gi/posterior-regularisation/prjava/lib/*.jar
+gi/posterior-regularisation/prjava/lib/prjava-20100713.jar
+gi/posterior-regularisation/prjava/lib/prjava-20100715.jar
+gi/posterior-regularisation/prjava/prjava.jar
+gi/pyp-topics/src/contexts_lexer.cc
+gi/pyp-topics/src/pyp-contexts-train
+gi/pyp-topics/src/pyp-topics-train
+install-sh
+jam-files/bjam
+jam-files/engine/bin.*
+jam-files/engine/bootstrap/
+klm/lm/bin/
+klm/lm/builder/builder
+klm/lm/builder/lmplz
+klm/lm/build_binary
+klm/lm/ngram_query
+klm/lm/query
+klm/util/bin/
+libtool
+ltmain.sh
+m4/libtool.m4
+m4/ltoptions.m4
+m4/ltsugar.m4
+m4/ltversion.m4
+m4/lt~obsolete.m4
+minrisk/minrisk_optimize
+mira/kbest_mira
+missing
+mteval/bin/
+mteval/fast_score
+mteval/mbr_kbest
+mteval/scorer_test
+phrasinator/gibbs_train_plm
+phrasinator/gibbs_train_plm_notables
+previous.sh
+pro-train/mr_pro_map
+pro-train/mr_pro_reduce
+python/build
+python/setup.py
+rampion/rampion_cccp
+rst_parser/mst_train
+rst_parser/random_tree
+rst_parser/rst_parse
+rst_parser/rst_train
+sa-extract/calignment.c
+sa-extract/cdat.c
+sa-extract/cfloatlist.c
+sa-extract/cintlist.c
+sa-extract/clex.c
+sa-extract/cstrmap.c
+sa-extract/csuf.c
+sa-extract/cveb.c
+sa-extract/lcp.c
+sa-extract/precomputation.c
+sa-extract/rule.c
+sa-extract/rulefactory.c
+sa-extract/sym.c
+stamp-h1
+tests/system_tests/hmm/foo.src
+training/Makefile
+training/Makefile.in
+training/atools
+training/augment_grammar
+training/cllh_filter_grammar
+training/collapse_weights
+training/grammar_convert
+training/lbfgs_test
+training/lbl_model
+training/liblbfgs/bin/
+training/liblbfgs/ll_test
+training/model1
+training/mpi_batch_optimize
+training/mpi_adagrad_optimize
+training/mpi_compute_cllh
+training/mpi_em_optimize
+training/mpi_extract_features
+training/mpi_extract_reachable
+training/mpi_flex_optimize
+training/mpi_online_optimize
+training/mr_em_adapted_reduce
+training/mr_em_map_adapter
+training/mr_optimize_reduce
+training/mr_reduce_to_weights
+training/optimize_test
+training/plftools
+training/test_ngram
+utils/atools
+utils/bin/
+utils/crp_test
+utils/dict_test
+utils/logval_test
+utils/m_test
+utils/mfcr_test
+utils/phmt
+utils/reconstruct_weights
+utils/small_vector_test
+utils/sv_test
+utils/ts
+utils/weights_test
+training/crf/mpi_adagrad_optimize
+training/crf/mpi_batch_optimize
+training/crf/mpi_baum_welch
+training/crf/mpi_compute_cllh
+training/crf/mpi_extract_features
+training/crf/mpi_extract_reachable
+training/crf/mpi_flex_optimize
+training/crf/mpi_online_optimize
+training/dpmert/lo_test
+training/dpmert/mr_dpmert_generate_mapper_input
+training/dpmert/mr_dpmert_map
+training/dpmert/mr_dpmert_reduce
+training/dpmert/sentclient
+training/dpmert/sentserver
+training/dtrain/dtrain
+training/latent_svm/latent_svm
+training/minrisk/minrisk_optimize
+training/mira/kbest_mira
+training/mira/kbest_cut_mira
+training/pro/mr_pro_map
+training/pro/mr_pro_reduce
+training/rampion/rampion_cccp
+training/utils/lbfgs_test
+training/utils/optimize_test
+training/utils/sentclient
+training/utils/sentserver
+word-aligner/fast_align

.travis.yml

@@ -0,0 +1,23 @@
+language: python
+python:
+ - "2.7"
+before_script:
+ - sudo apt-get install libboost-filesystem1.48-dev
+ - sudo apt-get install libboost-program-options1.48-dev
+ - sudo apt-get install libboost-serialization1.48-dev
+ - sudo apt-get install libboost-regex1.48-dev
+ - sudo apt-get install libboost-test1.48-dev
+ - sudo apt-get install libboost-system1.48-dev
+ - sudo apt-get install libboost-thread1.48-dev
+ - sudo apt-get install flex
+ - autoreconf -ifv
+ - ./configure
+script:
+ - make
+ - cd python
+ - python setup.py install
+ - cd ..
+after_script:
+ - make check
+ - ./tests/run-system-tests.pl
+ - nosetests python/tests

BUILDING

@@ -0,0 +1,40 @@
+To build cdec, you'll need:
+
+ * boost headers & boost program_options (you may need to install a package
+    like boost-devel)
+
+
+Instructions for building
+-----------------------------------
+
+ 1) Use automake / autoconf to generate the configure script.
+    I'm not an expert at using these tools, but this should be sufficient:
+
+        autoreconf -ifv
+
+ 2) Configure and build.  Your command will look something like this.
+
+        ./configure
+        make
+
+    If you get errors during configure about missing BOOST macros, then step 3
+    failed, and you need to keep working at it.  If you get errors during the
+    build, it's probably a problem with step 3 or possibly with some compiler
+    version idiosyncracies (generally, I assume you have a relatively new
+    of g++).
+
+    If you're building on cygwin, their libtool is buggy; this make command
+    works for now:
+
+        make LIBS+="-lz -lboost_program_options" \
+             CFLAGS+="-Wno-sign-compare"
+
+ 3) Test
+
+        ./tests/run-system-tests.pl
+
+    Everything should pass.
+
+
+ 4) Enjoy!
+

LICENSE.cctbx.txt

@@ -0,0 +1,45 @@
+*** License agreement ***
+
+cctbx Copyright (c) 2006, The Regents of the University of
+California, through Lawrence Berkeley National Laboratory (subject to
+receipt of any required approvals from the U.S. Dept. of Energy).  All
+rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+(1) Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+(2) Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+(3) Neither the name of the University of California, Lawrence Berkeley
+National Laboratory, U.S. Dept. of Energy nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+You are under no obligation whatsoever to provide any bug fixes,
+patches, or upgrades to the features, functionality or performance of
+the source code ("Enhancements") to anyone; however, if you choose to
+make your Enhancements available either publicly, or directly to
+Lawrence Berkeley National Laboratory, without imposing a separate
+written license agreement for such Enhancements, then you hereby grant
+the following license: a  non-exclusive, royalty-free perpetual license
+to install, use, modify, prepare derivative works, incorporate into
+other computer software, distribute, and sublicense such enhancements or
+derivative works thereof, in binary and source code form.
+