enable all tests

Author: phi-dbq

python/sparkdl/graph/input.py

@@ -225,7 +225,6 @@ def _build_impl(self, feed_names, fetch_names):
         gin = TFInputGraph._new_obj_internal()
         assert (feed_names is None) == (fetch_names is None)
         must_have_sig_def = fetch_names is None
-        print('builder-session', repr(self.sess))
         # NOTE(phi-dbq): both have to be set to default
         with self.sess.as_default(), self.graph.as_default():
             _ginfo = self.import_graph_fn(self.sess)
@@ -249,47 +248,3 @@ def build(self, feed_names=None, fetch_names=None):
             if self._should_clean:
                 self.sess.close()
         return gin
-
-# def the_rest(input_mapping, output_mapping):
-#     graph = tf.Graph()
-#     with tf.Session(graph=graph) as sess:
-#         # Append feeds and input mapping
-#         _input_mapping = {}
-#         if isinstance(input_mapping, dict):
-#             input_mapping = input_mapping.items()
-#         for input_colname, tnsr_or_sig in input_mapping:
-#             if sig_def:
-#                 tnsr = sig_def.inputs[tnsr_or_sig].name
-#             else:
-#                 tnsr = tnsr_or_sig
-#             _input_mapping[input_colname] = tfx.op_name(graph, tnsr)
-#         input_mapping = _input_mapping
-
-#         # Append fetches and output mapping
-#         fetches = []
-#         _output_mapping = {}
-#         # By default the output columns will have the name of their
-#         # corresponding `tf.Graph` operation names.
-#         # We have to convert them to the user specified output names
-#         if isinstance(output_mapping, dict):
-#             output_mapping = output_mapping.items()
-#         for tnsr_or_sig, requested_colname in output_mapping:
-#             if sig_def:
-#                 tnsr = sig_def.outputs[tnsr_or_sig].name
-#             else:
-#                 tnsr = tnsr_or_sig
-#             fetches.append(tfx.get_tensor(graph, tnsr))
-#             tf_output_colname = tfx.op_name(graph, tnsr)
-#             # NOTE(phi-dbq): put the check here as it will be the entry point to construct
-#             #                a `TFInputGraph` object.
-#             assert tf_output_colname not in _output_mapping, \
-#                 "operation {} has multiple output tensors and ".format(tf_output_colname) + \
-#                 "at least two of them are used in the output DataFrame. " + \
-#                 "Operation names are used to name columns which leads to conflicts. "  + \
-#                 "You can apply `tf.identity` ops to each to avoid name conflicts."
-#             _output_mapping[tf_output_colname] = requested_colname
-#         output_mapping = _output_mapping
-
-#         gdef = tfx.strip_and_freeze_until(fetches, graph, sess)
-
-#     return TFInputGraph(gdef), input_mapping, output_mapping

python/tests/transformers/tf_tensor_test.py

@@ -27,6 +27,7 @@
 
 from pyspark.sql.types import Row
 
+from sparkdl.graph.builder import IsolatedSession
 from sparkdl.graph.input import *
 import sparkdl.graph.utils as tfx
 from sparkdl.transformers.tf_tensor import TFTransformer
@@ -60,6 +61,15 @@ def setUp(self):
     def tearDown(self):
         shutil.rmtree(self.model_output_root, ignore_errors=True)
 
+    def _build_default_session_tests(self, sess):
+        gin = TFInputGraph.fromGraph(
+            sess.graph, sess, self.feed_names, self.fetch_names)
+        self.build_standard_transformers(sess, gin)
+
+        gin = TFInputGraph.fromGraphDef(
+            sess.graph.as_graph_def(), self.feed_names, self.fetch_names)
+        self.build_standard_transformers(sess, gin)
+
     def build_standard_transformers(self, sess, tf_input_graph):
         def _add_transformer(imap, omap):
             trnsfmr = TFTransformer(
@@ -113,7 +123,7 @@ def _run_test_in_tf_session(self):
 
         # Build the TensorFlow graph
         graph = tf.Graph()
-        with tf.Session(graph=graph) as sess:
+        with tf.Session(graph=graph) as sess, graph.as_default():
             # Build test graph and transformers from here
             yield sess
 
@@ -147,8 +157,7 @@ def _run_test_in_tf_session(self):
                 _results.append(np.ravel(curr_res))
             out_tgt = np.hstack(_results)
 
-            self.assertTrue(np.allclose(out_ref, out_tgt),
-                            msg=repr(transfomer))
+            self.assertTrue(np.allclose(out_ref, out_tgt), msg=repr(transfomer))
 
 
     def test_build_from_tf_graph(self):
@@ -159,13 +168,7 @@ def test_build_from_tf_graph(self):
             _ = tf.reduce_mean(x, axis=1, name=self.output_op_name)
             # End building graph
 
-            # Begin building transformers
-            self.build_standard_transformers(
-                sess, TFInputGraph.fromGraph(sess.graph, sess, self.feed_names, self.fetch_names))
-            gdef = sess.graph.as_graph_def()
-            self.build_standard_transformers(
-                sess, TFInputGraph.fromGraphDef(gdef, self.feed_names, self.fetch_names))
-            # End building transformers
+            self._build_default_session_tests(sess)
 
 
     def test_build_from_saved_model(self):
@@ -224,6 +227,10 @@ def test_build_from_saved_model(self):
                 saved_model_dir, serving_tag, self.feed_names, self.fetch_names)
             self.build_standard_transformers(sess, gin)
 
+            gin = TFInputGraph.fromGraph(
+                sess.graph, sess, self.feed_names, self.fetch_names)
+            self.build_standard_transformers(sess, gin)
+
 
     def test_build_from_checkpoint(self):
         """ Build TFTransformer from a model checkpoint """
@@ -285,6 +292,10 @@ def test_build_from_checkpoint(self):
             gin = TFInputGraph.fromCheckpoint(model_ckpt_dir, self.feed_names, self.fetch_names)
             self.build_standard_transformers(sess, gin)
 
+            gin = TFInputGraph.fromGraph(
+                sess.graph, sess, self.feed_names, self.fetch_names)
+            self.build_standard_transformers(sess, gin)
+
 
     def test_multi_io(self):
         """ Build TFTransformer with multiple I/O tensors """
@@ -300,41 +311,31 @@ def test_multi_io(self):
                 z = tf.reduce_mean(xs[i], axis=1, name=tnsr_op_name)
                 zs.append(z)
 
-            gin = TFInputGraph.fromGraph(
-                sess.graph, sess, self.feed_names, self.fetch_names)
-            self.build_standard_transformers(sess, gin)
-
-            gin = TFInputGraph.fromGraphDef(
-                sess.graph.as_graph_def(), self.feed_names, self.fetch_names)
-            self.build_standard_transformers(sess, gin)
-
+            self._build_default_session_tests(sess)
+
+
+    def test_mixed_keras_graph(self):
+        """ Build mixed keras graph """
+        with IsolatedSession(using_keras=True) as issn:
+            tnsr_in = tf.placeholder(
+                tf.double, shape=[None, self.vec_size], name=self.input_op_name)
+            inp = tf.expand_dims(tnsr_in, axis=2)
+            # Keras layers does not take tf.double
+            inp = tf.cast(inp, tf.float32)
+            conv = Conv1D(filters=4, kernel_size=2)(inp)
+            pool = MaxPool1D(pool_size=2)(conv)
+            flat = Flatten()(pool)
+            dense = Dense(1)(flat)
+            # We must keep the leading dimension of the output
+            redsum = tf.reduce_sum(dense, axis=1)
+            tnsr_out = tf.cast(redsum, tf.double, name=self.output_op_name)
+
+            # Initialize the variables
+            init_op = tf.global_variables_initializer()
+            issn.run(init_op)
+            # We could train the model ... but skip it here
+            gfn = issn.asGraphFunction([tnsr_in], [tnsr_out])
 
-    # def test_mixed_keras_graph(self):
-    #     # Build the graph: the output should have the same leading/batch dimension
-    #     with IsolatedSession(using_keras=True) as issn:
-    #         tnsr_in = tf.placeholder(
-    #             tf.double, shape=[None, self.vec_size], name=self.input_op_name)
-    #         inp = tf.expand_dims(tnsr_in, axis=2)
-    #         # Keras layers does not take tf.double
-    #         inp = tf.cast(inp, tf.float32)
-    #         conv = Conv1D(filters=4, kernel_size=2)(inp)
-    #         pool = MaxPool1D(pool_size=2)(conv)
-    #         flat = Flatten()(pool)
-    #         dense = Dense(1)(flat)
-    #         # We must keep the leading dimension of the output
-    #         redsum = tf.reduce_sum(dense, axis=1)
-    #         tnsr_out = tf.cast(redsum, tf.double, name=self.output_op_name)
-
-    #         # Initialize the variables
-    #         init_op = tf.global_variables_initializer()
-    #         issn.run(init_op)
-    #         # We could train the model ... but skip it here
-    #         gfn = issn.asGraphFunction([tnsr_in], [tnsr_out])
-
-    #     with self.run_test_in_tf_session() as sess:
-    #         tf.import_graph_def(gfn.graph_def, name='')
-
-    #         self.build_standard_transformers(sess, sess.graph)
-    #         self.build_standard_transformers(sess, TFInputGraphBuilder.fromGraph(sess.graph))
-    #         self.build_standard_transformers(sess, gfn.graph_def)
-    #         self.build_standard_transformers(sess, TFInputGraphBuilder.fromGraphDef(gfn.graph_def))
+        with self._run_test_in_tf_session() as sess:
+            tf.import_graph_def(gfn.graph_def, name='')
+            self._build_default_session_tests(sess)