issue during training the data

[karthika-devi]

I got the below error after 1220 steps of training get finished during running the udc_train.py file

Traceback (most recent call last):
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/client/session.py", line 1021, in _do_call
return fn(*args)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/client/session.py", line 1003, in _run_fn
status, run_metadata)
File "/usr/lib/python3.4/contextlib.py", line 66, in exit
next(self.gen)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/framework/errors_impl.py", line 469, in raise_exception_on_not_ok_status
pywrap_tensorflow.TF_GetCode(status))
tensorflow.python.framework.errors_impl.InvalidArgumentError: Incompatible shapes: [80,1] vs. [160,1]
[[Node: prediction/logistic_loss/mul = Mul[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"](prediction/Squeeze, prediction/ToFloat)]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "udc_train.py", line 64, in
tf.app.run()
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/platform/app.py", line 43, in run
sys.exit(main(sys.argv[:1] + flags_passthrough))
File "udc_train.py", line 61, in main
estimator.fit(input_fn=input_fn_train, steps=None, monitors=[eval_monitor])
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/util/deprecation.py", line 191, in new_func
return func(*args, **kwargs)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 355, in fit
max_steps=max_steps)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 733, in _train_model
max_steps=max_steps)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/graph_actions.py", line 301, in _monitored_train
None)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/training/monitored_session.py", line 473, in run
run_metadata=run_metadata)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/training/monitored_session.py", line 628, in run
run_metadata=run_metadata)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/training/monitored_session.py", line 595, in run
return self._sess.run(*args, **kwargs)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/training/monitored_session.py", line 737, in run
run_metadata=run_metadata))
File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/monitors.py", line 1210, in after_run
induce_stop = m.step_end(self._last_step, result)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/monitors.py", line 409, in step_end
return self.every_n_step_end(step, output)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/monitors.py", line 710, in every_n_step_end
steps=self.eval_steps, metrics=self.metrics, name=self.name)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/util/deprecation.py", line 191, in new_func
return func(*args, **kwargs)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 427, in evaluate
name=name)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 809, in _evaluate_model
max_steps=steps)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/graph_actions.py", line 754, in evaluate
session.run(update_op, feed_dict=feed_dict)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/client/session.py", line 766, in run
run_metadata_ptr)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/client/session.py", line 964, in _run
feed_dict_string, options, run_metadata)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/client/session.py", line 1014, in _do_run
target_list, options, run_metadata)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/client/session.py", line 1034, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Incompatible shapes: [80,1] vs. [160,1]
[[Node: prediction/logistic_loss/mul = Mul[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"](prediction/Squeeze, prediction/ToFloat)]]

Caused by op 'prediction/logistic_loss/mul', defined at:
File "udc_train.py", line 64, in
tf.app.run()
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/platform/app.py", line 43, in run
sys.exit(main(sys.argv[:1] + flags_passthrough))
File "udc_train.py", line 61, in main
estimator.fit(input_fn=input_fn_train, steps=None, monitors=[eval_monitor])
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/util/deprecation.py", line 191, in new_func
return func(*args, **kwargs)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 355, in fit
max_steps=max_steps)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 733, in _train_model
max_steps=max_steps)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/graph_actions.py", line 301, in _monitored_train
None)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/training/monitored_session.py", line 473, in run
run_metadata=run_metadata)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/training/monitored_session.py", line 628, in run
run_metadata=run_metadata)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/training/monitored_session.py", line 595, in run
return self._sess.run(*args, **kwargs)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/training/monitored_session.py", line 737, in run
run_metadata=run_metadata))
File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/monitors.py", line 1210, in after_run
induce_stop = m.step_end(self._last_step, result)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/monitors.py", line 409, in step_end
return self.every_n_step_end(step, output)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/monitors.py", line 710, in every_n_step_end
steps=self.eval_steps, metrics=self.metrics, name=self.name)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/util/deprecation.py", line 191, in new_func
return func(*args, **kwargs)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 427, in evaluate
name=name)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 793, in _evaluate_model
eval_ops = self._get_eval_ops(features, labels, metrics)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 1080, in _get_eval_ops
features, labels, model_fn_lib.ModeKeys.EVAL)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 1021, in _call_model_fn
model_fn_results = self._model_fn(features, labels, mode=mode)
File "/usr/src/chatbot-retrieval/udc_model.py", line 83, in model_fn
tf.concat(0, all_targets))
File "/usr/src/chatbot-retrieval/models/dual_encoder.py", line 81, in dual_encoder_model
losses = tf.nn.sigmoid_cross_entropy_with_logits(logits, tf.to_float(targets))
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/ops/nn.py", line 456, in sigmoid_cross_entropy_with_logits
return math_ops.add(relu_logits - logits * targets,
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/ops/math_ops.py", line 814, in binary_op_wrapper
return func(x, y, name=name)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/ops/math_ops.py", line 987, in _mul_dispatch
return gen_math_ops.mul(x, y, name=name)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/ops/gen_math_ops.py", line 1613, in mul
result = _op_def_lib.apply_op("Mul", x=x, y=y, name=name)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/framework/op_def_library.py", line 759, in apply_op
op_def=op_def)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/framework/ops.py", line 2240, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/framework/ops.py", line 1128, in init
self._traceback = _extract_stack()

InvalidArgumentError (see above for traceback): Incompatible shapes: [80,1] vs. [160,1]
[[Node: prediction/logistic_loss/mul = Mul[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"](prediction/Squeeze, prediction/ToFloat)]]

[rrivera1849]

Getting the same error, did you ever figure out what it was?

[vaibhavgeek]

I have the same error.

[vaibhavgeek]

Please check #15 issue.

[pengcao]

I have the same error. How to reslove? Thanks!