diff --git a/scripts/installer/windows/scripts/promptflow.spec.jinja2 b/scripts/installer/windows/scripts/promptflow.spec.jinja2
index a7d1cd636645..74d77feff6dc 100644
--- a/scripts/installer/windows/scripts/promptflow.spec.jinja2
+++ b/scripts/installer/windows/scripts/promptflow.spec.jinja2
@@ -17,17 +17,22 @@ for package in meta_packages:
     datas += copy_metadata(package)
 
 opentelemetry_datas, opentelemetry_binaries, opentelemetry_hiddenimports = collect_all('opentelemetry')
+promptflow_evals_datas, promptflow_evals_binaries, promptflow_evals_hiddenimports = collect_all('promptflow-evals')
+
 datas += opentelemetry_datas
+datas += promptflow_evals_datas
 datas += collect_data_files('streamlit_quill')
 datas += collect_data_files('promptflow')
 datas += copy_metadata('promptflow')
-datas += collect_data_files('promptflow-evals')
-datas += copy_metadata('promptflow-evals')
+
 hidden_imports = ['win32timezone', 'promptflow', 'opentelemetry.context.contextvars_context', 'streamlit.runtime.scriptrunner.magic_funcs'] + {{hidden_imports}}
 
 hidden_imports += opentelemetry_hiddenimports
+hidden_imports += promptflow_evals_hiddenimports
+
 binaries = []
 binaries += opentelemetry_binaries
+binaries += promptflow_evals_binaries
 
 block_cipher = None