Skip to content

Commit 5c1928a

Browse files
ben--ben--
committed
fix(oss): Build infra for thriftpy3 compilation
1 parent db0c2d4 commit 5c1928a

File tree

4 files changed

+208
-46
lines changed

4 files changed

+208
-46
lines changed

‎CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,11 +67,13 @@ if(NOT CMAKE_CXX_STANDARD)
6767
set(CMAKE_CXX_STANDARD_REQUIRED ON)
6868
message(STATUS "setting C++ standard to C++${CMAKE_CXX_STANDARD}")
6969
endif()
70+
set(FBTHRIFT_CXX_OPTIONS "-std=c++${CMAKE_CXX_STANDARD}")
7071
set(CMAKE_CXX_EXTENSIONS OFF)
7172

7273
# Explicitly enable coroutine support, since GCC does not enable it
7374
# by default when targeting C++17.
7475
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
76+
set(FBTHRIFT_CXX_OPTIONS "${FBTHRIFT_CXX_OPTIONS} -fcoroutines")
7577
add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-fcoroutines>)
7678
endif()
7779

‎build/fbcode_builder/CMake/FBPythonWrapper.cmake‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
3+
function(_write_manifest MANIFEST_FILE PY_TARGET EGG_NAME)
4+
set(egg_glob "${CMAKE_INSTALL_PREFIX}/lib/python*/site-packages/${EGG_NAME}-*.egg")
5+
6+
add_dependencies("${PY_TARGET}" "${MANIFEST_FILE}")
7+
set("${PY_TARGET}" INTERFACE_INCLUDE_DIRECTORIES ${MANIFEST_FILE})
8+
set("${PY_TARGET}" INTERFACE_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/lib")
9+
10+
set(_install_code [[
11+
file(GLOB _egg_root @egg_glob@)
12+
file(GLOB_RECURSE _egg_files
13+
LIST_DIRECTORIES false
14+
"${_egg_root}/*"
15+
)
16+
17+
set(_manifest_path "@MANIFEST_FILE@")
18+
19+
set(_manifest_contents "FBPY_MANIFEST 1\n")
20+
21+
foreach(_absfile IN LISTS _egg_files)
22+
cmake_path(RELATIVE_PATH _absfile BASE_DIRECTORY ${_egg_root} OUTPUT_VARIABLE _egg_relative)
23+
24+
cmake_path(RELATIVE_PATH _absfile BASE_DIRECTORY ${CMAKE_INSTALL_PREFIX} OUTPUT_VARIABLE _manifest_relative)
25+
set("${PY_TARGET}" INTERFACE_SOURCES $_absfile)
26+
if(NOT ${_egg_relative} MATCHES "^EGG-INFO/")
27+
string(APPEND _manifest_contents "${_manifest_relative} :: ${_egg_relative}\n")
28+
endif()
29+
endforeach()
30+
31+
file(WRITE ${_manifest_path} ${_manifest_contents})
32+
]])
33+
string(CONFIGURE "${_install_code}" _install_code @ONLY)
34+
install(CODE "${_install_code}" DESTINATION ${MANIFEST_FILE} EXPORT)
35+
endfunction()
36+
37+
function(wrap_non_fb_python_library TARGET EGG_NAME)
38+
set(py_lib "${TARGET}.py_lib")
39+
add_library("${py_lib}" INTERFACE)
40+
install(TARGETS "${py_lib}" EXPORT)
41+
42+
set(manifest_filename "${TARGET}.manifest")
43+
set(build_manifest "${CMAKE_CURRENT_BINARY_DIR}/${manifest_filename}")
44+
set(install_manifest "${CMAKE_INSTALL_PREFIX}/${manifest_filename}")
45+
target_include_directories(
46+
"${py_lib}" INTERFACE
47+
"$<BUILD_INTERFACE:${build_manifest}>"
48+
"$<INSTALL_INTERFACE:${install_manifest}>"
49+
)
50+
51+
_write_manifest("${install_manifest}" "${py_lib}" "${EGG_NAME}")
52+
53+
set(build_install_dir "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.lib_install")
54+
set(build_install_manifest "${build_install_dir}/${manifest_filename}")
55+
add_custom_command(
56+
OUTPUT
57+
"${build_install_manifest}"
58+
COMMAND false
59+
DEPENDS
60+
"${install_manifest}"
61+
)
62+
add_custom_target(
63+
"${TARGET}.py_lib_install"
64+
DEPENDS "${build_install_manifest}"
65+
)
66+
67+
# FIXME: do we need the logic from FBPythonBinary:543-553?
68+
endfunction()

‎thrift/lib/CMakeLists.txt‎

Lines changed: 108 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,17 @@ if(thriftpy)
2626
endif()
2727

2828
if(thriftpy3)
29+
include(FBPythonWrapper)
30+
find_package(Python3 COMPONENTS Interpreter Development REQUIRED)
31+
2932
set(_cybld "${CMAKE_CURRENT_BINARY_DIR}/cybld")
3033
file(MAKE_DIRECTORY "${_cybld}/thrift/python/")
3134
file(MAKE_DIRECTORY "${_cybld}/thrift/python/client/")
3235
file(MAKE_DIRECTORY "${_cybld}/thrift/python/server/")
3336
file(MAKE_DIRECTORY "${_cybld}/thrift/python/server/flagged/")
37+
file(MAKE_DIRECTORY "${_cybld}/thrift/python/server_impl/")
38+
file(MAKE_DIRECTORY "${_cybld}/thrift/python/server_impl/interceptor/")
39+
file(MAKE_DIRECTORY "${_cybld}/thrift/python/streaming/")
3440
file(MAKE_DIRECTORY "${_cybld}/thrift/py3/")
3541

3642
# So that cython includes work correctly
@@ -110,6 +116,8 @@ if(thriftpy3)
110116
${CMAKE_COMMAND} -E create_symlink "${_cybld}/thrift/python/server/server.pyx" "${_cybld}/thrift/python/server.pyx"
111117
)
112118

119+
########################################################################
120+
# Symlink files that get mapped directly
113121
add_custom_target(create_binding_symlink_python ALL)
114122
file(GLOB BindingFiles
115123
"python/client/async_client_factory.pxd"
@@ -133,6 +141,10 @@ if(thriftpy3)
133141
"python/server/server.pxd"
134142
"python/std_libcpp.pxd"
135143
"python/stream.pxd"
144+
"python/streaming/py_promise.pxd"
145+
"python/streaming/python_user_exception.pxd"
146+
"python/streaming/sink.pxd"
147+
"python/streaming/stream.pxd"
136148
"python/types.pxd"
137149
"python/util.pxd"
138150

@@ -156,8 +168,13 @@ if(thriftpy3)
156168
"python/mutable_types.pyx"
157169
"python/protocol.pyx"
158170
"python/serializer.pyx"
171+
"python/server/request_context.pyx"
159172
"python/server/server.pyx"
160173
"python/stream.pyx"
174+
"python/streaming/py_promise.pyx"
175+
"python/streaming/python_user_exception.pyx"
176+
"python/streaming/sink.pyx"
177+
"python/streaming/stream.pyx"
161178
"python/util.pyx"
162179

163180
"python/abstract_types.py"
@@ -169,14 +186,15 @@ if(thriftpy3)
169186
"python/Serializer.h"
170187
"python/types.h"
171188

172-
"python/server/flagged/EnableResourcePoolsForPython.cpp"
173189
"python/server/PythonAsyncProcessor.cpp"
174190
"python/server/PythonAsyncProcessorFactory.cpp"
191+
"python/streaming/PythonUserException.cpp"
192+
"python/streaming/Sink.cpp"
193+
"python/streaming/StreamElementEncoder.cpp"
175194
)
176195

177196
foreach(_src ${BindingFiles})
178197
file(RELATIVE_PATH _filerelpath ${CMAKE_CURRENT_SOURCE_DIR} ${_src})
179-
get_filename_component(_target_file "${_src}" NAME)
180198

181199
message(
182200
STATUS
@@ -192,6 +210,40 @@ if(thriftpy3)
192210
)
193211
endforeach()
194212

213+
214+
########################################################################
215+
# Symlink files that get mapped server -> server_impl
216+
file(GLOB BindingFiles
217+
"python/server/async_processor.pxd"
218+
"python/server/event_handler.pxd"
219+
"python/server/interceptor/server_module.pxd"
220+
"python/server/python_async_processor.pxd"
221+
"python/server/request_context.pxd"
222+
223+
"python/server/python_async_processor.pyx"
224+
225+
"python/server/PythonAsyncProcessor.cpp"
226+
"python/server/PythonAsyncProcessorFactory.cpp"
227+
)
228+
foreach(_src ${BindingFiles})
229+
file(RELATIVE_PATH _filerelpath ${CMAKE_CURRENT_SOURCE_DIR}/python/server ${_src})
230+
231+
message(
232+
STATUS
233+
"Linking ${_src} to ${_cybld}/thrift/python/server_impl/${_filerelpath}"
234+
)
235+
236+
add_custom_command(
237+
TARGET
238+
create_binding_symlink_python
239+
PRE_BUILD
240+
COMMAND
241+
${CMAKE_COMMAND} -E create_symlink "${_src}" "${_cybld}/thrift/python/server_impl/${_filerelpath}"
242+
)
243+
endforeach()
244+
245+
########################################################################
246+
# Symlink all the py3 files
195247
add_custom_target(create_binding_symlink_py3 ALL)
196248
file(GLOB BindingFiles
197249
"${CMAKE_CURRENT_SOURCE_DIR}/py3/*.pxd"
@@ -227,16 +279,28 @@ if(thriftpy3)
227279
thriftcpp2
228280
WORKING_DIRECTORY ${_cybld}
229281
)
282+
283+
set(_folly_site_packages "${CMAKE_INSTALL_PREFIX}/../folly/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/site-packages")
284+
# FIXME: For some reason, FOLLY_VERSION is not defined
285+
set(_folly_version "0.0.1")
286+
set(_folly_PYTHONPATH "${_folly_site_packages}/folly-${_folly_version}-py${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}-linux-x86_64.egg")
287+
230288
add_custom_command(TARGET thrift_python_types_bindings
231289
COMMAND ${CMAKE_COMMAND} -E env
232-
"CFLAGS=${CMAKE_C_FLAGS}"
233-
"CXXFLAGS=${CMAKE_CXX_FLAGS}"
290+
"CFLAGS=${CMAKE_C_FLAGS} ${FBTHRIFT_CXX_OPTIONS}"
291+
"CXXFLAGS=${CMAKE_CXX_FLAGS} ${FBTHRIFT_CXX_OPTIONS}"
234292
"CXX=${CMAKE_CXX_COMPILER}"
235-
python3 ${CMAKE_CURRENT_SOURCE_DIR}/setup.py -v --api-only
293+
"PYTHONPATH=${_cybld}/lib:${_cybld}:${_folly_PYTHONPATH}:$ENV{PYTHONPATH}"
294+
${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/setup.py -v --api-only
236295
BYPRODUCTS
237-
${_cybld}/thrift/python/types_api.h
296+
${_cybld}/thrift/python/_types_api.h
238297
${_cybld}/thrift/python/server/server_api.h
239-
${_cybld}/thrift/python/util_api.h
298+
${_cybld}/thrift/python/server_impl/python_async_processor_api.h
299+
${_cybld}/thrift/python/streaming/sink_api.h
300+
${_cybld}/thrift/python/_stream_api.h
301+
DEPENDS
302+
${CMAKE_CURRENT_SOURCE_DIR}/setup.py
303+
# FIXME: add .pxd/.pyx files here
240304
WORKING_DIRECTORY ${_cybld}
241305
)
242306

@@ -247,6 +311,7 @@ if(thriftpy3)
247311
WORKING_DIRECTORY ${_cybld})
248312
file(MAKE_DIRECTORY "${_cybld}/thrift/lib/python/")
249313
file(MAKE_DIRECTORY "${_cybld}/thrift/lib/python/server")
314+
file(MAKE_DIRECTORY "${_cybld}/thrift/lib/python/streaming")
250315
file(MAKE_DIRECTORY "${_cybld}/thrift/lib/py3/")
251316

252317
add_custom_command(
@@ -271,13 +336,13 @@ if(thriftpy3)
271336
TARGET
272337
create_cython_types_api_symlink
273338
COMMAND
274-
${CMAKE_COMMAND} -E create_symlink "${_cybld}/thrift/python/_util_api.h" "${_cybld}/thrift/lib/python/util_api.h"
339+
${CMAKE_COMMAND} -E create_symlink "${_cybld}/thrift/python/server_impl/python_async_processor_api.h" "${_cybld}/thrift/lib/python/server/python_async_processor_api.h"
275340
)
276341
add_custom_command(
277342
TARGET
278343
create_cython_types_api_symlink
279344
COMMAND
280-
${CMAKE_COMMAND} -E create_symlink "${_cybld}/thrift/python/_util.h" "${_cybld}/thrift/lib/python/_util.h"
345+
${CMAKE_COMMAND} -E create_symlink "${_cybld}/thrift/python/streaming/sink_api.h" "${_cybld}/thrift/lib/python/streaming/sink_api.h"
281346
)
282347
add_custom_command(
283348
TARGET
@@ -296,8 +361,9 @@ if(thriftpy3)
296361
python/client/OmniClient.cpp
297362
python/client/RequestChannel.cpp
298363
python/client/ssl.cpp
364+
python/streaming/PythonUserException.cpp
365+
python/streaming/StreamElementEncoder.cpp
299366
python/types.cpp
300-
python/util.cpp
301367
python/Serializer.cpp
302368
py3/stream.cpp
303369
)
@@ -306,22 +372,34 @@ if(thriftpy3)
306372
target_compile_definitions(thrift_python_cpp PRIVATE BOOST_NO_AUTO_PTR)
307373
target_compile_definitions(thrift_python_cpp PRIVATE THRIFT_NO_HTTP_CLIENT_CHANNEL)
308374
target_include_directories(thrift_python_cpp PRIVATE "${_cybld}")
375+
target_include_directories(thrift_python_cpp PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}")
376+
target_include_directories(thrift_python_cpp PRIVATE Folly::folly_python_cpp)
309377
target_link_libraries(
310378
thrift_python_cpp
311379
PUBLIC
312380
thriftcpp2
313381
Folly::folly
314382
Folly::folly_python_cpp
383+
PRIVATE
384+
Python3::Python
315385
)
316386
install(
317387
TARGETS thrift_python_cpp
318-
EXPORT thrift
388+
EXPORT fbthrift-exports
389+
# FIXME: Consider DESTINATION ${LIB_INSTALL_DIR}
319390
)
320391
install(
321392
TARGETS thrift_python_cpp
322393
DESTINATION ${py_install_dir}
323394
)
324395

396+
wrap_non_fb_python_library(thrift_python_cpp thrift)
397+
install(
398+
TARGETS thrift_python_cpp.py_lib
399+
EXPORT fbthrift-exports
400+
DESTINATION ${LIB_INSTALL_DIR}
401+
)
402+
325403
#####
326404
# Now build everything else in lib/python, including all remaining Cython
327405
# modules that depend on types_api.h and thrift_python_cpp.
@@ -332,8 +410,10 @@ if(thriftpy3)
332410
generate_apache_thrift_metadata_python
333411
create_binding_symlink_python
334412
create_binding_symlink_py3
413+
Folly::folly_python_cpp
335414
thriftcpp2
336415
thrift_python_cpp
416+
thrift_python_types_bindings
337417
WORKING_DIRECTORY ${_cybld}
338418
)
339419

@@ -357,29 +437,38 @@ if(thriftpy3)
357437

358438
add_custom_command(TARGET thrift_python_and_py3_bindings
359439
COMMAND ${CMAKE_COMMAND} -E env
360-
"CFLAGS=${CMAKE_C_FLAGS}"
361-
"CXXFLAGS=${CMAKE_CXX_FLAGS}"
440+
"CFLAGS=${CMAKE_C_FLAGS} ${FBTHRIFT_CXX_OPTIONS}"
441+
"CXXFLAGS=${CMAKE_CXX_FLAGS} ${FBTHRIFT_CXX_OPTIONS}"
362442
"CXX=${CMAKE_CXX_COMPILER}"
443+
"PYTHONPATH=${_cybld}/lib:${_cybld}:${_folly_PYTHONPATH}:$ENV{PYTHONPATH}"
363444
python3 ${CMAKE_CURRENT_SOURCE_DIR}/setup.py -v
364445
build_ext -f ${incs} ${libs} --libpython ${python_libname}
446+
DEPENDS
447+
${CMAKE_CURRENT_SOURCE_DIR}/setup.py
365448
WORKING_DIRECTORY ${_cybld}
366449
)
367450
add_custom_command(TARGET thrift_python_and_py3_bindings
368451
COMMAND ${CMAKE_COMMAND} -E env
369-
"CFLAGS=${CMAKE_C_FLAGS}"
370-
"CXXFLAGS=${CMAKE_CXX_FLAGS}"
452+
"CFLAGS=${CMAKE_C_FLAGS} ${FBTHRIFT_CXX_OPTIONS}"
453+
"CXXFLAGS=${CMAKE_CXX_FLAGS} ${FBTHRIFT_CXX_OPTIONS}"
371454
"CXX=${CMAKE_CXX_COMPILER}"
455+
"PYTHONPATH=${_cybld}/lib:${_cybld}:${_folly_PYTHONPATH}:$ENV{PYTHONPATH}"
372456
python3 ${CMAKE_CURRENT_SOURCE_DIR}/setup.py -v
373457
bdist_wheel --libpython ${python_libname}
458+
DEPENDS
459+
${CMAKE_CURRENT_SOURCE_DIR}/setup.py
374460
WORKING_DIRECTORY ${_cybld}
375461
)
376462
add_custom_command(TARGET thrift_python_and_py3_bindings POST_BUILD
377463
COMMAND ${CMAKE_COMMAND} -E env
378-
"CFLAGS=${CMAKE_C_FLAGS}"
379-
"CXXFLAGS=${CMAKE_CXX_FLAGS}"
464+
"CFLAGS=${CMAKE_C_FLAGS} ${FBTHRIFT_CXX_OPTIONS}"
465+
"CXXFLAGS=${CMAKE_CXX_FLAGS} ${FBTHRIFT_CXX_OPTIONS}"
380466
"CXX=${CMAKE_CXX_COMPILER}"
467+
"PYTHONPATH=${_cybld}/lib:${_cybld}:${_folly_PYTHONPATH}:$ENV{PYTHONPATH}"
381468
python3 ${CMAKE_CURRENT_SOURCE_DIR}/setup.py -v
382469
install --prefix ${CMAKE_INSTALL_PREFIX} --libpython ${python_libname}
470+
DEPENDS
471+
${CMAKE_CURRENT_SOURCE_DIR}/setup.py
383472
WORKING_DIRECTORY ${_cybld}
384473
)
385474

@@ -390,8 +479,8 @@ if(thriftpy3)
390479
install(CODE "
391480
execute_process(
392481
COMMAND ${CMAKE_COMMAND} -E env
393-
\"CFLAGS=${CMAKE_C_FLAGS}\"
394-
\"CXXFLAGS=${CMAKE_CXX_FLAGS}\"
482+
\"CFLAGS=${CMAKE_C_FLAGS} ${FBTHRIFT_CXX_OPTIONS}\"
483+
\"CXXFLAGS=${CMAKE_CXX_FLAGS} ${FBTHRIFT_CXX_OPTIONS}\"
395484
python3 ${CMAKE_CURRENT_SOURCE_DIR}/setup.py install -v
396485
--prefix ${py_install_dir} --libpython ${python_libname}
397486
COMMAND_ECHO STDOUT

0 commit comments

Comments
 (0)