Skip to content

Commit

Permalink
Scan preprocessed code for implicit dependencies
Browse files Browse the repository at this point in the history
Fixes: #5
  • Loading branch information
hasselmm committed Aug 23, 2024
1 parent f06ca95 commit d5968af
Show file tree
Hide file tree
Showing 5 changed files with 249 additions and 3 deletions.
2 changes: 2 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ add_custom_target(
toolchain/Arduino/RulesOverride.cmake
toolchain/Arduino/ScriptMode.cmake
toolchain/Platform/Arduino.cmake
toolchain/Scripts/CollectLibraries.cmake
toolchain/Scripts/Preprocess.cmake
toolchain/Templates/ArduinoLibraryCMakeLists.txt.in
toolchain/Templates/CollectLibrariesConfig.cmake.in
toolchain/Templates/PreprocessConfig.cmake.in
toolchain/arduino-cli-toolchain.cmake)

Expand Down
178 changes: 178 additions & 0 deletions toolchain/Scripts/CollectLibraries.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
include(Arduino/ScriptMode NO_POLICY_SCOPE)
include("${ARGUMENTS}" OPTIONAL)

arduino_script_require(
COLLECT_LIBRARIES_CACHE
COLLECT_LIBRARIES_TARGET
COLLECT_LIBRARIES_OUTPUT
COLLECT_LIBRARIES_SOURCES)

# ----------------------------------------------------------------------------------------------------------------------
# Extracts include directives from the files in `SOURCES_LIST` and stores the filename list in `OUTPUT_VARIABLE`.
# ----------------------------------------------------------------------------------------------------------------------
function(__arduino_collect_required_libraries SOURCES_LIST OUTPUT_VARIABLE)
unset(_required_includes)

foreach(_filepath IN LISTS SOURCES_LIST)
message(STATUS " Scanning ${_filepath} for include directives")

# FIXME actually run preprocessor on the file to handle conditional includes (#ifdef/#else)
file(STRINGS "${_filepath}" _include_directives REGEX "^${_hash}[ /t]*include")

foreach(_line IN LISTS _include_directives)
if (_line MATCHES "${_hash}[ /t]*include[ /t]*<([^>]+\\.[Hh])>")
list(APPEND _required_includes "${CMAKE_MATCH_1}")
endif()
endforeach()
endforeach()

list(REMOVE_DUPLICATES _required_includes)
list(REMOVE_ITEM _required_includes "Arduino.h")

set("${OUTPUT_VARIABLE}" ${_required_includes} PARENT_SCOPE)
endfunction()

# ----------------------------------------------------------------------------------------------------------------------
# Filters the known installed libraries by `LOCATION` and reports the result in `OUTPUT_VARIABLE`.
# The result is a list of `(type, name, dirpath)` tuples separated by '|'.
# ----------------------------------------------------------------------------------------------------------------------
function(__arduino_find_installed_libraries LOCATION OUTPUT_VARIABLE)
message(VERBOSE "Reading installed libraries from ${COLLECT_LIBRARIES_CACHE}")
file(READ "${COLLECT_LIBRARIES_CACHE}" _installed_libraries)

string(JSON _installed_libraries GET "${_installed_libraries}" "installed_libraries")
string(JSON _count LENGTH "${_installed_libraries}")
math(EXPR _last "${_count} - 1")

unset(_library_list)

foreach(_library_index RANGE ${_last})
string(JSON _dirpath GET "${_installed_libraries}" ${_library_index} "library" "source_dir")
string(JSON _type GET "${_installed_libraries}" ${_library_index} "library" "location")
string(JSON _name GET "${_installed_libraries}" ${_library_index} "library" "name")

cmake_path(NORMAL_PATH _dirpath)
list(APPEND _library_list "${_type}|${_name}|${_dirpath}")
endforeach()

set("${OUTPUT_VARIABLE}" ${_library_list} PARENT_SCOPE)
endfunction()

# ----------------------------------------------------------------------------------------------------------------------
# Splits a `LIBRARY` tuple into its components `(type, name, dirpath)`.
# ----------------------------------------------------------------------------------------------------------------------
function(__arduino_split_library_tuple LIBRARY TYPE_VARIABLE NAME_VARIABLE DIRPATH_VARIABLE)
string(REGEX MATCH "^([^|]+)\\|([^|]+)\\|([^|]+)" _ "${_library}")

set("${TYPE_VARIABLE}" "${CMAKE_MATCH_1}" PARENT_SCOPE)
set("${NAME_VARIABLE}" "${CMAKE_MATCH_2}" PARENT_SCOPE)
set("${DIRPATH_VARIABLE}" "${CMAKE_MATCH_3}" PARENT_SCOPE)
endfunction()

# ----------------------------------------------------------------------------------------------------------------------
# Tries to find the libraries that provide `REQUIRED_INCLUDES`
# and stores the identified library tuples in `OUTPUT_VARIABLE`.
# ----------------------------------------------------------------------------------------------------------------------
function(__arduino_resolve_libraries REQUIRED_INCLUDES OUTPUT_VARIABLE)
# FIXME Read LINK_LIBRARIES of COLLECT_LIBRARIES_TARGET

# Cluster the libraries reported by arduino-cli by location to implement the location priority
# described by https://arduino.github.io/arduino-cli/1.0/sketch-build-process/#location-priority

__arduino_find_installed_libraries("user" _user_librarys)
__arduino_find_installed_libraries("platform" _platform_librarys)
__arduino_find_installed_libraries("ref-platform" _board_librarys)
__arduino_find_installed_libraries("ide" _ide_libraries)

unset(_resolved_includes)
unset(_required_libraries)
unset(_unresolved_includes)

while (REQUIRED_INCLUDES)
list(POP_FRONT REQUIRED_INCLUDES _next_include)
message(VERBOSE " Searching library that provides ${_next_include}")

unset(_matching_library)

foreach(_library IN LISTS _user_librarys _platform_librarys _board_librarys _ide_libraries)
__arduino_split_library_tuple("${_library}" _type _name _dirpath)
message(TRACE "Checking ${_type} library ${_name} at ${_dirpath}")

if (EXISTS "${_dirpath}/${_next_include}"
AND NOT IS_DIRECTORY "${_dirpath}/${_next_include}")
set(_matching_library "${_library}")
break()
endif()
endforeach()

if (_matching_library)
message(VERBOSE " Using ${_next_include} from ${_type} library ${_name} at ${_dirpath}")

list(APPEND _resolved_includes "${_next_include}")
list(APPEND _required_libraries "${_matching_library}")
else()
list(APPEND _unresolved_includes "${_next_include}")
endif()
endwhile()

if (_unresolved_includes)
list(JOIN _unresolved_includes ", " _unresolved_includes)
message(WARNING "Could not resolve all required libraries. Unresolved includes: ${_unresolved_includes}")
endif()

list(REMOVE_DUPLICATES _required_libraries)
set("${OUTPUT_VARIABLE}" "${_required_libraries}" PARENT_SCOPE)
endfunction()

# ----------------------------------------------------------------------------------------------------------------------
# Generates a CMake file that defines import libraries from `REQUIRED_LIBRARIES` and links `TARGET` with them.
# ----------------------------------------------------------------------------------------------------------------------
function(__arduino_generate_library_definitions TARGET REQUIRED_LIBRARIES OUTPUT_FILEPATH)
set(_library_definitions "# Generated by ${CMAKE_SCRIPT_MODE_FILE}")
unset(_link_libraries)

foreach(_library IN LISTS REQUIRED_LIBRARIES) # <--------------------- generate __arduino_add_import_library() calls
__arduino_split_library_tuple("${_library}" _ _name _dirpath)
string(REPLACE " " "_" _target_name "${_name}")

if (_target_name MATCHES "[^A-Za-z0-9]_")
message(FATAL_ERROR "Unexpected character '${CMAKE_MATCH_0}' in library name '${_name}'")
return()
endif()

list(
APPEND _library_definitions
""
"if (NOT TARGET Arduino::${_target_name})"
" __arduino_add_import_library(${_target_name} \"${_dirpath}\")"
"endif()")

list(APPEND _link_libraries "Arduino::${_target_name}")
endforeach()

if (_link_libraries) # <-------------------------------------------- link `TARGET` with the defined import libraries
list(APPEND _library_definitions
""
"target_link_libraries(\"${TARGET}\" PUBLIC ${_link_libraries})")
endif()

list(JOIN _library_definitions "\n" _library_definitions) # <------------ write the definitions to `OUTPUT_FILEPATH`

if (EXISTS "${OUTPUT_FILEPATH}")
file(READ "${OUTPUT_FILEPATH}" _previous_definitions)
else()
unset(_previous_definitions)
endif()

if (NOT _library_definitions STREQUAL _previous_definitions)
message(STATUS "Generating ${OUTPUT_FILEPATH}")
file(WRITE "${OUTPUT_FILEPATH}" "${_library_definitions}")
endif()
endfunction()

__arduino_collect_required_libraries("${COLLECT_LIBRARIES_SOURCES}" _required_includes)
__arduino_resolve_libraries("${_required_includes}" _required_libraries)

__arduino_generate_library_definitions(
"${COLLECT_LIBRARIES_TARGET}" "${_required_libraries}"
"${COLLECT_LIBRARIES_OUTPUT}")
4 changes: 4 additions & 0 deletions toolchain/Templates/ArduinoLibraryCMakeLists.txt.in
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,7 @@ add_library(
target_include_directories(
${_libname} PUBLIC
"${_quoted_library_directories}")

if (NOT "${_libname}" STREQUAL "ArduinoCore")
target_link_libraries(${_libname} PUBLIC Arduino::Core)
endif()
4 changes: 4 additions & 0 deletions toolchain/Templates/CollectLibrariesConfig.cmake.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
set(COLLECT_LIBRARIES_CACHE "${__ARDUINO_INSTALLED_LIBRARIES_CACHE}")
set(COLLECT_LIBRARIES_OUTPUT "${_required_libraries_include}")
set(COLLECT_LIBRARIES_SOURCES "${_preprocessed_sources_list}")
set(COLLECT_LIBRARIES_TARGET "${TARGET}")
64 changes: 61 additions & 3 deletions toolchain/arduino-cli-toolchain.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -957,7 +957,7 @@ function(__arduino_preprocess OUTPUT_VARIABLE OUTPUT_DIRPATH SOURCE_DIRPATH MODE
"${OUTPUT_DIRPATH}" _output_filepath)

string(MD5 _filepath_hash "${_output_filepath}")
set(_config_filepath "${CMAKE_BINARY_DIR}/ArduinoFiles/${_target}/preprocess-config-${_filepath_hash}.cmake")
set(_config_filepath "${CMAKE_BINARY_DIR}/ArduinoFiles/${_target}/preprocess-${_filepath_hash}.cmake")

__arduino_add_code_generator(
SCRIPT_OUTPUT "${_output_filepath}"
Expand Down Expand Up @@ -996,6 +996,61 @@ function(__arduino_preprocess_sketch TARGET OUTPUT_DIRPATH SOURCE_DIRPATH SOURCE
endforeach()
endfunction()

# ----------------------------------------------------------------------------------------------------------------------
# Remove obsolete files from preprocesses the sources files directory.
# Such step is neccessary as the prebuild hooks of various cores place files in this folder to implement dynamic
# dependency chains. Therefore this folder has to be searched for files to resolve dependencies, instead of using
# a computed list. For this search to succeed previous build-artifacts must be removed.
# ----------------------------------------------------------------------------------------------------------------------
function(__arduino_remove_obsolete_sketch_files TARGET SKETCH_DIRPATH)
set(_source_list_cache "${CMAKE_BINARY_DIR}/ArduinoFiles/${TARGET}/sources.txt")

if (EXISTS "${_source_list_cache}") # <----------------------------------- read target SOURCE list from previous run
file(READ "${_source_list_cache}" _cached_source_list)
else()
unset(_cached_source_list)
endif()

get_property(_source_list TARGET "${TARGET}" PROPERTY SOURCES) # <---------- check if files got removed from SOURCES

list(REMOVE_DUPLICATES _source_list)
list(REMOVE_ITEM _cached_source_list ${_source_list})
file(WRITE "${_source_list_cache}" "${_source_list}")

foreach(_filename IN LISTS _cached_source_list) # <------------------- delete preprocessed files for removed SOURCES
__arduino_resolve_preprocessed_filepath(
"${_source_dirpath}" "${_filename}"
"${SKETCH_DIRPATH}" _sketch_filepath)

message(STATUS "Removing obsolete ${_filename}")
file(REMOVE "${_sketch_filepath}")
endforeach()
endfunction()

# ----------------------------------------------------------------------------------------------------------------------
# Scan the preprocessed source code of `TARGET` for #include directives to detect implicit library dependencies.
# ----------------------------------------------------------------------------------------------------------------------
function(__arduino_link_implicit_libraries TARGET SKETCH_DIRPATH)
# Can't simple use a pre-computed source file list here, but have to list files in SKETCH_DIRPATH
# since the pre-build hooks of some platforms place generated files in the sketch folder to enable
# selective linking of system libraries.
__arduino_remove_obsolete_sketch_files("${_target}" "${SKETCH_DIRPATH}")
__arduino_collect_source_files(_preprocessed_sources_list "${SKETCH_DIRPATH}")

set(_required_libraries_include "${CMAKE_BINARY_DIR}/ArduinoFiles/${_target}/libraries-include.cmake")
target_sources("${_target}" PRIVATE "${_required_libraries_include}")

__arduino_add_code_generator(
SCRIPT_OUTPUT "${_required_libraries_include}"
SCRIPT_FILEPATH "${__ARDUINO_TOOLCHAIN_COLLECT_LIBRARIES}"
CONFIG_TEMPLATE "${ARDUINO_TOOLCHAIN_DIR}/Templates/CollectLibrariesConfig.cmake.in"
CONFIG_FILEPATH "${CMAKE_BINARY_DIR}/ArduinoFiles/${_target}/libraries-config.cmake"
COMMENT "Collecting required libraries for ${TARGET}"
DEPENDS ${_preprocessed_sources_list})

include("${_required_libraries_include}")
endfunction()

# ----------------------------------------------------------------------------------------------------------------------
# Iterates all subdirectories of the project and finalizes Arduino sketches.
# ----------------------------------------------------------------------------------------------------------------------
Expand Down Expand Up @@ -1043,7 +1098,8 @@ function(__arduino_toolchain_finalize DIRECTORY)
"${_target}" "${_sketch_dirpath}"
"${_source_dirpath}" "${_source_list}")

target_link_libraries("${_target}" PUBLIC Arduino::Core)
target_link_libraries("${_target}" PUBLIC Arduino::Core) # <------------- link implicitly required libraries
__arduino_link_implicit_libraries("${_target}" "${_sketch_dirpath}")

set_property(TARGET "${_target}" PROPERTY SUFFIX ".elf") # <----------------------- build the final firmware
__arduino_add_firmware_target("${_target}" _firmware_filename)
Expand Down Expand Up @@ -1100,9 +1156,11 @@ cmake_path(GET CMAKE_CURRENT_LIST_FILE PARENT_PATH ARDUINO_TOOLCHAIN_DIR) # <---
list(APPEND CMAKE_MODULE_PATH ${ARDUINO_TOOLCHAIN_DIR})

set(__ARDUINO_SKETCH_SUFFIX "\\.(ino|pde)\$") # <-------------------------------------------- generally useful constants
set(__ARDUINO_TOOLCHAIN_PREPROCESS "${ARDUINO_TOOLCHAIN_DIR}/Scripts/Preprocess.cmake")
set(__ARDUINO_TOOLCHAIN_COLLECT_LIBRARIES "${ARDUINO_TOOLCHAIN_DIR}/Scripts/CollectLibraries.cmake")
set(__ARDUINO_TOOLCHAIN_PREPROCESS "${ARDUINO_TOOLCHAIN_DIR}/Scripts/Preprocess.cmake")

list(APPEND CMAKE_CONFIGURE_DEPENDS # <------------------------------------- rerun CMake when helper scripts are changed
"${__ARDUINO_TOOLCHAIN_COLLECT_LIBRARIES}"
"${__ARDUINO_TOOLCHAIN_PREPROCESS}")

find_program( # <-------------------------------------------------------------------------------------- find android-cli
Expand Down

0 comments on commit d5968af

Please sign in to comment.