diff --git a/Project.toml b/Project.toml index cf07adca3..5af70d25f 100644 --- a/Project.toml +++ b/Project.toml @@ -88,7 +88,6 @@ SIRUS = "cdeec39e-fb35-4959-aadb-a1dd5dede958" SelfOrganizingMaps = "ba4b7379-301a-4be0-bee6-171e4e152787" StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3" Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb" -SymbolicRegression = "8254be44-1295-4e6a-a16d-46603ac705cb" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] @@ -125,7 +124,6 @@ test = [ "PartitionedLS", "SelfOrganizingMaps", "SIRUS", -"SymbolicRegression", "StableRNGs", "Suppressor", "Test", diff --git a/test/integration.jl b/test/integration.jl index 5c505e3ff..ab804f604 100644 --- a/test/integration.jl +++ b/test/integration.jl @@ -8,12 +8,17 @@ const OTHER_TEST_LEVEL = 3 # # IMPORTANT -# There are two main ways to flag a problem model for integration test purposes. +# There are three main ways to flag a problem model for integration test purposes. # - Adding to `FILTER_GIVEN_ISSUE` means the model is allowed to fail silently, unless # tests pass, a fact that will be reported in the log. -# - Adding to `PATHOLOGIES` completely excludes the model from testing. +# - Adding to `PATHOLOGIES` excludes the model from testing (although there will still be +# an attempt to load the model type, unless the package providing it is excluded from +# the MLJ/Project.toml) + +# - Completetely exclude all models provided by a particular package from testing, by +# excluding the package from the [extras] section of MLJ/Project.toml. # Obviously the first method is strongly preferred. @@ -52,9 +57,34 @@ FILTER_GIVEN_ISSUE = Dict( ) +# # SEE IF PROJECT FILE INCLUDES ALL MODEL-PROVIDING PACKAGES + +# helper; `project_lines` are lines from a Project.toml file: +function pkgs(project_lines) + project = TOML.parse(join(project_lines, "\n")) + headings = Set(keys(project)) ∩ ["deps", "extras"] + return vcat(collect.(keys.([project[h] for h in headings]))...) +end + +# identify missing pkgs: +project_path = joinpath(@__DIR__, "..", "Project.toml") +project_lines = open(project_path) do io + readlines(io) +end +pkgs_in_project = pkgs(project_lines) +registry_project_lines = MLJModels.Registry.registry_project() +pkgs_in_registry = pkgs(registry_project_lines) +missing_pkgs = setdiff(pkgs_in_registry, pkgs_in_project) +# If there are missing packages a warning is issued at the end + + # # LOG OUTSTANDING ISSUES TO STDOUT -const MODELS = models(); +const MODELS = filter(models()) do m + pkg = m.package_name + api_pkg = split(m.load_path, '.') |> first + api_pkg in pkgs_in_project +end const JULIA_MODELS = filter(m->m.is_pure_julia, MODELS); const OTHER_MODELS = setdiff(MODELS, JULIA_MODELS); @@ -158,32 +188,6 @@ end WITHOUT_DATASETS = vcat(WITHOUT_DATASETS, PATHOLOGIES) -# # CHECK PROJECT FILE INCLUDES ALL MODEL-PROVIDING PACKAGES - -# helper; `project_lines` are lines from a Project.toml file: -function pkgs(project_lines) - project = TOML.parse(join(project_lines, "\n")) - headings = Set(keys(project)) ∩ ["deps", "extras"] - return vcat(collect.(keys.([project[h] for h in headings]))...) -end - -# identify missing pkgs: -project_path = joinpath(@__DIR__, "..", "Project.toml") -project_lines = open(project_path) do io - readlines(io) -end -pkgs_in_project = pkgs(project_lines) -registry_project_lines = MLJModels.Registry.registry_project() -pkgs_in_registry = pkgs(registry_project_lines) -missing_pkgs = setdiff(pkgs_in_registry, pkgs_in_project) - -# throw error if there are any: -isempty(missing_pkgs) || error( - "Integration tests cannot proceed because the following packages are "* - "missing from the [extras] section of the MLJ Project.toml file: "* - join(missing_pkgs, ", ") -) - # # LOAD ALL MODEL CODE # Load all the model providing packages with a broad level=1 test: @@ -236,10 +240,14 @@ for (model_set, level) in [ end end +isempty(missing_pkgs) || @warn "Integration tests for the following packages in the "* + "model registry were omitted, as they do not have an entry in the [extras] "* + "section of the MLJ Project.toml file: "*join(missing_pkgs, ", ") okay = isempty(problems) -okay || print("Integration tests failed for these models: \n $problems") -println() +okay || @error "Integration tests failed for these models: \n $problems" +println() +# throw error if there are any issues: @test okay true